xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/TD-16394

This commit is contained in:
Ganlin Zhao 2022-06-21 19:23:39 +08:00
parent a46abd974b 55c524e811
commit bae7143865
62 changed files with 1748 additions and 584 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
include/client/taos.h
View File

@ -187,8 +187,8 @@ DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res);
DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
DLL_EXPORT const char *taos_get_client_info(); DLL_EXPORT const char *taos_get_client_info();
DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); DLL_EXPORT const char *taos_errstr(TAOS_RES *res);
DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT int taos_errno(TAOS_RES *res);
DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
@ -252,6 +252,16 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
enum tmq_res_t {
TMQ_RES_INVALID = -1,
TMQ_RES_DATA = 1,
TMQ_RES_TABLE_META = 2,
};
typedef enum tmq_res_t tmq_res_t;
DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, const void **raw_meta, int32_t *raw_meta_len);
DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);

1
include/common/tcommon.h
View File

@ -34,6 +34,7 @@ enum {
enum { enum {
TMQ_MSG_TYPE__DUMMY = 0, TMQ_MSG_TYPE__DUMMY = 0,
TMQ_MSG_TYPE__POLL_RSP, TMQ_MSG_TYPE__POLL_RSP,
TMQ_MSG_TYPE__POLL_META_RSP,
TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__EP_RSP,
TMQ_MSG_TYPE__END_RSP, TMQ_MSG_TYPE__END_RSP,
}; };

32
include/common/tmsg.h
View File

@ -1560,6 +1560,7 @@ typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic
int8_t igExists; int8_t igExists;
int8_t subType; int8_t subType;
int8_t withMeta;
char* sql; char* sql;
char subDbName[TSDB_DB_FNAME_LEN]; char subDbName[TSDB_DB_FNAME_LEN];
union { union {
@ -2306,6 +2307,7 @@ typedef struct {
int64_t newConsumerId; int64_t newConsumerId;
char subKey[TSDB_SUBSCRIBE_KEY_LEN]; char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int8_t subType; int8_t subType;
int8_t withMeta;
char* qmsg; char* qmsg;
int64_t suid; int64_t suid;
} SMqRebVgReq; } SMqRebVgReq;
@ -2318,6 +2320,7 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR
tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
tlen += taosEncodeString(buf, pReq->subKey); tlen += taosEncodeString(buf, pReq->subKey);
tlen += taosEncodeFixedI8(buf, pReq->subType); tlen += taosEncodeFixedI8(buf, pReq->subType);
tlen += taosEncodeFixedI8(buf, pReq->withMeta);
if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) {
tlen += taosEncodeString(buf, pReq->qmsg); tlen += taosEncodeString(buf, pReq->qmsg);
} else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
@ -2333,6 +2336,7 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq)
buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
buf = taosDecodeStringTo(buf, pReq->subKey); buf = taosDecodeStringTo(buf, pReq->subKey);
buf = taosDecodeFixedI8(buf, &pReq->subType); buf = taosDecodeFixedI8(buf, &pReq->subType);
buf = taosDecodeFixedI8(buf, &pReq->withMeta);
if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) {
buf = taosDecodeString(buf, &pReq->qmsg); buf = taosDecodeString(buf, &pReq->qmsg);
} else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
@ -2677,6 +2681,34 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
taosArrayDestroy(pSubTopicEp->vgs); taosArrayDestroy(pSubTopicEp->vgs);
} }
typedef struct {
SMqRspHead head;
int64_t reqOffset;
int64_t rspOffset;
int16_t resMsgType;
int32_t metaRspLen;
void* metaRsp;
} SMqMetaRsp;
static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
tlen += taosEncodeFixedI16(buf, pRsp->resMsgType);
tlen += taosEncodeFixedI32(buf, pRsp->metaRspLen);
tlen += taosEncodeBinary(buf, pRsp->metaRsp, pRsp->metaRspLen);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
buf = taosDecodeFixedI16(buf, &pRsp->resMsgType);
buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen);
buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen);
return (void*)buf;
}
typedef struct { typedef struct {
SMqRspHead head; SMqRspHead head;
int64_t reqOffset; int64_t reqOffset;

2
include/common/tmsgdef.h
View File

@ -187,7 +187,6 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqPollReq, SMqDataBlkRsp)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_DISPATCH_WRITE, "vnode-stream-task-dispatch-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_DISPATCH_WRITE, "vnode-stream-task-dispatch-write", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_DROP, "vnode-stream-task-drop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
@ -204,6 +203,7 @@ enum {
//shared by snode and vnode //shared by snode and vnode
TD_NEW_MSG_SEG(TDMT_STREAM_MSG) TD_NEW_MSG_SEG(TDMT_STREAM_MSG)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL)

2
include/libs/nodes/querynodes.h
View File

@ -52,7 +52,7 @@ typedef struct SExprNode {
SArray* pAssociation; SArray* pAssociation;
} SExprNode; } SExprNode;
typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG } EColumnType; typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG, COLUMN_TYPE_TBNAME } EColumnType;
typedef struct SColumnNode { typedef struct SColumnNode {
SExprNode node; // QUERY_NODE_COLUMN SExprNode node; // QUERY_NODE_COLUMN

21
include/libs/stream/tstream.h
View File

@ -30,9 +30,14 @@ extern "C" {
typedef struct SStreamTask SStreamTask; typedef struct SStreamTask SStreamTask;
enum { enum {
TASK_STATUS__IDLE = 1, TASK_STATUS__NORMAL = 0,
TASK_STATUS__EXECUTING, TASK_STATUS__DROPPING,
TASK_STATUS__CLOSING, };
enum {
TASK_EXEC_STATUS__IDLE = 1,
TASK_EXEC_STATUS__EXECUTING,
TASK_EXEC_STATUS__CLOSING,
}; };
enum { enum {
@ -50,16 +55,12 @@ enum {
TASK_OUTPUT_STATUS__BLOCKED, TASK_OUTPUT_STATUS__BLOCKED,
}; };
enum {
STREAM_CREATED_BY__USER = 1,
STREAM_CREATED_BY__SMA,
};
enum { enum {
STREAM_INPUT__DATA_SUBMIT = 1, STREAM_INPUT__DATA_SUBMIT = 1,
STREAM_INPUT__DATA_BLOCK, STREAM_INPUT__DATA_BLOCK,
STREAM_INPUT__TRIGGER, STREAM_INPUT__TRIGGER,
STREAM_INPUT__CHECKPOINT, STREAM_INPUT__CHECKPOINT,
STREAM_INPUT__DROP,
}; };
typedef struct { typedef struct {
@ -237,7 +238,9 @@ struct SStreamTask {
int64_t streamId; int64_t streamId;
int32_t taskId; int32_t taskId;
int8_t inputType; int8_t inputType;
int8_t status; int8_t taskStatus;
int8_t execStatus;
int8_t execType; int8_t execType;
int8_t sinkType; int8_t sinkType;

62
source/client/inc/clientInt.h
View File

@ -20,9 +20,9 @@
extern "C" { extern "C" {
#endif #endif
#include "catalog.h"
#include "parser.h" #include "parser.h"
#include "planner.h" #include "planner.h"
#include "catalog.h"
#include "query.h" #include "query.h"
#include "taos.h" #include "taos.h"
#include "tcommon.h" #include "tcommon.h"
@ -51,10 +51,12 @@ extern "C" {
enum { enum {
RES_TYPE__QUERY = 1, RES_TYPE__QUERY = 1,
RES_TYPE__TMQ, RES_TYPE__TMQ,
RES_TYPE__TMQ_META,
}; };
#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY)
#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) #define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ)
#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META)
typedef struct SAppInstInfo SAppInstInfo; typedef struct SAppInstInfo SAppInstInfo;
@ -66,9 +68,9 @@ typedef struct {
int64_t reportBytes; // not implemented int64_t reportBytes; // not implemented
int64_t startTime; int64_t startTime;
// ctl // ctl
SRWLatch lock; // lock is used in serialization SRWLatch lock; // lock is used in serialization
SAppInstInfo* pAppInstInfo; SAppInstInfo* pAppInstInfo;
SHashObj* activeInfo; // hash<SClientHbKey, SClientHbReq> SHashObj* activeInfo; // hash<SClientHbKey, SClientHbReq>
} SAppHbMgr; } SAppHbMgr;
typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
@ -76,13 +78,13 @@ typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp);
typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req);
typedef struct { typedef struct {
int8_t inited; int8_t inited;
int64_t appId; int64_t appId;
// ctl // ctl
int8_t threadStop; int8_t threadStop;
TdThread thread; TdThread thread;
TdThreadMutex lock; // used when app init and cleanup TdThreadMutex lock; // used when app init and cleanup
SHashObj *appSummary; SHashObj* appSummary;
SArray* appHbMgrs; // SArray<SAppHbMgr*> one for each cluster SArray* appHbMgrs; // SArray<SAppHbMgr*> one for each cluster
FHbReqHandle reqHandle[CONN_TYPE__MAX]; FHbReqHandle reqHandle[CONN_TYPE__MAX];
FHbRspHandle rspHandle[CONN_TYPE__MAX]; FHbRspHandle rspHandle[CONN_TYPE__MAX];
@ -129,7 +131,7 @@ typedef struct STscObj {
int8_t connType; int8_t connType;
int32_t acctId; int32_t acctId;
uint32_t connId; uint32_t connId;
TAOS *id; // ref ID returned by taosAddRef TAOS* id; // ref ID returned by taosAddRef
TdThreadMutex mutex; // used to protect the operation on db TdThreadMutex mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj bound to this connection int32_t numOfReqs; // number of sqlObj bound to this connection
SAppInstInfo* pAppInfo; SAppInstInfo* pAppInfo;
@ -188,6 +190,14 @@ typedef struct {
SReqResultInfo resInfo; SReqResultInfo resInfo;
} SMqRspObj; } SMqRspObj;
typedef struct {
int8_t resType;
char topic[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t vgId;
SMqMetaRsp metaRsp;
} SMqMetaRspObj;
typedef struct SRequestObj { typedef struct SRequestObj {
int8_t resType; // query or tmq int8_t resType; // query or tmq
uint64_t requestId; uint64_t requestId;
@ -206,9 +216,9 @@ typedef struct SRequestObj {
SRequestSendRecvBody body; SRequestSendRecvBody body;
bool stableQuery; bool stableQuery;
bool killed; bool killed;
uint32_t prevCode; //previous error code: todo refactor, add update flag for catalog uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog
uint32_t retry; uint32_t retry;
} SRequestObj; } SRequestObj;
typedef struct SSyncQueryParam { typedef struct SSyncQueryParam {
@ -216,15 +226,15 @@ typedef struct SSyncQueryParam {
SRequestObj* pRequest; SRequestObj* pRequest;
} SSyncQueryParam; } SSyncQueryParam;
void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4);
void doSetOneRowPtr(SReqResultInfo* pResultInfo); void doSetOneRowPtr(SReqResultInfo* pResultInfo);
void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision);
int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4,
bool freeAfterUse); bool freeAfterUse);
void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols);
void doFreeReqResultInfo(SReqResultInfo* pResInfo); void doFreeReqResultInfo(SReqResultInfo* pResInfo);
SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen); SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen);
static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) {
@ -289,7 +299,7 @@ bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType);
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db,
uint16_t port, int connType); uint16_t port, int connType);
SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen); SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen);
@ -299,7 +309,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest);
void taos_close_internal(void *taos); void taos_close_internal(void* taos);
// --- heartbeat // --- heartbeat
// global, called by mgmt // global, called by mgmt
@ -320,12 +330,12 @@ void hbMgrInitMqHbRspHandle();
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res);
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta); void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta);
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta); void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
int32_t removeMeta(STscObj* pTscObj, SArray* tbList);// todo move to clientImpl.c and become a static function int32_t removeMeta(STscObj* pTscObj, SArray* tbList); // todo move to clientImpl.c and become a static function
int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);// todo move to xxx int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); // todo move to xxx
bool qnodeRequired(SRequestObj* pRequest); bool qnodeRequired(SRequestObj* pRequest);
#ifdef __cplusplus #ifdef __cplusplus

111
source/client/src/clientMain.c
View File

@ -17,6 +17,7 @@
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "clientStmt.h" #include "clientStmt.h"
#include "functionMgt.h"
#include "os.h" #include "os.h"
#include "query.h" #include "query.h"
#include "scheduler.h" #include "scheduler.h"
@ -25,7 +26,6 @@
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
#include "version.h" #include "version.h"
#include "functionMgt.h"
#define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
@ -97,11 +97,11 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
pass = TSDB_DEFAULT_PASS; pass = TSDB_DEFAULT_PASS;
} }
STscObj* pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY);
if (pObj) { if (pObj) {
return pObj->id; return pObj->id;
} }
return NULL; return NULL;
} }
@ -111,41 +111,40 @@ void taos_close_internal(void *taos) {
} }
STscObj *pTscObj = (STscObj *)taos; STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", *(int64_t*)pTscObj->id, pTscObj->numOfReqs); tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", *(int64_t *)pTscObj->id, pTscObj->numOfReqs);
taosRemoveRef(clientConnRefPool, *(int64_t*)pTscObj->id); taosRemoveRef(clientConnRefPool, *(int64_t *)pTscObj->id);
} }
void taos_close(TAOS *taos) { void taos_close(TAOS *taos) {
if (taos == NULL) { if (taos == NULL) {
return; return;
} }
STscObj* pObj = acquireTscObj(*(int64_t*)taos); STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (NULL == pObj) { if (NULL == pObj) {
return; return;
} }
taos_close_internal(pObj); taos_close_internal(pObj);
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
taosMemoryFree(taos); taosMemoryFree(taos);
} }
int taos_errno(TAOS_RES *res) {
int taos_errno(TAOS_RES *tres) { if (res == NULL || TD_RES_TMQ_META(res)) {
if (tres == NULL) {
return terrno; return terrno;
} }
if (TD_RES_TMQ(tres)) { if (TD_RES_TMQ(res)) {
return 0; return 0;
} }
return ((SRequestObj *)tres)->code; return ((SRequestObj *)res)->code;
} }
const char *taos_errstr(TAOS_RES *res) { const char *taos_errstr(TAOS_RES *res) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
return (const char *)tstrerror(terrno); return (const char *)tstrerror(terrno);
} }
@ -179,11 +178,15 @@ void taos_free_result(TAOS_RES *res) {
if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
pRsp->resInfo.pRspMsg = NULL; pRsp->resInfo.pRspMsg = NULL;
doFreeReqResultInfo(&pRsp->resInfo); doFreeReqResultInfo(&pRsp->resInfo);
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res;
taosMemoryFree(pRspObj->metaRsp.metaRsp);
taosMemoryFree(pRspObj);
} }
} }
int taos_field_count(TAOS_RES *res) { int taos_field_count(TAOS_RES *res) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
return 0; return 0;
} }
@ -194,7 +197,7 @@ int taos_field_count(TAOS_RES *res) {
int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); } int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
if (taos_num_fields(res) == 0) { if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res)) {
return NULL; return NULL;
} }
@ -215,8 +218,8 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL; return NULL;
} }
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
if (pTscObj == NULL || sql == NULL) { if (pTscObj == NULL || sql == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL; return NULL;
@ -229,21 +232,21 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) {
taos_query_a(taos, sql, syncQueryFn, param); taos_query_a(taos, sql, syncQueryFn, param);
tsem_wait(&param->sem); tsem_wait(&param->sem);
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
return param->pRequest; return param->pRequest;
#else #else
size_t sqlLen = strlen(sql); size_t sqlLen = strlen(sql);
if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) {
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
return NULL; return NULL;
} }
TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen); TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen);
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
return pRes; return pRes;
#endif #endif
@ -380,7 +383,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
} }
int *taos_fetch_lengths(TAOS_RES *res) { int *taos_fetch_lengths(TAOS_RES *res) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
return NULL; return NULL;
} }
@ -389,7 +392,7 @@ int *taos_fetch_lengths(TAOS_RES *res) {
} }
TAOS_ROW *taos_result_block(TAOS_RES *res) { TAOS_ROW *taos_result_block(TAOS_RES *res) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return NULL; return NULL;
} }
@ -438,7 +441,7 @@ const char *taos_data_type(int type) {
const char *taos_get_client_info() { return version; } const char *taos_get_client_info() { return version; }
int taos_affected_rows(TAOS_RES *res) { int taos_affected_rows(TAOS_RES *res) {
if (res == NULL || TD_RES_TMQ(res)) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) {
return 0; return 0;
} }
@ -448,7 +451,7 @@ int taos_affected_rows(TAOS_RES *res) {
} }
int taos_result_precision(TAOS_RES *res) { int taos_result_precision(TAOS_RES *res) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
return TSDB_TIME_PRECISION_MILLI; return TSDB_TIME_PRECISION_MILLI;
} }
@ -463,15 +466,15 @@ int taos_result_precision(TAOS_RES *res) {
} }
int taos_select_db(TAOS *taos, const char *db) { int taos_select_db(TAOS *taos, const char *db) {
STscObj* pObj = acquireTscObj(*(int64_t*)taos); STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (pObj == NULL) { if (pObj == NULL) {
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return TSDB_CODE_TSC_DISCONNECTED; return TSDB_CODE_TSC_DISCONNECTED;
} }
if (db == NULL || strlen(db) == 0) { if (db == NULL || strlen(db) == 0) {
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
terrno = TSDB_CODE_TSC_INVALID_INPUT; terrno = TSDB_CODE_TSC_INVALID_INPUT;
return terrno; return terrno;
} }
@ -483,19 +486,19 @@ int taos_select_db(TAOS *taos, const char *db) {
int32_t code = taos_errno(pRequest); int32_t code = taos_errno(pRequest);
taos_free_result(pRequest); taos_free_result(pRequest);
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
return code; return code;
} }
void taos_stop_query(TAOS_RES *res) { void taos_stop_query(TAOS_RES *res) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) {
return; return;
} }
SRequestObj *pRequest = (SRequestObj *)res; SRequestObj *pRequest = (SRequestObj *)res;
pRequest->killed = true; pRequest->killed = true;
int32_t numOfFields = taos_num_fields(pRequest); int32_t numOfFields = taos_num_fields(pRequest);
// It is not a query, no need to stop. // It is not a query, no need to stop.
if (numOfFields == 0) { if (numOfFields == 0) {
tscDebug("request %" PRIx64 " no need to be killed since not query", pRequest->requestId); tscDebug("request %" PRIx64 " no need to be killed since not query", pRequest->requestId);
@ -510,6 +513,9 @@ void taos_stop_query(TAOS_RES *res) {
} }
bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
if (res == NULL || TD_RES_TMQ_META(res)) {
return true;
}
SReqResultInfo *pResultInfo = tscGetCurResInfo(res); SReqResultInfo *pResultInfo = tscGetCurResInfo(res);
if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) { if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) {
return true; return true;
@ -532,7 +538,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
} }
int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
return 0; return 0;
} }
@ -575,7 +581,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) {
} }
int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
return 0; return 0;
} }
@ -615,7 +621,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) {
} }
int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
if (res == NULL) { if (res == NULL || TD_RES_TMQ_META(res)) {
return 0; return 0;
} }
@ -636,7 +642,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) {
int taos_validate_sql(TAOS *taos, const char *sql) { return true; } int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
void taos_reset_current_db(TAOS *taos) { void taos_reset_current_db(TAOS *taos) {
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
if (pTscObj == NULL) { if (pTscObj == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return; return;
@ -644,17 +650,17 @@ void taos_reset_current_db(TAOS *taos) {
resetConnectDB(pTscObj); resetConnectDB(pTscObj);
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
} }
const char *taos_get_server_info(TAOS *taos) { const char *taos_get_server_info(TAOS *taos) {
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
if (pTscObj == NULL) { if (pTscObj == NULL) {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL; return NULL;
} }
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
return pTscObj->ver; return pTscObj->ver;
} }
@ -682,7 +688,7 @@ static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) {
void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
tscDebug("enter meta callback, code %s", tstrerror(code)); tscDebug("enter meta callback, code %s", tstrerror(code));
SqlParseWrapper *pWrapper = (SqlParseWrapper *)param; SqlParseWrapper *pWrapper = (SqlParseWrapper *)param;
SQuery *pQuery = pWrapper->pQuery; SQuery *pQuery = pWrapper->pQuery;
SRequestObj *pRequest = pWrapper->pRequest; SRequestObj *pRequest = pWrapper->pRequest;
@ -723,11 +729,11 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) {
} }
void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) {
STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); STscObj *pTscObj = acquireTscObj(*(int64_t *)taos);
if (pTscObj == NULL || sql == NULL || NULL == fp) { if (pTscObj == NULL || sql == NULL || NULL == fp) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
if (pTscObj) { if (pTscObj) {
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
} else { } else {
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
} }
@ -745,7 +751,7 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param
} }
SRequestObj *pRequest = NULL; SRequestObj *pRequest = NULL;
int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
fp(param, NULL, terrno); fp(param, NULL, terrno);
@ -849,8 +855,8 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
SReqResultInfo *pResultInfo = &pRequest->body.resInfo; SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
pRequest->self, code, tstrerror(code), pRequest->requestId); tstrerror(code), pRequest->requestId);
pResultInfo->pData = pResult; pResultInfo->pData = pResult;
pResultInfo->numOfRows = 0; pResultInfo->numOfRows = 0;
@ -884,6 +890,7 @@ static void fetchCallback(void *pResult, void *param, int32_t code) {
void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
ASSERT(res != NULL && fp != NULL); ASSERT(res != NULL && fp != NULL);
ASSERT(TD_RES_QUERY(res));
SRequestObj *pRequest = res; SRequestObj *pRequest = res;
pRequest->body.fetchFp = fp; pRequest->body.fetchFp = fp;
@ -910,6 +917,7 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
ASSERT(res != NULL && fp != NULL); ASSERT(res != NULL && fp != NULL);
ASSERT(TD_RES_QUERY(res));
SRequestObj *pRequest = res; SRequestObj *pRequest = res;
pRequest->body.resInfo.convertUcs4 = false; pRequest->body.resInfo.convertUcs4 = false;
@ -923,6 +931,7 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) {
const void *taos_get_raw_block(TAOS_RES *res) { const void *taos_get_raw_block(TAOS_RES *res) {
ASSERT(res != NULL); ASSERT(res != NULL);
ASSERT(TD_RES_QUERY(res));
SRequestObj *pRequest = res; SRequestObj *pRequest = res;
return pRequest->body.resInfo.pData; return pRequest->body.resInfo.pData;
@ -949,16 +958,16 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
} }
TAOS_STMT *taos_stmt_init(TAOS *taos) { TAOS_STMT *taos_stmt_init(TAOS *taos) {
STscObj* pObj = acquireTscObj(*(int64_t*)taos); STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (NULL == pObj) { if (NULL == pObj) {
tscError("invalid parameter for %s", __FUNCTION__); tscError("invalid parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_TSC_DISCONNECTED; terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL; return NULL;
} }
TAOS_STMT* pStmt = stmtInit(pObj); TAOS_STMT *pStmt = stmtInit(pObj);
releaseTscObj(*(int64_t*)taos); releaseTscObj(*(int64_t *)taos);
return pStmt; return pStmt;
} }

8
source/client/src/clientSml.c
View File

@ -2389,17 +2389,19 @@ static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) {
static void smlInsertCallback(void *param, void *res, int32_t code) { static void smlInsertCallback(void *param, void *res, int32_t code) {
SRequestObj *pRequest = (SRequestObj *)res; SRequestObj *pRequest = (SRequestObj *)res;
SSmlHandle *info = (SSmlHandle *)param; SSmlHandle *info = (SSmlHandle *)param;
int32_t rows = taos_affected_rows(pRequest);
uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf);
// lock // lock
taosThreadSpinLock(&info->params->lock);
info->params->request->body.resInfo.numOfRows += rows;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosThreadSpinLock(&info->params->lock);
info->params->request->code = code; info->params->request->code = code;
taosThreadSpinUnlock(&info->params->lock);
} }
taosThreadSpinUnlock(&info->params->lock);
// unlock // unlock
printf("SML:0x%" PRIx64 " insert finished, code: %d, total: %d\n", info->id, code, info->affectedRows); uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows);
Params *pParam = info->params; Params *pParam = info->params;
bool isLast = info->isLast; bool isLast = info->isLast;
info->cost.endTime = taosGetTimestampUs(); info->cost.endTime = taosGetTimestampUs();

112
source/client/src/tmq.c
View File

@ -149,7 +149,10 @@ typedef struct {
int32_t epoch; int32_t epoch;
SMqClientVg* vgHandle; SMqClientVg* vgHandle;
SMqClientTopic* topicHandle; SMqClientTopic* topicHandle;
SMqDataBlkRsp msg; union {
SMqDataBlkRsp dataRsp;
SMqMetaRsp metaRsp;
};
} SMqPollRspWrapper; } SMqPollRspWrapper;
typedef struct { typedef struct {
@ -401,8 +404,17 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
int32_t code = -1; int32_t code = -1;
if (msg != NULL) { if (msg != NULL) {
SMqRspObj* pRspObj = (SMqRspObj*)msg; char* topic;
if (!TD_RES_TMQ(pRspObj)) { int32_t vgId;
if (TD_RES_TMQ(msg)) {
SMqRspObj* pRspObj = (SMqRspObj*)msg;
topic = pRspObj->topic;
vgId = pRspObj->vgId;
} else if (TD_RES_TMQ_META(msg)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg;
topic = pMetaRspObj->topic;
vgId = pMetaRspObj->vgId;
} else {
return TSDB_CODE_TMQ_INVALID_MSG; return TSDB_CODE_TMQ_INVALID_MSG;
} }
@ -421,10 +433,10 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_
for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (strcmp(pTopic->topicName, pRspObj->topic) == 0) { if (strcmp(pTopic->topicName, topic) == 0) {
for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
if (pVg->vgId == pRspObj->vgId) { if (pVg->vgId == vgId) {
if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) {
tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld",
tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset);
@ -1131,6 +1143,11 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch);
} }
// handle meta rsp
int8_t rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType;
if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) {
}
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM);
if (pRspWrapper == NULL) { if (pRspWrapper == NULL) {
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
@ -1138,17 +1155,23 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
goto CREATE_MSG_FAIL; goto CREATE_MSG_FAIL;
} }
pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP; pRspWrapper->tmqRspType = rspType;
pRspWrapper->vgHandle = pVg; pRspWrapper->vgHandle = pVg;
pRspWrapper->topicHandle = pTopic; pRspWrapper->topicHandle = pTopic;
memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead));
if (rspType == TMQ_MSG_TYPE__POLL_RSP) {
tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);
} else {
ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP);
tDecodeSMqMetaRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->metaRsp);
}
tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg);
taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pData);
tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId,
pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset);
taosWriteQitem(tmq->mqueue, pRspWrapper); taosWriteQitem(tmq->mqueue, pRspWrapper);
tsem_post(&tmq->rspSem); tsem_post(&tmq->rspSem);
@ -1516,6 +1539,17 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic*
return pReq; return pReq;
} }
SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj));
pRspObj->resType = RES_TYPE__TMQ;
tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN);
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
pRspObj->vgId = pWrapper->vgHandle->vgId;
memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp));
return pRspObj;
}
SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj));
pRspObj->resType = RES_TYPE__TMQ; pRspObj->resType = RES_TYPE__TMQ;
@ -1523,11 +1557,11 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN);
pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->vgId = pWrapper->vgHandle->vgId;
pRspObj->resIter = -1; pRspObj->resIter = -1;
memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp)); memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataBlkRsp));
pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.totalRows = 0;
pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
if (!pWrapper->msg.withSchema) { if (!pWrapper->dataRsp.withSchema) {
setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
} }
@ -1643,12 +1677,12 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
/*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/
int32_t consumerEpoch = atomic_load_32(&tmq->epoch); int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
if (pollRspWrapper->msg.head.epoch == consumerEpoch) { if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
SMqClientVg* pVg = pollRspWrapper->vgHandle; SMqClientVg* pVg = pollRspWrapper->vgHandle;
/*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
pVg->currentOffset = pollRspWrapper->msg.rspOffset; pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
if (pollRspWrapper->msg.blockNum == 0) { if (pollRspWrapper->dataRsp.blockNum == 0) {
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
rspWrapper = NULL; rspWrapper = NULL;
continue; continue;
@ -1658,8 +1692,25 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
return pRsp; return pRsp;
} else { } else {
tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", pollRspWrapper->msg.head.epoch, tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n",
consumerEpoch); pollRspWrapper->dataRsp.head.epoch, consumerEpoch);
taosFreeQitem(pollRspWrapper);
}
} else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) {
SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper;
int32_t consumerEpoch = atomic_load_32(&tmq->epoch);
if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) {
SMqClientVg* pVg = pollRspWrapper->vgHandle;
/*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/
pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset;
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// build rsp
SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper);
taosFreeQitem(pollRspWrapper);
return pRsp;
} else {
tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n",
pollRspWrapper->dataRsp.head.epoch, consumerEpoch);
taosFreeQitem(pollRspWrapper); taosFreeQitem(pollRspWrapper);
} }
} else { } else {
@ -1747,10 +1798,23 @@ const char* tmq_err2str(int32_t err) {
} }
} }
tmq_res_t tmq_get_res_type(TAOS_RES* res) {
if (TD_RES_TMQ(res)) {
return TMQ_RES_DATA;
} else if (TD_RES_TMQ_META(res)) {
return TMQ_RES_TABLE_META;
} else {
return TMQ_RES_INVALID;
}
}
const char* tmq_get_topic_name(TAOS_RES* res) { const char* tmq_get_topic_name(TAOS_RES* res) {
if (TD_RES_TMQ(res)) { if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res; SMqRspObj* pRspObj = (SMqRspObj*)res;
return strchr(pRspObj->topic, '.') + 1; return strchr(pRspObj->topic, '.') + 1;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
return strchr(pMetaRspObj->topic, '.') + 1;
} else { } else {
return NULL; return NULL;
} }
@ -1760,6 +1824,9 @@ const char* tmq_get_db_name(TAOS_RES* res) {
if (TD_RES_TMQ(res)) { if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res; SMqRspObj* pRspObj = (SMqRspObj*)res;
return strchr(pRspObj->db, '.') + 1; return strchr(pRspObj->db, '.') + 1;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
return strchr(pMetaRspObj->db, '.') + 1;
} else { } else {
return NULL; return NULL;
} }
@ -1769,6 +1836,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
if (TD_RES_TMQ(res)) { if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res; SMqRspObj* pRspObj = (SMqRspObj*)res;
return pRspObj->vgId; return pRspObj->vgId;
} else if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
return pMetaRspObj->vgId;
} else { } else {
return -1; return -1;
} }
@ -1786,6 +1856,16 @@ const char* tmq_get_table_name(TAOS_RES* res) {
return NULL; return NULL;
} }
int32_t tmq_get_raw_meta(TAOS_RES* res, const void** raw_meta, int32_t* raw_meta_len) {
if (TD_RES_TMQ_META(res)) {
SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res;
*raw_meta = pMetaRspObj->metaRsp.metaRsp;
*raw_meta_len = pMetaRspObj->metaRsp.metaRspLen;
return 0;
}
return -1;
}
void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) {
tmqCommitInner2(tmq, msg, 0, 1, cb, param); tmqCommitInner2(tmq, msg, 0, 1, cb, param);
} }

8
source/client/test/smlTest.cpp
View File

@ -1272,10 +1272,10 @@ TEST(testCase, sml_dup_time_Test) {
const char *sql[] = { const char *sql[] = {
//"test_ms,t0=t c0=f 1626006833641", //"test_ms,t0=t c0=f 1626006833641",
"ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000",
"ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000",
"ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000",
"ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000",
"ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"
}; };
pRes = taos_query(taos, "use dup_time"); pRes = taos_query(taos, "use dup_time");
taos_free_result(pRes); taos_free_result(pRes);

6
source/common/src/tmsg.c
View File

@ -127,7 +127,7 @@ int32_t tEncodeSEpSet(SEncoder *pEncoder, const SEpSet *pEp) {
if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1;
for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) {
if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1; if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1;
if (tEncodeCStr(pEncoder, pEp->eps[i].fqdn) < 0) return -1; if (tEncodeCStrWithLen(pEncoder, pEp->eps[i].fqdn, TSDB_FQDN_LEN) < 0) return -1;
} }
return 0; return 0;
} }
@ -2956,6 +2956,7 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
if (tEncodeI8(&encoder, pReq->subType) < 0) return -1; if (tEncodeI8(&encoder, pReq->subType) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withMeta) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->subDbName) < 0) return -1; if (tEncodeCStr(&encoder, pReq->subDbName) < 0) return -1;
if (TOPIC_SUB_TYPE__DB == pReq->subType) { if (TOPIC_SUB_TYPE__DB == pReq->subType) {
} else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) {
@ -2985,6 +2986,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->subType) < 0) return -1; if (tDecodeI8(&decoder, &pReq->subType) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withMeta) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->subDbName) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->subDbName) < 0) return -1;
if (TOPIC_SUB_TYPE__DB == pReq->subType) { if (TOPIC_SUB_TYPE__DB == pReq->subType) {
} else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) {
@ -3052,7 +3054,7 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) {
if (tEncodeCStr(&encoder, pReq->app) < 0) return -1; if (tEncodeCStr(&encoder, pReq->app) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->passwd) < 0) return -1; if (tEncodeCStrWithLen(&encoder, pReq->passwd, TSDB_PASSWORD_LEN) < 0) return -1;
if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1; if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);

15
source/common/src/tname.c
View File

@ -312,8 +312,10 @@ void buildChildTableName(RandTableName* rName) {
taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen);
taosArraySort(rName->tags, compareKv); taosArraySort(rName->tags, compareKv);
for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
taosStringBuilderAppendChar(&sb, ',');
SSmlKv* tagKv = taosArrayGetP(rName->tags, j); SSmlKv* tagKv = taosArrayGetP(rName->tags, j);
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
taosStringBuilderAppendChar(&sb, '=');
if (IS_VAR_DATA_TYPE(tagKv->type)) { if (IS_VAR_DATA_TYPE(tagKv->type)) {
taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->length); taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->length);
} else { } else {
@ -326,9 +328,14 @@ void buildChildTableName(RandTableName* rName) {
tMD5Init(&context); tMD5Init(&context);
tMD5Update(&context, (uint8_t*)keyJoined, (uint32_t)len); tMD5Update(&context, (uint8_t*)keyJoined, (uint32_t)len);
tMD5Final(&context); tMD5Final(&context);
uint64_t digest1 = *(uint64_t*)(context.digest);
uint64_t digest2 = *(uint64_t*)(context.digest + 8); char temp[8] = {0};
snprintf(rName->childTableName, TSDB_TABLE_NAME_LEN, "t_%016" PRIx64 "%016" PRIx64, digest1, digest2); rName->childTableName[0] = 't';
rName->childTableName[1] = '_';
for(int i = 0; i < 16; i++){
sprintf(temp, "%02x", context.digest[i]);
strcat(rName->childTableName, temp);
}
taosStringBuilderDestroy(&sb); taosStringBuilderDestroy(&sb);
rName->uid = digest1; rName->uid = *(uint64_t*)(context.digest);
} }

14
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -73,6 +73,7 @@ int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
SDCreateMnodeReq createReq = {0}; SDCreateMnodeReq createReq = {0};
if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
@ -81,7 +82,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
if (createReq.replica != 1) { if (createReq.replica != 1) {
terrno = TSDB_CODE_INVALID_OPTION; terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to create mnode since %s", terrstr()); dGError("failed to create mnode since %s", terrstr());
return -1; return -1;
} }
@ -91,7 +92,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
mgmt.path = pInput->path; mgmt.path = pInput->path;
mgmt.name = pInput->name; mgmt.name = pInput->name;
if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) {
dError("failed to write mnode file since %s", terrstr()); dGError("failed to write mnode file since %s", terrstr());
return -1; return -1;
} }
@ -99,7 +100,8 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
} }
int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
SDDropMnodeReq dropReq = {0}; const STraceId *trace = &pMsg->info.traceId;
SDDropMnodeReq dropReq = {0};
if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return -1; return -1;
@ -107,7 +109,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to drop mnode since %s", terrstr()); dGError("failed to drop mnode since %s", terrstr());
return -1; return -1;
} }
@ -117,7 +119,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) {
mgmt.path = pInput->path; mgmt.path = pInput->path;
mgmt.name = pInput->name; mgmt.name = pInput->name;
if (mmWriteFile(&mgmt, NULL, deployed) != 0) { if (mmWriteFile(&mgmt, NULL, deployed) != 0) {
dError("failed to write mnode file since %s", terrstr()); dGError("failed to write mnode file since %s", terrstr());
return -1; return -1;
} }
@ -216,7 +218,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

26
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -18,7 +18,6 @@
static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) { static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) {
int32_t code = 0; int32_t code = 0;
taosThreadRwlockRdlock(&pMgmt->lock); taosThreadRwlockRdlock(&pMgmt->lock);
if (pMgmt->stopped) { if (pMgmt->stopped) {
code = -1; code = -1;
@ -48,7 +47,8 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) {
static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SMnodeMgmt *pMgmt = pInfo->ahandle; SMnodeMgmt *pMgmt = pInfo->ahandle;
int32_t code = -1; int32_t code = -1;
STraceId * trace = &pMsg->info.traceId;
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, get from mnode queue", pMsg); dGTrace("msg:%p, get from mnode queue", pMsg);
switch (pMsg->msgType) { switch (pMsg->msgType) {
@ -72,7 +72,7 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
mndPostProcessQueryMsg(pMsg); mndPostProcessQueryMsg(pMsg);
} }
dTrace("msg:%p, is freed, code:0x%x", pMsg, code); dGTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
@ -80,7 +80,9 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
SMnodeMgmt *pMgmt = pInfo->ahandle; SMnodeMgmt *pMgmt = pInfo->ahandle;
pMsg->info.node = pMgmt->pMnode; pMsg->info.node = pMgmt->pMnode;
dTrace("msg:%p, get from mnode-sync queue", pMsg);
const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, get from mnode-sync queue", pMsg);
SMsgHead *pHead = pMsg->pCont; SMsgHead *pHead = pMsg->pCont;
pHead->contLen = ntohl(pHead->contLen); pHead->contLen = ntohl(pHead->contLen);
@ -88,20 +90,22 @@ static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) {
int32_t code = mndProcessSyncMsg(pMsg); int32_t code = mndProcessSyncMsg(pMsg);
dTrace("msg:%p, is freed, code:0x%x", pMsg, code); dGTrace("msg:%p, is freed, code:0x%x", pMsg, code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pMsg) { static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pMsg) {
const STraceId *trace = &pMsg->info.traceId;
if (mmAcquire(pMgmt) == 0) { if (mmAcquire(pMgmt) == 0) {
dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); dGTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType));
taosWriteQitem(pWorker->queue, pMsg); taosWriteQitem(pWorker->queue, pMsg);
mmRelease(pMgmt); mmRelease(pMgmt);
return 0; return 0;
} else { } else {
dTrace("msg:%p, failed to put into %s queue since %s, type:%s", pMsg, pWorker->name, terrstr(), dGTrace("msg:%p, failed to put into %s queue since %s, type:%s", pMsg, pWorker->name, terrstr(),
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
return -1; return -1;
} }
} }
@ -121,19 +125,17 @@ int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
pMsg->info.node = pMgmt->pMnode; pMsg->info.node = pMgmt->pMnode;
if (mndPreProcessQueryMsg(pMsg) != 0) { if (mndPreProcessQueryMsg(pMsg) != 0) {
dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); const STraceId *trace = &pMsg->info.traceId;
dGError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
return -1; return -1;
} }
return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg);
} }
int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
pMsg->info.node = pMgmt->pMnode;
return mmPutMsgToWorker(pMgmt, &pMgmt->fetchWorker, pMsg); return mmPutMsgToWorker(pMgmt, &pMgmt->fetchWorker, pMsg);
} }
int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg);
} }

2
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -95,7 +95,7 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MON_SM_INFO, smPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MON_SM_INFO, smPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -351,7 +351,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;

163
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -17,9 +17,44 @@
#include "dmMgmt.h" #include "dmMgmt.h"
#include "qworker.h" #include "qworker.h"
static void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); static inline void dmSendRsp(SRpcMsg *pMsg) {
static void dmSendRsp(SRpcMsg *pMsg); SMgmtWrapper *pWrapper = pMsg->info.wrapper;
static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); if (InChildProc(pWrapper)) {
dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP);
} else {
rpcSendResponse(pMsg);
}
}
static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) {
SEpSet epSet = {0};
dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet);
const int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
pMsg->pCont = rpcMallocCont(contLen);
if (pMsg->pCont == NULL) {
pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
} else {
tSerializeSEpSet(pMsg->pCont, contLen, &epSet);
pMsg->contLen = contLen;
}
}
static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) {
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet);
rsp.pCont = rpcMallocCont(contLen);
if (rsp.pCont == NULL) {
pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
} else {
tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet);
rsp.contLen = contLen;
}
dmSendRsp(&rsp);
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)]; NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)];
@ -28,31 +63,38 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
return -1; return -1;
} }
dTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); const STraceId *trace = &pMsg->info.traceId;
dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name);
pMsg->info.wrapper = pWrapper; pMsg->info.wrapper = pWrapper;
return (*msgFp)(pWrapper->pMgmt, pMsg); return (*msgFp)(pWrapper->pMgmt, pMsg);
} }
static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
SDnodeTrans * pTrans = &pDnode->trans; SDnodeTrans *pTrans = &pDnode->trans;
int32_t code = -1; int32_t code = -1;
SRpcMsg * pMsg = NULL; SRpcMsg *pMsg = NULL;
SMgmtWrapper *pWrapper = NULL; SMgmtWrapper *pWrapper = NULL;
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)];
STraceId *trace = &pRpc->info.traceId; const STraceId *trace = &pRpc->info.traceId;
dGTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), dGTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType),
pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId);
if (pRpc->msgType == TDMT_DND_NET_TEST) { switch (pRpc->msgType) {
dmProcessNetTestReq(pDnode, pRpc); case TDMT_DND_NET_TEST:
return; dmProcessNetTestReq(pDnode, pRpc);
} else if (pRpc->msgType == TDMT_MND_SYSTABLE_RETRIEVE_RSP || pRpc->msgType == TDMT_VND_FETCH_RSP) { return;
qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0); case TDMT_MND_SYSTABLE_RETRIEVE_RSP:
return; case TDMT_VND_FETCH_RSP:
} else if (pRpc->msgType == TDMT_MND_STATUS_RSP && pEpSet != NULL) { qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0);
dmSetMnodeEpSet(&pDnode->data, pEpSet); return;
} else { case TDMT_MND_STATUS_RSP:
if (pEpSet != NULL) {
dmSetMnodeEpSet(&pDnode->data, pEpSet);
}
break;
default:
break;
} }
if (pDnode->status != DND_STAT_RUNNING) { if (pDnode->status != DND_STAT_RUNNING) {
@ -73,39 +115,43 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
if (pHandle->defaultNtype == NODE_END) { if (pHandle->defaultNtype == NODE_END) {
terrno = TSDB_CODE_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
goto _OVER; goto _OVER;
} else { }
pWrapper = &pDnode->wrappers[pHandle->defaultNtype];
if (pHandle->needCheckVgId) { pWrapper = &pDnode->wrappers[pHandle->defaultNtype];
if (pRpc->contLen > 0) { if (pHandle->needCheckVgId) {
SMsgHead *pHead = pRpc->pCont; if (pRpc->contLen > 0) {
int32_t vgId = ntohl(pHead->vgId); const SMsgHead *pHead = pRpc->pCont;
if (vgId == QNODE_HANDLE) { const int32_t vgId = ntohl(pHead->vgId);
switch (vgId) {
case QNODE_HANDLE:
pWrapper = &pDnode->wrappers[QNODE]; pWrapper = &pDnode->wrappers[QNODE];
} else if (vgId == SNODE_HANDLE) { break;
case SNODE_HANDLE:
pWrapper = &pDnode->wrappers[SNODE]; pWrapper = &pDnode->wrappers[SNODE];
} else if (vgId == MNODE_HANDLE) { break;
case MNODE_HANDLE:
pWrapper = &pDnode->wrappers[MNODE]; pWrapper = &pDnode->wrappers[MNODE];
} else { break;
} default:
} else { break;
terrno = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER;
} }
} else {
terrno = TSDB_CODE_INVALID_MSG_LEN;
goto _OVER;
} }
} }
if (dmMarkWrapper(pWrapper) != 0) { if (dmMarkWrapper(pWrapper) != 0) {
pWrapper = NULL; pWrapper = NULL;
goto _OVER; goto _OVER;
} else {
pRpc->info.wrapper = pWrapper;
} }
pRpc->info.wrapper = pWrapper;
pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM);
if (pMsg == NULL) goto _OVER; if (pMsg == NULL) goto _OVER;
memcpy(pMsg, pRpc, sizeof(SRpcMsg));
dTrace("msg:%p, is created, type:%s handle:%p", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle); memcpy(pMsg, pRpc, sizeof(SRpcMsg));
dGTrace("msg:%p, is created, type:%s handle:%p", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle);
if (InParentProc(pWrapper)) { if (InParentProc(pWrapper)) {
code = dmPutToProcCQueue(&pWrapper->proc, pMsg, DND_FUNC_REQ); code = dmPutToProcCQueue(&pWrapper->proc, pMsg, DND_FUNC_REQ);
@ -115,13 +161,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
dTrace("failed to process msg:%p since %s, handle:%p", pMsg, terrstr(), pRpc->info.handle);
if (terrno != 0) code = terrno; if (terrno != 0) code = terrno;
dGTrace("msg:%p, failed to process since %s", pMsg, terrstr());
if (IsReq(pRpc)) { if (IsReq(pRpc)) {
SRpcMsg rsp = {.code = code, .info = pRpc->info}; SRpcMsg rsp = {.code = code, .info = pRpc->info};
if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG && if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG &&
pRpc->msgType < TDMT_VND_MSG) { pRpc->msgType < TDMT_VND_MSG) {
dmBuildMnodeRedirectRsp(pDnode, &rsp); dmBuildMnodeRedirectRsp(pDnode, &rsp);
@ -135,7 +179,7 @@ _OVER:
} }
if (pMsg != NULL) { if (pMsg != NULL) {
dTrace("msg:%p, is freed", pMsg); dGTrace("msg:%p, is freed", pMsg);
taosFreeQitem(pMsg); taosFreeQitem(pMsg);
} }
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
@ -149,11 +193,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
SArray * pArray = (*pWrapper->func.getHandlesFp)(); SArray *pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1; if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SMgmtHandle * pMgmt = taosArrayGet(pArray, i); SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) { if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId; pHandle->needCheckVgId = pMgmt->needCheckVgId;
@ -184,45 +228,6 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
} }
} }
static inline void dmSendRsp(SRpcMsg *pMsg) {
SMgmtWrapper *pWrapper = pMsg->info.wrapper;
if (InChildProc(pWrapper)) {
dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP);
} else {
rpcSendResponse(pMsg);
}
}
static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) {
SEpSet epSet = {0};
dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet);
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
pMsg->pCont = rpcMallocCont(contLen);
if (pMsg->pCont == NULL) {
pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
} else {
tSerializeSEpSet(pMsg->pCont, contLen, &epSet);
pMsg->contLen = contLen;
}
}
static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) {
SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info};
int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet);
rsp.pCont = rpcMallocCont(contLen);
if (rsp.pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
} else {
tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet);
rsp.contLen = contLen;
}
dmSendRsp(&rsp);
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) {
SMgmtWrapper *pWrapper = pMsg->info.wrapper; SMgmtWrapper *pWrapper = pMsg->info.wrapper;
if (InChildProc(pWrapper)) { if (InChildProc(pWrapper)) {

26
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -40,18 +40,27 @@
#include "wal.h" #include "wal.h"
#include "libs/function/function.h" #include "libs/function/function.h"
// clang-format off
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} // clang-format off
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} #define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} #define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} #define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} #define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define dGTrace(param, ...) do { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ",GTID: %s", __VA_ARGS__, buf);} while(0) #define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
#define dGFatal(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);}
#define dGError(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);}
#define dGWarn(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn (param ", gtid:%s", __VA_ARGS__, buf);}
#define dGInfo(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo (param ", gtid:%s", __VA_ARGS__, buf);}
#define dGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", gtid:%s", __VA_ARGS__, buf);}
#define dGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", gtid:%s", __VA_ARGS__, buf);}
// clang-format on
typedef enum { typedef enum {
DNODE = 0, DNODE = 0,
@ -185,4 +194,3 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
#endif #endif
#endif /*_TD_DM_INT_H_*/ #endif /*_TD_DM_INT_H_*/
// clang-format on

1
source/dnode/mgmt/test/sut/src/sut.cpp
View File

@ -102,6 +102,7 @@ int32_t Testbase::SendShowReq(int8_t showType, const char* tb, const char* db) {
ASSERT(pRsp->pCont != nullptr); ASSERT(pRsp->pCont != nullptr);
if (pRsp->contLen == 0) return -1; if (pRsp->contLen == 0) return -1;
if (pRsp->code != 0) return -1;
showRsp = (SRetrieveMetaTableRsp*)pRsp->pCont; showRsp = (SRetrieveMetaTableRsp*)pRsp->pCont;
showRsp->handle = htobe64(showRsp->handle); // show Id showRsp->handle = htobe64(showRsp->handle); // show Id

4
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -420,7 +420,8 @@ typedef struct {
int64_t uid; int64_t uid;
int64_t dbUid; int64_t dbUid;
int32_t version; int32_t version;
int8_t subType; // column, db or stable int8_t subType; // column, db or stable
int8_t withMeta; // TODO
SRWLatch lock; SRWLatch lock;
int32_t sqlLen; int32_t sqlLen;
int32_t astLen; int32_t astLen;
@ -487,6 +488,7 @@ typedef struct {
int64_t dbUid; int64_t dbUid;
int32_t vgNum; int32_t vgNum;
int8_t subType; int8_t subType;
int8_t withMeta;
int64_t stbUid; int64_t stbUid;
SHashObj* consumerHash; // consumerId -> SMqConsumerEp SHashObj* consumerHash; // consumerId -> SMqConsumerEp
SArray* unassignedVgs; // SArray<SMqVgEp*> SArray* unassignedVgs; // SArray<SMqVgEp*>

21
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -34,13 +34,20 @@ extern "C" {
#endif #endif
// clang-format off // clang-format off
#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} #define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} #define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }} #define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} #define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }}
#define mGTrace(param, ...) do { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", GTID: %s", __VA_ARGS__, buf);} while(0) #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }}
#define mGFatal(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", gtid:%s", __VA_ARGS__, buf);}
#define mGError(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", gtid:%s", __VA_ARGS__, buf);}
#define mGWarn(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", gtid:%s", __VA_ARGS__, buf);}
#define mGInfo(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);}
#define mGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);}
#define mGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);}
// clang-format on // clang-format on

3
source/dnode/mnode/impl/src/mndDef.c
View File

@ -381,6 +381,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->dbUid = pSub->dbUid; pSubNew->dbUid = pSub->dbUid;
pSubNew->stbUid = pSub->stbUid; pSubNew->stbUid = pSub->stbUid;
pSubNew->subType = pSub->subType; pSubNew->subType = pSub->subType;
pSubNew->withMeta = pSub->withMeta;
pSubNew->vgNum = pSub->vgNum; pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
@ -414,6 +415,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
tlen += taosEncodeFixedI64(buf, pSub->dbUid); tlen += taosEncodeFixedI64(buf, pSub->dbUid);
tlen += taosEncodeFixedI32(buf, pSub->vgNum); tlen += taosEncodeFixedI32(buf, pSub->vgNum);
tlen += taosEncodeFixedI8(buf, pSub->subType); tlen += taosEncodeFixedI8(buf, pSub->subType);
tlen += taosEncodeFixedI8(buf, pSub->withMeta);
tlen += taosEncodeFixedI64(buf, pSub->stbUid); tlen += taosEncodeFixedI64(buf, pSub->stbUid);
void *pIter = NULL; void *pIter = NULL;
@ -440,6 +442,7 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
buf = taosDecodeFixedI64(buf, &pSub->dbUid); buf = taosDecodeFixedI64(buf, &pSub->dbUid);
buf = taosDecodeFixedI32(buf, &pSub->vgNum); buf = taosDecodeFixedI32(buf, &pSub->vgNum);
buf = taosDecodeFixedI8(buf, &pSub->subType); buf = taosDecodeFixedI8(buf, &pSub->subType);
buf = taosDecodeFixedI8(buf, &pSub->withMeta);
buf = taosDecodeFixedI64(buf, &pSub->stbUid); buf = taosDecodeFixedI64(buf, &pSub->stbUid);
int32_t sz; int32_t sz;

78
source/dnode/mnode/impl/src/mndMain.c
View File

@ -59,22 +59,28 @@ static void *mndBuildTimerMsg(int32_t *pContLen) {
static void mndPullupTrans(SMnode *pMnode) { static void mndPullupTrans(SMnode *pMnode) {
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; if (pReq != NULL) {
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
} }
static void mndCalMqRebalance(SMnode *pMnode) { static void mndCalMqRebalance(SMnode *pMnode) {
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; if (pReq != NULL) {
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
}
} }
static void mndPullupTelem(SMnode *pMnode) { static void mndPullupTelem(SMnode *pMnode) {
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; if (pReq != NULL) {
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
}
} }
static void mndPushTtlTime(SMnode *pMnode) { static void mndPushTtlTime(SMnode *pMnode) {
@ -89,10 +95,11 @@ static void mndPushTtlTime(SMnode *pMnode) {
int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t); int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t);
SMsgHead *pHead = rpcMallocCont(contLen); SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) { if (pHead == NULL) {
mError("ttl time malloc err. contLen:%d", contLen); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
continue; continue;
} }
pHead->contLen = htonl(contLen); pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId); pHead->vgId = htonl(pVgroup->vgId);
@ -100,13 +107,13 @@ static void mndPushTtlTime(SMnode *pMnode) {
*(int32_t *)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t); *(int32_t *)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen}; SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg); int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) { if (code != 0) {
mError("ttl time seed err. code:%d", code); mError("failed to send ttl time seed msg, code:0x%x", code);
} else {
mInfo("send ttl time seed msg, time:%d", t);
} }
mError("ttl time seed succ. time:%d", t);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
} }
@ -117,7 +124,7 @@ static void *mndThreadFp(void *param) {
setThreadName("mnode-timer"); setThreadName("mnode-timer");
while (1) { while (1) {
if (lastTime % (864000) == 0) { // sleep 1 day for ttl if (lastTime % 864000 == 0) {
mndPushTtlTime(pMnode); mndPushTtlTime(pMnode);
} }
@ -549,23 +556,25 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) {
static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { static int32_t mndCheckMnodeState(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0; if (!IsReq(pMsg)) return 0;
if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0;
if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER ||
pMsg->msgType == TDMT_MND_TRANS_TIMER) {
return -1;
}
if (pMsg->msgType != TDMT_MND_MQ_TIMER && pMsg->msgType != TDMT_MND_TELEM_TIMER && const STraceId *trace = &pMsg->info.traceId;
pMsg->msgType != TDMT_MND_TRANS_TIMER) { mGError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType));
SEpSet epSet = {0}; SEpSet epSet = {0};
mndGetMnodeEpSet(pMsg->info.node, &epSet); mndGetMnodeEpSet(pMsg->info.node, &epSet);
int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet);
pMsg->info.rsp = rpcMallocCont(contLen); pMsg->info.rsp = rpcMallocCont(contLen);
if (pMsg->info.rsp != NULL) { if (pMsg->info.rsp != NULL) {
tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet);
pMsg->info.rspLen = contLen; pMsg->info.rspLen = contLen;
terrno = TSDB_CODE_RPC_REDIRECT; terrno = TSDB_CODE_RPC_REDIRECT;
} else { } else {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
}
} }
return -1; return -1;
@ -575,17 +584,20 @@ static int32_t mndCheckMsgContent(SRpcMsg *pMsg) {
if (!IsReq(pMsg)) return 0; if (!IsReq(pMsg)) return 0;
if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0; if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0;
mError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen, const STraceId *trace = &pMsg->info.traceId;
mGError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen,
pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_INVALID_MSG_LEN; terrno = TSDB_CODE_INVALID_MSG_LEN;
return -1; return -1;
} }
int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
const STraceId *trace = &pMsg->info.traceId;
MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)];
if (fp == NULL) { if (fp == NULL) {
mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
terrno = TSDB_CODE_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
return -1; return -1;
} }
@ -593,18 +605,17 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) {
if (mndCheckMsgContent(pMsg) != 0) return -1; if (mndCheckMsgContent(pMsg) != 0) return -1;
if (mndCheckMnodeState(pMsg) != 0) return -1; if (mndCheckMnodeState(pMsg) != 0) return -1;
STraceId *trace = &pMsg->info.traceId;
mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType));
int32_t code = (*fp)(pMsg); int32_t code = (*fp)(pMsg);
mndReleaseRpcRef(pMnode); mndReleaseRpcRef(pMnode);
if (code == TSDB_CODE_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_ACTION_IN_PROGRESS) {
mTrace("msg:%p, won't response immediately since in progress", pMsg); mGTrace("msg:%p, won't response immediately since in progress", pMsg);
} else if (code == 0) { } else if (code == 0) {
mTrace("msg:%p, successfully processed", pMsg); mGTrace("msg:%p, successfully processed", pMsg);
} else { } else {
mError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle,
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
} }
return code; return code;
@ -620,7 +631,6 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
// Note: uid 0 is reserved // Note: uid 0 is reserved
int64_t mndGenerateUid(char *name, int32_t len) { int64_t mndGenerateUid(char *name, int32_t len) {
int32_t hashval = MurmurHash3_32(name, len); int32_t hashval = MurmurHash3_32(name, len);
do { do {
int64_t us = taosGetTimestampUs(); int64_t us = taosGetTimestampUs();
int64_t x = (us & 0x000000FFFFFFFFFF) << 24; int64_t x = (us & 0x000000FFFFFFFFFF) << 24;

99
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -122,30 +122,33 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType
int32_t pid, const char *app, int64_t startTime) { int32_t pid, const char *app, int64_t startTime) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
char connStr[255] = {0}; char connStr[255] = {0};
int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app);
uint32_t connId = mndGenerateUid(connStr, len); uint32_t connId = mndGenerateUid(connStr, len);
if (startTime == 0) startTime = taosGetTimestampMs(); if (startTime == 0) startTime = taosGetTimestampMs();
SConnObj connObj = {.id = connId, SConnObj connObj = {
.connType = connType, .id = connId,
.appStartTimeMs = startTime, .connType = connType,
.pid = pid, .appStartTimeMs = startTime,
.ip = ip, .pid = pid,
.port = port, .ip = ip,
.killed = 0, .port = port,
.loginTimeMs = taosGetTimestampMs(), .killed = 0,
.lastAccessTimeMs = 0, .loginTimeMs = taosGetTimestampMs(),
.killId = 0, .lastAccessTimeMs = 0,
.numOfQueries = 0, .killId = 0,
.pQueries = NULL}; .numOfQueries = 0,
.pQueries = NULL,
};
connObj.lastAccessTimeMs = connObj.loginTimeMs; connObj.lastAccessTimeMs = connObj.loginTimeMs;
tstrncpy(connObj.user, user, TSDB_USER_LEN); tstrncpy(connObj.user, user, TSDB_USER_LEN);
tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN);
int32_t keepTime = tsShellActivityTimer * 3; int32_t keepTime = tsShellActivityTimer * 3;
SConnObj *pConn = taosCachePut(pMgmt->connCache, &connId, sizeof(uint32_t), &connObj, sizeof(connObj), keepTime * 1000); SConnObj *pConn =
taosCachePut(pMgmt->connCache, &connId, sizeof(uint32_t), &connObj, sizeof(connObj), keepTime * 1000);
if (pConn == NULL) { if (pConn == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr()); mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr());
@ -174,7 +177,6 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId) {
} }
pConn->lastAccessTimeMs = taosGetTimestampMs(); pConn->lastAccessTimeMs = taosGetTimestampMs();
mTrace("conn:%u, acquired from cache, data:%p", pConn->id, pConn); mTrace("conn:%u, acquired from cache, data:%p", pConn->id, pConn);
return pConn; return pConn;
} }
@ -207,13 +209,14 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
} }
static int32_t mndProcessConnectReq(SRpcMsg *pReq) { static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SUserObj *pUser = NULL; SUserObj *pUser = NULL;
SDbObj *pDb = NULL; SDbObj *pDb = NULL;
SConnObj *pConn = NULL; SConnObj *pConn = NULL;
int32_t code = -1; int32_t code = -1;
SConnectReq connReq = {0}; SConnectReq connReq = {0};
char ip[30] = {0}; char ip[30] = {0};
const STraceId *trace = &pReq->info.traceId;
if (tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq) != 0) { if (tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
@ -224,11 +227,11 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
pUser = mndAcquireUser(pMnode, pReq->info.conn.user); pUser = mndAcquireUser(pMnode, pReq->info.conn.user);
if (pUser == NULL) { if (pUser == NULL) {
mError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr()); mGError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr());
goto CONN_OVER; goto CONN_OVER;
} }
if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
mError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd); mGError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd);
code = TSDB_CODE_RPC_AUTH_FAILURE; code = TSDB_CODE_RPC_AUTH_FAILURE;
goto CONN_OVER; goto CONN_OVER;
} }
@ -239,8 +242,8 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
pDb = mndAcquireDb(pMnode, db); pDb = mndAcquireDb(pMnode, db);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_INVALID_DB; terrno = TSDB_CODE_MND_INVALID_DB;
mError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db, mGError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db,
terrstr()); terrstr());
goto CONN_OVER; goto CONN_OVER;
} }
} }
@ -248,7 +251,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp, pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp,
pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime); pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime);
if (pConn == NULL) { if (pConn == NULL) {
mError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr()); mGError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr());
goto CONN_OVER; goto CONN_OVER;
} }
@ -273,7 +276,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) {
pReq->info.rspLen = contLen; pReq->info.rspLen = contLen;
pReq->info.rsp = pRsp; pReq->info.rsp = pRsp;
mDebug("user:%s, login from %s:%d, conn:%u, app:%s", pReq->info.conn.user, ip, pConn->port, pConn->id, connReq.app); mGDebug("user:%s, login from %s:%d, conn:%u, app:%s", pReq->info.conn.user, ip, pConn->port, pConn->id, connReq.app);
code = 0; code = 0;
@ -302,7 +305,7 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq* pReq) { static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq *pReq) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SAppObj app; SAppObj app;
@ -314,22 +317,19 @@ static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq* pReq)
memcpy(&app.summary, &pReq->summary, sizeof(pReq->summary)); memcpy(&app.summary, &pReq->summary, sizeof(pReq->summary));
app.lastAccessTimeMs = taosGetTimestampMs(); app.lastAccessTimeMs = taosGetTimestampMs();
int32_t keepTime = tsShellActivityTimer * 3; const int32_t keepTime = tsShellActivityTimer * 3;
SAppObj *pApp = taosCachePut(pMgmt->appCache, &pReq->appId, sizeof(pReq->appId), &app, sizeof(app), keepTime * 1000); SAppObj *pApp = taosCachePut(pMgmt->appCache, &pReq->appId, sizeof(pReq->appId), &app, sizeof(app), keepTime * 1000);
if (pApp == NULL) { if (pApp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to app %" PRIx64 " into cache since %s", pReq->appId, terrstr()); mError("failed to app %" PRIx64 " into cache since %s", pReq->appId, terrstr());
return NULL; return NULL;
} }
mTrace("app %" PRIx64 " is put into cache", pReq->appId); mTrace("app %" PRIx64 " is put into cache", pReq->appId);
return pApp; return pApp;
} }
static void mndFreeApp(SAppObj *pApp) { static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); }
mTrace("app %" PRIx64 " is destroyed", pApp->appId);
}
static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) { static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
@ -356,7 +356,7 @@ static void mndReleaseApp(SMnode *pMnode, SAppObj *pApp) {
void *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { void *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) {
SAppObj *pApp = NULL; SAppObj *pApp = NULL;
bool hasNext = taosCacheIterNext(pIter); bool hasNext = taosCacheIterNext(pIter);
if (hasNext) { if (hasNext) {
size_t dataLen = 0; size_t dataLen = 0;
pApp = taosCacheIterGetData(pIter, &dataLen); pApp = taosCacheIterGetData(pIter, &dataLen);
@ -439,8 +439,8 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) {
} }
static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) { static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) {
SAppHbReq* pReq = &pHbReq->app; SAppHbReq *pReq = &pHbReq->app;
SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId); SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId);
if (pApp == NULL) { if (pApp == NULL) {
pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq); pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq);
if (pApp == NULL) { if (pApp == NULL) {
@ -448,7 +448,7 @@ static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnIn
return -1; return -1;
} else { } else {
mDebug("a new app %" PRIx64 "created", pReq->appId); mDebug("a new app %" PRIx64 "created", pReq->appId);
mndReleaseApp(pMnode, pApp); mndReleaseApp(pMnode, pApp);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
@ -464,7 +464,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
SClientHbBatchRsp *pBatchRsp) { SClientHbBatchRsp *pBatchRsp) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
SRpcConnInfo connInfo = pMsg->info.conn; SRpcConnInfo connInfo = pMsg->info.conn;
mndUpdateAppInfo(pMnode, pHbReq, &connInfo); mndUpdateAppInfo(pMnode, pHbReq, &connInfo);
@ -637,9 +637,9 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) {
} }
mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); mInfo("kill query msg is received, queryId:%s", killReq.queryStrId);
int32_t connId = 0; int32_t connId = 0;
uint64_t queryId = 0; uint64_t queryId = 0;
char* p = strchr(killReq.queryStrId, ':'); char *p = strchr(killReq.queryStrId, ':');
if (NULL == p) { if (NULL == p) {
mError("invalid query id %s", killReq.queryStrId); mError("invalid query id %s", killReq.queryStrId);
terrno = TSDB_CODE_MND_INVALID_QUERY_ID; terrno = TSDB_CODE_MND_INVALID_QUERY_ID;
@ -853,12 +853,12 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p
} }
static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
int32_t cols = 0; int32_t cols = 0;
SAppObj *pApp = NULL; SAppObj *pApp = NULL;
if (pShow->pIter == NULL) { if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->appCache); pShow->pIter = taosCacheCreateIter(pMgmt->appCache);
@ -931,7 +931,6 @@ static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo
return numOfRows; return numOfRows;
} }
static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) {
if (pIter != NULL) { if (pIter != NULL) {
taosCacheDestroyIter(pIter); taosCacheDestroyIter(pIter);

37
source/dnode/mnode/impl/src/mndStream.c
View File

@ -54,9 +54,10 @@ int32_t mndInitStream(SMnode *pMnode) {
}; };
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream);
@ -477,7 +478,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet));
action.pCont = pReq; action.pCont = pReq;
action.contLen = sizeof(SVDropStreamTaskReq); action.contLen = sizeof(SVDropStreamTaskReq);
action.msgType = TDMT_VND_STREAM_TASK_DROP; action.msgType = TDMT_STREAM_TASK_DROP;
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -670,20 +671,24 @@ _OVER:
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
/*SDbObj *pDb = NULL;*/ /*SDbObj *pDb = NULL;*/
/*SUserObj *pUser = NULL;*/ /*SUserObj *pUser = NULL;*/
SMDropStreamReq dropReq = *(SMDropStreamReq *)pReq->pCont; SMDropStreamReq dropReq = {0};
if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) {
ASSERT(0);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
pStream = mndAcquireStream(pMnode, dropReq.name); pStream = mndAcquireStream(pMnode, dropReq.name);
if (pStream == NULL) { if (pStream == NULL) {
if (dropReq.igNotExists) { if (dropReq.igNotExists) {
mDebug("stream:%s, not exist, ignore not exist is set", dropReq.name); mDebug("stream:%s, not exist, ignore not exist is set", dropReq.name);
code = 0; sdbRelease(pMnode->pSdb, pStream);
goto DROP_STREAM_OVER; return -1;
} else { } else {
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
return -1; return -1;
@ -701,14 +706,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to drop since %s", dropReq.name, terrstr()); mError("stream:%s, failed to drop since %s", dropReq.name, terrstr());
return code; sdbRelease(pMnode->pSdb, pStream);
return -1;
} }
mDebug("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name); mDebug("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name);
// drop all tasks // drop all tasks
if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr());
return code; sdbRelease(pMnode->pSdb, pStream);
return -1;
} }
// drop stream // drop stream
@ -717,8 +724,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
DROP_STREAM_OVER: if (mndTransPrepare(pMnode, pTrans) != 0) {
return code; mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
sdbRelease(pMnode->pSdb, pStream);
return TSDB_CODE_ACTION_IN_PROGRESS;
} }
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {

2
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -96,6 +96,7 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
pSub->dbUid = pTopic->dbUid; pSub->dbUid = pTopic->dbUid;
pSub->stbUid = pTopic->stbUid; pSub->stbUid = pTopic->stbUid;
pSub->subType = pTopic->subType; pSub->subType = pTopic->subType;
pSub->withMeta = pTopic->withMeta;
ASSERT(pSub->unassignedVgs->size == 0); ASSERT(pSub->unassignedVgs->size == 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0); ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
@ -120,6 +121,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
req.vgId = pRebVg->pVgEp->vgId; req.vgId = pRebVg->pVgEp->vgId;
req.qmsg = pRebVg->pVgEp->qmsg; req.qmsg = pRebVg->pVgEp->qmsg;
req.subType = pSub->subType; req.subType = pSub->subType;
req.withMeta = pSub->withMeta;
req.suid = pSub->stbUid; req.suid = pSub->stbUid;
strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);

6
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -141,6 +141,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withMeta, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->stbUid, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->stbUid, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
@ -208,6 +209,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withMeta, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
@ -357,6 +359,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
topicObj.sql = strdup(pCreate->sql); topicObj.sql = strdup(pCreate->sql);
topicObj.sqlLen = strlen(pCreate->sql) + 1; topicObj.sqlLen = strlen(pCreate->sql) + 1;
topicObj.subType = pCreate->subType; topicObj.subType = pCreate->subType;
topicObj.withMeta = pCreate->withMeta;
if (topicObj.withMeta) {
ASSERT(topicObj.subType != TOPIC_SUB_TYPE__COLUMN);
}
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
topicObj.ast = strdup(pCreate->ast); topicObj.ast = strdup(pCreate->ast);

4
source/dnode/snode/src/snode.c
View File

@ -92,7 +92,7 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) {
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
pTask->status = TASK_STATUS__IDLE; pTask->execStatus = TASK_EXEC_STATUS__IDLE;
pTask->inputQueue = streamQueueOpen(); pTask->inputQueue = streamQueueOpen();
pTask->outputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen();
@ -205,7 +205,7 @@ int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) {
switch (pMsg->msgType) { switch (pMsg->msgType) {
case TDMT_STREAM_TASK_DEPLOY: case TDMT_STREAM_TASK_DEPLOY:
return sndProcessTaskDeployReq(pSnode, pMsg); return sndProcessTaskDeployReq(pSnode, pMsg);
case TDMT_VND_STREAM_TASK_DROP: case TDMT_STREAM_TASK_DROP:
return sndProcessTaskDropReq(pSnode, pMsg); return sndProcessTaskDropReq(pSnode, pMsg);
default: default:
ASSERT(0); ASSERT(0);

1
source/dnode/vnode/src/inc/tq.h
View File

@ -114,6 +114,7 @@ typedef struct {
char subKey[TSDB_SUBSCRIBE_KEY_LEN]; char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId; int64_t consumerId;
int32_t epoch; int32_t epoch;
int8_t fetchMeta;
// reader // reader
SWalReadHandle* pWalReader; SWalReadHandle* pWalReader;

22
source/dnode/vnode/src/inc/vnd.h
View File

@ -26,13 +26,21 @@ extern "C" {
#endif #endif
// clang-format off // clang-format off
#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0)
#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) #define vFatal(...) { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("VND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define vWarn(...) do { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) #define vError(...) { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("VND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define vInfo(...) do { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) #define vWarn(...) { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("VND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }} while(0) #define vInfo(...) { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("VND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }} while(0) #define vDebug(...) { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("VND ", DEBUG_DEBUG, vDebugFlag, __VA_ARGS__); }}
#define vGTrace(param, ...) do { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param " GTID: %s", __VA_ARGS__, buf);} while(0)//#define vDye(...) do #define vTrace(...) { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("VND ", DEBUG_TRACE, vDebugFlag, __VA_ARGS__); }}
#define vGFatal(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vFatal(param ", gtid:%s", __VA_ARGS__, buf);}
#define vGError(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vError(param ", gtid:%s", __VA_ARGS__, buf);}
#define vGWarn(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn (param ", gtid:%s", __VA_ARGS__, buf);}
#define vGInfo(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo (param ", gtid:%s", __VA_ARGS__, buf);}
#define vGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", gtid:%s", __VA_ARGS__, buf);}
#define vGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vTrace(param ", gtid:%s", __VA_ARGS__, buf);}
// clang-format on // clang-format on
// vnodeCfg.c // vnodeCfg.c

88
source/dnode/vnode/src/tq/tq.c
View File

@ -85,6 +85,34 @@ void tqClose(STQ* pTq) {
} }
} }
int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) {
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqMetaRsp(NULL, pRsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
return -1;
}
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP;
((SMqRspHead*)buf)->epoch = pReq->epoch;
((SMqRspHead*)buf)->consumerId = pReq->consumerId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqMetaRsp(&abuf, pRsp);
SRpcMsg resp = {
.info = pMsg->info,
.pCont = buf,
.contLen = tlen,
.code = 0,
};
tmsgSendRsp(&resp);
tqDebug("vg %d from consumer %ld (epoch %d) send rsp, res msg type %d, reqOffset: %ld, rspOffset: %ld",
TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset);
return 0;
}
int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) { int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) {
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp); int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp);
void* buf = rpcMallocCont(tlen); void* buf = rpcMallocCont(tlen);
@ -250,8 +278,23 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
/*ASSERT(0);*/ /*ASSERT(0);*/
} }
} else { } else {
// TODO ASSERT(pHandle->fetchMeta);
ASSERT(0); ASSERT(pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB ||
pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
pHead->msgType == TDMT_VND_DROP_TTL_TABLE);
// return
SMqMetaRsp metaRsp = {0};
metaRsp.reqOffset = pReq->currentOffset;
metaRsp.rspOffset = fetchOffset;
metaRsp.resMsgType = pHead->msgType;
metaRsp.metaRspLen = pHead->bodyLen;
metaRsp.metaRsp = pHead->body;
if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
code = -1;
}
code = 0;
goto OVER;
} }
// TODO batch optimization: // TODO batch optimization:
@ -276,7 +319,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) { if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {
code = -1; code = -1;
} }
OVER:
// TODO wrap in destroy func // TODO wrap in destroy func
taosArrayDestroy(rsp.blockDataLen); taosArrayDestroy(rsp.blockDataLen);
taosArrayDestroyP(rsp.blockData, (FDelete)taosMemoryFree); taosArrayDestroyP(rsp.blockData, (FDelete)taosMemoryFree);
@ -384,7 +427,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
pTask->status = TASK_STATUS__IDLE; pTask->execStatus = TASK_EXEC_STATUS__IDLE;
pTask->inputQueue = streamQueueOpen(); pTask->inputQueue = streamQueueOpen();
pTask->outputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen();
@ -459,6 +502,9 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) {
pIter = taosHashIterate(pTq->pStreamTasks, pIter); pIter = taosHashIterate(pTq->pStreamTasks, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SStreamTask* pTask = *(SStreamTask**)pIter; SStreamTask* pTask = *(SStreamTask**)pIter;
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
continue;
}
if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue; if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue;
if (!failed) { if (!failed) {
@ -487,6 +533,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
SStreamTaskRunReq* pReq = pMsg->pCont; SStreamTaskRunReq* pReq = pMsg->pCont;
int32_t taskId = pReq->taskId; int32_t taskId = pReq->taskId;
SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
return 0;
}
streamProcessRunReq(pTask); streamProcessRunReq(pTask);
return 0; return 0;
} }
@ -501,9 +550,12 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) {
tDecodeStreamDispatchReq(&decoder, &req); tDecodeStreamDispatchReq(&decoder, &req);
int32_t taskId = req.taskId; int32_t taskId = req.taskId;
SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
SRpcMsg rsp = { if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
.info = pMsg->info, return 0;
.code = 0, }
SRpcMsg rsp = {
.info = pMsg->info,
.code = 0,
}; };
streamProcessDispatchReq(pTask, &req, &rsp); streamProcessDispatchReq(pTask, &req, &rsp);
return 0; return 0;
@ -513,6 +565,9 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) {
SStreamTaskRecoverReq* pReq = pMsg->pCont; SStreamTaskRecoverReq* pReq = pMsg->pCont;
int32_t taskId = pReq->taskId; int32_t taskId = pReq->taskId;
SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
return 0;
}
streamProcessRecoverReq(pTask, pReq, pMsg); streamProcessRecoverReq(pTask, pReq, pMsg);
return 0; return 0;
} }
@ -521,6 +576,9 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) {
SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t taskId = pRsp->taskId; int32_t taskId = pRsp->taskId;
SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
return 0;
}
streamProcessDispatchRsp(pTask, pRsp); streamProcessDispatchRsp(pTask, pRsp);
return 0; return 0;
} }
@ -529,16 +587,32 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) {
SStreamTaskRecoverRsp* pRsp = pMsg->pCont; SStreamTaskRecoverRsp* pRsp = pMsg->pCont;
int32_t taskId = pRsp->taskId; int32_t taskId = pRsp->taskId;
SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t));
if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) {
return 0;
}
streamProcessRecoverRsp(pTask, pRsp); streamProcessRecoverRsp(pTask, pRsp);
return 0; return 0;
} }
int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg;
SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t));
atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING);
// todo
// clear queue
// push drop req into queue
// launch exec to free memory
// remove from hash
return 0;
#if 0
int32_t code = taosHashRemove(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t)); int32_t code = taosHashRemove(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t));
// set status dropping
ASSERT(code == 0); ASSERT(code == 0);
if (code == 0) { if (code == 0) {
// sendrsp // sendrsp
} }
return code; return code;
#endif
} }

14
source/dnode/vnode/src/tq/tqPush.c
View File

@ -45,9 +45,9 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm
int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) {
SMqDataBlkRsp rsp = {0}; SMqDataBlkRsp rsp = {0};
// 1. guard and set status executing // 1. guard and set status executing
int8_t execStatus = int8_t execStatus = atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE,
atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); TASK_EXEC_STATUS__EXECUTING);
if (execStatus == TASK_STATUS__IDLE) { if (execStatus == TASK_EXEC_STATUS__IDLE) {
SStreamDataSubmit* pSubmit = NULL; SStreamDataSubmit* pSubmit = NULL;
// 2. check processedVer // 2. check processedVer
// 2.1. if not missed, get msg from queue // 2.1. if not missed, get msg from queue
@ -68,18 +68,18 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) {
goto SEND_RSP; goto SEND_RSP;
} }
// set exec status closing // set exec status closing
atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__CLOSING); atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__CLOSING);
// second run // second run
if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) { if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) {
goto SEND_RSP; goto SEND_RSP;
} }
// set exec status idle // set exec status idle
atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE);
} }
SEND_RSP: SEND_RSP:
// 4. if get result // 4. if get result
// 4.1 set exec input status blocked and exec status idle // 4.1 set exec input status blocked and exec status idle
atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE);
// 4.2 rpc send // 4.2 rpc send
rsp.rspOffset = pHandle->pushHandle.processedVer; rsp.rspOffset = pHandle->pushHandle.processedVer;
/*if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {*/ /*if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {*/
@ -150,7 +150,7 @@ int32_t tqEnqueueAll(STQ* pTq, SSubmitReq* pReq) {
continue; continue;
} }
int8_t execStatus = atomic_load_8(&pHandle->pushHandle.execStatus); int8_t execStatus = atomic_load_8(&pHandle->pushHandle.execStatus);
if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { if (execStatus == TASK_EXEC_STATUS__IDLE || execStatus == TASK_EXEC_STATUS__CLOSING) {
tqSendExecReq(pTq, pHandle); tqSendExecReq(pTq, pHandle);
} }
} }

19
source/dnode/vnode/src/tq/tqRead.c
View File

@ -42,6 +42,25 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead*
code = 0; code = 0;
goto END; goto END;
} else { } else {
if (pHandle->fetchMeta) {
SWalReadHead* pHead = &((*ppHeadWithCkSum)->head);
if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB ||
pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE ||
pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE ||
pHead->msgType == TDMT_VND_DROP_TTL_TABLE) {
code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum);
if (code < 0) {
ASSERT(0);
*fetchOffset = offset;
code = -1;
goto END;
}
*fetchOffset = offset;
code = 0;
goto END;
}
}
code = walSkipFetchBody(pHandle->pWalReader, *ppHeadWithCkSum); code = walSkipFetchBody(pHandle->pWalReader, *ppHeadWithCkSum);
if (code < 0) { if (code < 0) {
ASSERT(0); ASSERT(0);

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -172,7 +172,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp
goto _err; goto _err;
} }
} break; } break;
case TDMT_VND_STREAM_TASK_DROP: { case TDMT_STREAM_TASK_DROP: {
if (tqProcessTaskDropReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) { if (tqProcessTaskDropReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) {
goto _err; goto _err;
} }

41
source/libs/function/src/builtinsimpl.c
View File

@ -195,13 +195,13 @@ typedef struct SMavgInfo {
} SMavgInfo; } SMavgInfo;
typedef struct SSampleInfo { typedef struct SSampleInfo {
int32_t samples; int32_t samples;
int32_t totalPoints; int32_t totalPoints;
int32_t numSampled; int32_t numSampled;
uint8_t colType; uint8_t colType;
int16_t colBytes; int16_t colBytes;
char* data; char* data;
int64_t* timestamp; STuplePos* tuplePos;
} SSampleInfo; } SSampleInfo;
typedef struct STailItem { typedef struct STailItem {
@ -4348,7 +4348,7 @@ bool getSampleFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1);
int32_t numOfSamples = pVal->datum.i; int32_t numOfSamples = pVal->datum.i;
pEnv->calcMemSize = sizeof(SSampleInfo) + numOfSamples * (pCol->node.resType.bytes + sizeof(int64_t)); pEnv->calcMemSize = sizeof(SSampleInfo) + numOfSamples * (pCol->node.resType.bytes + sizeof(STuplePos));
return true; return true;
} }
@ -4369,25 +4369,30 @@ bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo)
return false; return false;
} }
pInfo->data = (char*)pInfo + sizeof(SSampleInfo); pInfo->data = (char*)pInfo + sizeof(SSampleInfo);
pInfo->timestamp = (int64_t*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); pInfo->tuplePos = (STuplePos*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes);
return true; return true;
} }
static void sampleAssignResult(SSampleInfo* pInfo, char* data, TSKEY ts, int32_t index) { static void sampleAssignResult(SSampleInfo* pInfo, char* data, int32_t index) {
assignVal(pInfo->data + index * pInfo->colBytes, data, pInfo->colBytes, pInfo->colType); assignVal(pInfo->data + index * pInfo->colBytes, data, pInfo->colBytes, pInfo->colType);
*(pInfo->timestamp + index) = ts;
} }
static void doReservoirSample(SSampleInfo* pInfo, char* data, TSKEY ts, int32_t index) { static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* data, int32_t index) {
pInfo->totalPoints++; pInfo->totalPoints++;
if (pInfo->numSampled < pInfo->samples) { if (pInfo->numSampled < pInfo->samples) {
sampleAssignResult(pInfo, data, ts, pInfo->numSampled); sampleAssignResult(pInfo, data, pInfo->numSampled);
if (pCtx->subsidiaries.num > 0) {
saveTupleData(pCtx, index, pCtx->pSrcBlock, pInfo->tuplePos + pInfo->numSampled * sizeof(STuplePos));
}
pInfo->numSampled++; pInfo->numSampled++;
} else { } else {
int32_t j = taosRand() % (pInfo->totalPoints); int32_t j = taosRand() % (pInfo->totalPoints);
if (j < pInfo->samples) { if (j < pInfo->samples) {
sampleAssignResult(pInfo, data, ts, j); sampleAssignResult(pInfo, data, j);
if (pCtx->subsidiaries.num > 0) {
copyTupleData(pCtx, index, pCtx->pSrcBlock, pInfo->tuplePos + j * sizeof(STuplePos));
}
} }
} }
} }
@ -4398,11 +4403,6 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input; SInputColumnInfoData* pInput = &pCtx->input;
TSKEY* tsList = NULL;
if (pInput->pPTS != NULL) {
tsList = (int64_t*)pInput->pPTS->pData;
}
SColumnInfoData* pInputCol = pInput->pData[0]; SColumnInfoData* pInputCol = pInput->pData[0];
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) {
if (colDataIsNull_s(pInputCol, i)) { if (colDataIsNull_s(pInputCol, i)) {
@ -4410,7 +4410,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
} }
char* data = colDataGetData(pInputCol, i); char* data = colDataGetData(pInputCol, i);
doReservoirSample(pInfo, data, /*tsList[i]*/ 0, i); doReservoirSample(pCtx, pInfo, data, i);
} }
SET_VAL(pResInfo, pInfo->numSampled, pInfo->numSampled); SET_VAL(pResInfo, pInfo->numSampled, pInfo->numSampled);
@ -4429,6 +4429,7 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t currentRow = pBlock->info.rows; int32_t currentRow = pBlock->info.rows;
for (int32_t i = 0; i < pInfo->numSampled; ++i) { for (int32_t i = 0; i < pInfo->numSampled; ++i) {
colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false);
setSelectivityValue(pCtx, pBlock, pInfo->tuplePos + i * sizeof(STuplePos), currentRow + i);
} }
return pInfo->numSampled; return pInfo->numSampled;

10
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -557,6 +557,14 @@ static SNode* physiSessionCopy(const SSessionWinodwPhysiNode* pSrc, SSessionWino
return (SNode*)pDst; return (SNode*)pDst;
} }
static SNode* physiPartitionCopy(const SPartitionPhysiNode* pSrc, SPartitionPhysiNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, physiNodeCopy);
CLONE_NODE_LIST_FIELD(pExprs);
CLONE_NODE_LIST_FIELD(pPartitionKeys);
CLONE_NODE_LIST_FIELD(pTargets);
return (SNode*)pDst;
}
static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) { static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
COPY_SCALAR_FIELD(dataBlockId); COPY_SCALAR_FIELD(dataBlockId);
CLONE_NODE_LIST_FIELD(pSlots); CLONE_NODE_LIST_FIELD(pSlots);
@ -702,6 +710,8 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION:
case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION:
return physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst); return physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst);
case QUERY_NODE_PHYSICAL_PLAN_PARTITION:
return physiPartitionCopy((const SPartitionPhysiNode*)pNode, (SPartitionPhysiNode*)pDst);
default: default:
break; break;
} }

21
source/libs/parser/src/parInsert.c
View File

@ -257,7 +257,7 @@ static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass)
if (pBasicCtx->async) { if (pBasicCtx->async) {
return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass);
} }
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId, .requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid, .requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet}; .mgmtEps = pBasicCtx->mgmtEpSet};
@ -270,11 +270,11 @@ static int32_t getTableSchema(SInsertParseContext* pCxt, SName* pTbName, bool is
if (pBasicCtx->async) { if (pBasicCtx->async) {
return getTableMetaFromCache(pCxt->pMetaCache, pTbName, pTableMeta); return getTableMetaFromCache(pCxt->pMetaCache, pTbName, pTableMeta);
} }
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId, .requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid, .requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet}; .mgmtEps = pBasicCtx->mgmtEpSet};
if (isStb) { if (isStb) {
return catalogGetSTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta); return catalogGetSTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta);
} }
@ -286,7 +286,7 @@ static int32_t getTableVgroup(SInsertParseContext* pCxt, SName* pTbName, SVgroup
if (pBasicCtx->async) { if (pBasicCtx->async) {
return getTableVgroupFromCache(pCxt->pMetaCache, pTbName, pVg); return getTableVgroupFromCache(pCxt->pMetaCache, pTbName, pVg);
} }
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId, .requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid, .requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet}; .mgmtEps = pBasicCtx->mgmtEpSet};
@ -322,7 +322,7 @@ static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgI
if (pBasicCtx->async) { if (pBasicCtx->async) {
CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo)); CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo));
} else { } else {
SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter,
.requestId = pBasicCtx->requestId, .requestId = pBasicCtx->requestId,
.requestObjRefId = pBasicCtx->requestRid, .requestObjRefId = pBasicCtx->requestRid,
.mgmtEps = pBasicCtx->mgmtEpSet}; .mgmtEps = pBasicCtx->mgmtEpSet};
@ -1315,15 +1315,6 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) {
destroyBlockArrayList(pCxt->pVgDataBlocks); destroyBlockArrayList(pCxt->pVgDataBlocks);
} }
static int32_t checkSchemalessDb(SInsertParseContext* pCxt, char* pDbName) {
// SDbCfgInfo pInfo = {0};
// char fullName[TSDB_TABLE_FNAME_LEN];
// snprintf(fullName, sizeof(fullName), "%d.%s", pCxt->pComCxt->acctId, pDbName);
// CHECK_CODE(getDBCfg(pCxt, fullName, &pInfo));
// return pInfo.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS;
return TSDB_CODE_SUCCESS;
}
// tb_name // tb_name
// [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)]
// [(field1_name, ...)] // [(field1_name, ...)]
@ -1377,8 +1368,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
SName name; SName name;
CHECK_CODE(createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); CHECK_CODE(createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg));
CHECK_CODE(checkSchemalessDb(pCxt, name.dbname));
tNameExtractFullName(&name, tbFName); tNameExtractFullName(&name, tbFName);
CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName)));
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];

8
source/libs/planner/src/planLogicCreater.c
View File

@ -56,8 +56,12 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
pCol->node.resType = pToBeRewrittenExpr->resType; pCol->node.resType = pToBeRewrittenExpr->resType;
strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName);
strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName);
if (QUERY_NODE_FUNCTION == nodeType(pExpr) && FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pExpr)->funcType) { if (QUERY_NODE_FUNCTION == nodeType(pExpr)) {
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pExpr)->funcType) {
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
} else if (FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pExpr)->funcType) {
pCol->colType = COLUMN_TYPE_TBNAME;
}
} }
nodesDestroyNode(*pNode); nodesDestroyNode(*pNode);
*pNode = (SNode*)pCol; *pNode = (SNode*)pCol;

43
source/libs/planner/src/planOptimizer.c
View File

@ -1042,7 +1042,7 @@ static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan)
static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) { if (QUERY_NODE_COLUMN == nodeType(pNode)) {
if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType) { if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) {
*(bool*)pContext = true; *(bool*)pContext = true;
return DEAL_RES_END; return DEAL_RES_END;
} }
@ -1057,9 +1057,9 @@ static bool partTagsOptHasCol(SNodeList* pPartKeys) {
} }
static bool partTagsIsOptimizableNode(SLogicNode* pNode) { static bool partTagsIsOptimizableNode(SLogicNode* pNode) {
return ((QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) /*|| return ((QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) ||
(QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && NULL != ((SAggLogicNode*)pNode)->pGroupKeys && (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && NULL != ((SAggLogicNode*)pNode)->pGroupKeys &&
NULL != ((SAggLogicNode*)pNode)->pAggFuncs)*/) && NULL != ((SAggLogicNode*)pNode)->pAggFuncs)) &&
1 == LIST_LENGTH(pNode->pChildren) && 1 == LIST_LENGTH(pNode->pChildren) &&
QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0))); QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0)));
} }
@ -1080,6 +1080,28 @@ static bool partTagsOptMayBeOptimized(SLogicNode* pNode) {
return !partTagsOptHasCol(partTagsGetPartKeys(pNode)); return !partTagsOptHasCol(partTagsGetPartKeys(pNode));
} }
static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(*pNode) && COLUMN_TYPE_TBNAME == ((SColumnNode*)*pNode)->colType) {
SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION);
if (NULL == pFunc) {
*(int32_t*)pContext = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
strcpy(pFunc->functionName, "tbname");
pFunc->funcType = FUNCTION_TYPE_TBNAME;
nodesDestroyNode(*pNode);
*pNode = (SNode*)pFunc;
return DEAL_RES_IGNORE_CHILD;
}
return DEAL_RES_CONTINUE;
}
static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) {
int32_t code = TSDB_CODE_SUCCESS;
nodesRewriteExprs(pPartKeys, partTagsOptRebuildTbanmeImpl, &code);
return code;
}
static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized);
if (NULL == pNode) { if (NULL == pNode) {
@ -1096,7 +1118,18 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
nodesDestroyNode((SNode*)pNode); nodesDestroyNode((SNode*)pNode);
} }
} else { } else {
TSWAP(((SAggLogicNode*)pNode)->pGroupKeys, pScan->pPartTags); SNode* pGroupKey = NULL;
FOREACH(pGroupKey, ((SAggLogicNode*)pNode)->pGroupKeys) {
code = nodesListMakeStrictAppend(
&pScan->pPartTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0)));
if (TSDB_CODE_SUCCESS != code) {
break;
}
}
DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys);
}
if (TSDB_CODE_SUCCESS == code) {
code = partTagsOptRebuildTbanme(pScan->pPartTags);
} }
return code; return code;
} }
@ -1184,7 +1217,7 @@ static const SOptimizeRule optimizeRuleSet[] = {
{.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize},
{.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize},
{.pName = "SmaIndex", .optimizeFunc = smaOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize},
{.pName = "PartitionByTags", .optimizeFunc = partTagsOptimize}, // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize},
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}
}; };
// clang-format on // clang-format on

53
source/libs/planner/src/planSpliter.c
View File

@ -176,7 +176,7 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) {
return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
case QUERY_NODE_LOGIC_PLAN_WINDOW: { case QUERY_NODE_LOGIC_PLAN_WINDOW: {
SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode; SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode;
if (WINDOW_TYPE_STATE == pWindow->winType || (!streamQuery && WINDOW_TYPE_SESSION == pWindow->winType) ) { if (WINDOW_TYPE_STATE == pWindow->winType || (!streamQuery && WINDOW_TYPE_SESSION == pWindow->winType)) {
return false; return false;
} }
return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode);
@ -380,6 +380,7 @@ static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent
SExchangeLogicNode* pExchange = NULL; SExchangeLogicNode* pExchange = NULL;
int32_t code = splCreateExchangeNode(pCxt, pPartChild, &pExchange); int32_t code = splCreateExchangeNode(pCxt, pPartChild, &pExchange);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pExchange->node.pParent = pParent;
code = nodesListMakeAppend(&pParent->pChildren, (SNode*)pExchange); code = nodesListMakeAppend(&pParent->pChildren, (SNode*)pExchange);
} }
return code; return code;
@ -484,7 +485,27 @@ static int32_t stbSplSplitSession(SSplitContext* pCxt, SStableSplitInfo* pInfo)
} }
} }
static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
return ((SScanLogicNode*)pNode)->pPartTags;
} else {
return NULL;
}
}
static bool stbSplIsPartTbanme(SNodeList* pPartKeys) {
if (NULL == pPartKeys || 1 != LIST_LENGTH(pPartKeys)) {
return false;
}
SNode* pPartKey = nodesListGetNode(pPartKeys, 0);
return QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType;
}
static bool stbSplIsMultiTableWinodw(SWindowLogicNode* pWindow) {
return stbSplIsPartTbanme(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0)));
}
static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) { switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) {
case WINDOW_TYPE_INTERVAL: case WINDOW_TYPE_INTERVAL:
return stbSplSplitInterval(pCxt, pInfo); return stbSplSplitInterval(pCxt, pInfo);
@ -496,6 +517,34 @@ static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInf
return TSDB_CODE_PLAN_INTERNAL_ERROR; return TSDB_CODE_PLAN_INTERNAL_ERROR;
} }
static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
if (pCxt->pPlanCxt->streamQuery) {
SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT);
return TSDB_CODE_SUCCESS;
}
SExchangeLogicNode* pExchange = NULL;
int32_t code = splCreateExchangeNode(pCxt, pInfo->pSplitNode, &pExchange);
if (TSDB_CODE_SUCCESS == code) {
code = replaceLogicNode(pInfo->pSubplan, pInfo->pSplitNode, (SLogicNode*)pExchange);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren,
(SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT));
}
pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE;
++(pCxt->groupId);
return code;
}
static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) {
if (stbSplIsMultiTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) {
return stbSplSplitWindowForMultiTable(pCxt, pInfo);
} else {
return stbSplSplitWindowForMergeTable(pCxt, pInfo);
}
}
static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pOutput) { static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pOutput) {
SNodeList* pFunc = pMergeAgg->pAggFuncs; SNodeList* pFunc = pMergeAgg->pAggFuncs;
pMergeAgg->pAggFuncs = NULL; pMergeAgg->pAggFuncs = NULL;

4
source/libs/planner/test/planOtherTest.cpp
View File

@ -37,7 +37,9 @@ TEST_F(PlanOtherTest, createStream) {
TEST_F(PlanOtherTest, createStreamUseSTable) { TEST_F(PlanOtherTest, createStreamUseSTable) {
useDb("root", "test"); useDb("root", "test");
run("create stream if not exists s1 as select count(*) from st1 interval(10s)"); run("CREATE STREAM IF NOT EXISTS s1 as SELECT COUNT(*) FROM st1 INTERVAL(10s)");
run("CREATE STREAM IF NOT EXISTS s1 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)");
} }
TEST_F(PlanOtherTest, createSmaIndex) { TEST_F(PlanOtherTest, createSmaIndex) {

9
source/libs/stream/src/stream.c
View File

@ -50,6 +50,10 @@ void streamCleanUp() {
void streamTriggerByTimer(void* param, void* tmrId) { void streamTriggerByTimer(void* param, void* tmrId) {
SStreamTask* pTask = (void*)param; SStreamTask* pTask = (void*)param;
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
return;
}
if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) {
SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM); SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM);
if (trigger == NULL) return; if (trigger == NULL) return;
@ -82,8 +86,8 @@ int32_t streamSetupTrigger(SStreamTask* pTask) {
} }
int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId) { int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId) {
int8_t execStatus = atomic_load_8(&pTask->status); int8_t execStatus = atomic_load_8(&pTask->execStatus);
if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { if (execStatus == TASK_EXEC_STATUS__IDLE || execStatus == TASK_EXEC_STATUS__CLOSING) {
SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
if (pRunReq == NULL) return -1; if (pRunReq == NULL) return -1;
@ -188,6 +192,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) {
int32_t streamProcessRunReq(SStreamTask* pTask) { int32_t streamProcessRunReq(SStreamTask* pTask) {
streamExec(pTask, pTask->pMsgCb); streamExec(pTask, pTask->pMsgCb);
if (pTask->dispatchType != TASK_DISPATCH__NONE) { if (pTask->dispatchType != TASK_DISPATCH__NONE) {
streamDispatch(pTask, pTask->pMsgCb); streamDispatch(pTask, pTask->pMsgCb);
} }

35
source/libs/stream/src/streamExec.c
View File

@ -33,6 +33,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes)
ASSERT(pTask->inputType == STREAM_INPUT__DATA_BLOCK); ASSERT(pTask->inputType == STREAM_INPUT__DATA_BLOCK);
SArray* blocks = pBlock->blocks; SArray* blocks = pBlock->blocks;
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false);
} else if (pItem->type == STREAM_INPUT__DROP) {
// TODO exec drop
return 0;
} }
// exec // exec
@ -58,6 +61,10 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
streamTaskExecImpl(pTask, data, pRes); streamTaskExecImpl(pTask, data, pRes);
if (pTask->taskStatus == TASK_STATUS__DROPPING) {
return NULL;
}
if (taosArrayGetSize(pRes) != 0) { if (taosArrayGetSize(pRes) != 0) {
SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM);
if (qRes == NULL) { if (qRes == NULL) {
@ -75,12 +82,17 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) {
return NULL; return NULL;
} }
if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { if (((SStreamQueueItem*)data)->type == STREAM_INPUT__TRIGGER) {
streamDataSubmitRefDec((SStreamDataSubmit*)data); blockDataDestroy(((SStreamTrigger*)data)->pBlock);
taosFreeQitem(data); taosFreeQitem(data);
} else { } else {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) {
taosFreeQitem(data); streamDataSubmitRefDec((SStreamDataSubmit*)data);
taosFreeQitem(data);
} else {
taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock);
taosFreeQitem(data);
}
} }
streamQueueProcessSuccess(pTask->inputQueue); streamQueueProcessSuccess(pTask->inputQueue);
return taosArrayInit(0, sizeof(SSDataBlock)); return taosArrayInit(0, sizeof(SSDataBlock));
@ -94,25 +106,26 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock));
if (pRes == NULL) return -1; if (pRes == NULL) return -1;
while (1) { while (1) {
int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); int8_t execStatus =
if (execStatus == TASK_STATUS__IDLE) { atomic_val_compare_exchange_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE, TASK_EXEC_STATUS__EXECUTING);
if (execStatus == TASK_EXEC_STATUS__IDLE) {
// first run // first run
pRes = streamExecForQall(pTask, pRes); pRes = streamExecForQall(pTask, pRes);
if (pRes == NULL) goto FAIL; if (pRes == NULL) goto FAIL;
// set status closing // set status closing
atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__CLOSING);
// second run, make sure inputQ and qall are cleared // second run, make sure inputQ and qall are cleared
pRes = streamExecForQall(pTask, pRes); pRes = streamExecForQall(pTask, pRes);
if (pRes == NULL) goto FAIL; if (pRes == NULL) goto FAIL;
taosArrayDestroy(pRes); taosArrayDestroy(pRes);
atomic_store_8(&pTask->status, TASK_STATUS__IDLE); atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE);
return 0; return 0;
} else if (execStatus == TASK_STATUS__CLOSING) { } else if (execStatus == TASK_EXEC_STATUS__CLOSING) {
continue; continue;
} else if (execStatus == TASK_STATUS__EXECUTING) { } else if (execStatus == TASK_EXEC_STATUS__EXECUTING) {
ASSERT(taosArrayGetSize(pRes) == 0); ASSERT(taosArrayGetSize(pRes) == 0);
taosArrayDestroy(pRes); taosArrayDestroy(pRes);
return 0; return 0;
@ -122,7 +135,7 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) {
} }
FAIL: FAIL:
if (pRes) taosArrayDestroy(pRes); if (pRes) taosArrayDestroy(pRes);
atomic_store_8(&pTask->status, TASK_STATUS__IDLE); atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE);
return -1; return -1;
} }

8
source/libs/stream/src/streamTask.c
View File

@ -23,7 +23,7 @@ SStreamTask* tNewSStreamTask(int64_t streamId) {
} }
pTask->taskId = tGenIdPI32(); pTask->taskId = tGenIdPI32();
pTask->streamId = streamId; pTask->streamId = streamId;
pTask->status = TASK_STATUS__IDLE; pTask->execStatus = TASK_EXEC_STATUS__IDLE;
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
@ -35,7 +35,8 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->execStatus) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1;
if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1;
@ -83,7 +84,8 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->execStatus) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1;
if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1;

1
source/libs/sync/inc/syncInt.h
View File

@ -253,6 +253,7 @@ void syncNodePrint(SSyncNode* pObj);
void syncNodePrint2(char* s, SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj);
void syncNodeLog(SSyncNode* pObj); void syncNodeLog(SSyncNode* pObj);
void syncNodeLog2(char* s, SSyncNode* pObj); void syncNodeLog2(char* s, SSyncNode* pObj);
void syncNodeLog3(char* s, SSyncNode* pObj);
#ifdef __cplusplus #ifdef __cplusplus
} }

12
source/libs/sync/inc/syncRaftCfg.h
View File

@ -27,7 +27,7 @@ extern "C" {
#include "syncInt.h" #include "syncInt.h"
#include "taosdef.h" #include "taosdef.h"
#define CONFIG_FILE_LEN 1024 #define CONFIG_FILE_LEN 2048
#define MAX_CONFIG_INDEX_COUNT 512 #define MAX_CONFIG_INDEX_COUNT 512
@ -49,14 +49,14 @@ int32_t raftCfgClose(SRaftCfg *pRaftCfg);
int32_t raftCfgPersist(SRaftCfg *pRaftCfg); int32_t raftCfgPersist(SRaftCfg *pRaftCfg);
int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex); int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex);
cJSON * syncCfg2Json(SSyncCfg *pSyncCfg); cJSON *syncCfg2Json(SSyncCfg *pSyncCfg);
char * syncCfg2Str(SSyncCfg *pSyncCfg); char *syncCfg2Str(SSyncCfg *pSyncCfg);
char * syncCfg2SimpleStr(SSyncCfg *pSyncCfg); char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg);
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg); int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg);
int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg); int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg);
cJSON * raftCfg2Json(SRaftCfg *pRaftCfg); cJSON *raftCfg2Json(SRaftCfg *pRaftCfg);
char * raftCfg2Str(SRaftCfg *pRaftCfg); char *raftCfg2Str(SRaftCfg *pRaftCfg);
int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg);
int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg);

18
source/libs/sync/inc/syncSnapshot.h
View File

@ -39,8 +39,8 @@ typedef struct SSyncSnapshotSender {
bool start; bool start;
int32_t seq; int32_t seq;
int32_t ack; int32_t ack;
void *pReader; void * pReader;
void *pCurrentBlock; void * pCurrentBlock;
int32_t blockLen; int32_t blockLen;
SSnapshot snapshot; SSnapshot snapshot;
SSyncCfg lastConfig; SSyncCfg lastConfig;
@ -55,19 +55,19 @@ typedef struct SSyncSnapshotSender {
SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex);
void snapshotSenderDestroy(SSyncSnapshotSender *pSender); void snapshotSenderDestroy(SSyncSnapshotSender *pSender);
bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender);
void snapshotSenderStart(SSyncSnapshotSender *pSender); void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader);
void snapshotSenderStop(SSyncSnapshotSender *pSender); void snapshotSenderStop(SSyncSnapshotSender *pSender);
int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender);
int32_t snapshotReSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender);
cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender);
char *snapshotSender2Str(SSyncSnapshotSender *pSender); char * snapshotSender2Str(SSyncSnapshotSender *pSender);
char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); char * snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event);
typedef struct SSyncSnapshotReceiver { typedef struct SSyncSnapshotReceiver {
bool start; bool start;
int32_t ack; int32_t ack;
void *pWriter; void * pWriter;
SyncTerm term; SyncTerm term;
SyncTerm privateTerm; SyncTerm privateTerm;
SSnapshot snapshot; SSnapshot snapshot;
@ -83,8 +83,8 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateT
bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver);
void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply);
cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver);
char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver);
char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event);
int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg);
int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg);

45
source/libs/sync/src/syncAppendEntriesReply.c
View File

@ -173,21 +173,44 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
// get sender // get sender
SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId));
ASSERT(pSender != NULL); ASSERT(pSender != NULL);
bool hasSnapshot = syncNodeHasSnapshot(ths);
SSnapshot snapshot;
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
// start sending snapshot first time SSnapshot snapshot;
// start here, stop by receiver void* pReader = NULL;
if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) && ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot, NULL, &pReader);
pMsg->privateTerm < pSender->privateTerm) { if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN && nextIndex <= snapshot.lastApplyIndex + 1 &&
snapshotSenderStart(pSender); !snapshotSenderIsStart(pSender) && pMsg->privateTerm < pSender->privateTerm) {
// has snapshot
ASSERT(pReader != NULL);
snapshotSenderStart(pSender, snapshot, pReader);
char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start");
syncNodeEventLog(ths, eventLog); syncNodeEventLog(ths, eventLog);
taosMemoryFree(eventLog); taosMemoryFree(eventLog);
} else {
// no snapshot
if (pReader != NULL) {
ths->pFsm->FpSnapshotStopRead(ths->pFsm, pReader);
}
} }
/*
bool hasSnapshot = syncNodeHasSnapshot(ths);
SSnapshot snapshot;
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
// start sending snapshot first time
// start here, stop by receiver
if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) &&
pMsg->privateTerm < pSender->privateTerm) {
snapshotSenderStart(pSender);
char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start");
syncNodeEventLog(ths, eventLog);
taosMemoryFree(eventLog);
}
*/
SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1; SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1;
// update nextIndex to sentryIndex // update nextIndex to sentryIndex
@ -207,12 +230,6 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries
syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex);
syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex);
if (gRaftDetailLog) {
SSnapshot snapshot;
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
sTrace("recv SyncAppendEntriesReply, after snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu",
snapshot.lastApplyIndex, snapshot.lastApplyTerm);
}
return ret; return ret;
} }

19
source/libs/sync/src/syncIndexMgr.c
View File

@ -63,7 +63,12 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId,
} }
// maybe config change // maybe config change
assert(0); // assert(0);
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d index mgr set for %s:%d, index:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, index);
} }
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
@ -73,7 +78,9 @@ SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaf
return idx; return idx;
} }
} }
assert(0);
syncNodeLog3("syncIndexMgrGetIndex", pSyncIndexMgr->pSyncNode);
ASSERT(0);
} }
cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
@ -119,7 +126,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) {
cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr);
char * serialized = cJSON_Print(pJson); char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
return serialized; return serialized;
} }
@ -162,7 +169,11 @@ void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, S
} }
// maybe config change // maybe config change
assert(0); // assert(0);
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d index mgr set for %s:%d, term:%lu error", pSyncIndexMgr->pSyncNode->vgId, host, port, term);
} }
SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {

40
source/libs/sync/src/syncMain.c
View File

@ -1282,6 +1282,9 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
// snapshot receivers // snapshot receivers
cJSON* pReceivers = cJSON_CreateArray(); cJSON* pReceivers = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver));
// changing
cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing);
} }
cJSON* pJson = cJSON_CreateObject(); cJSON* pJson = cJSON_CreateObject();
@ -1304,26 +1307,31 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) {
pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot);
} }
SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore);
SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore);
if (userStrLen < 256) { if (userStrLen < 256) {
char logBuf[128 + 256]; char logBuf[128 + 256];
snprintf(logBuf, sizeof(logBuf), snprintf(logBuf, sizeof(logBuf),
"vgId:%d, sync %s %s, term:%lu, commit:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, replica-num:%d, " "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
"replica-num:%d, "
"lconfig:%ld, changing:%d", "lconfig:%ld, changing:%d",
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing); pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing);
sDebug("%s", logBuf); sDebug("%s", logBuf);
} else { } else {
int len = 128 + userStrLen; int len = 128 + userStrLen;
char* s = (char*)taosMemoryMalloc(len); char* s = (char*)taosMemoryMalloc(len);
snprintf(s, len, snprintf(s, len,
"vgId:%d, sync %s %s, term:%lu, commit:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, replica-num:%d, " "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, "
"replica-num:%d, "
"lconfig:%ld, changing:%d", "lconfig:%ld, changing:%d",
pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm,
pSyncNode->commitIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex,
pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing); pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex,
pSyncNode->changing);
sDebug("%s", s); sDebug("%s", s);
taosMemoryFree(s); taosMemoryFree(s);
} }
@ -1400,7 +1408,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
pSyncNode->pRaftCfg->isStandBy = 1; // set standby pSyncNode->pRaftCfg->isStandBy = 1; // set standby
} }
// persist last config index // add last config index
raftCfgAddConfigIndex(pSyncNode->pRaftCfg, lastConfigChangeIndex); raftCfgAddConfigIndex(pSyncNode->pRaftCfg, lastConfigChangeIndex);
if (IamInNew) { if (IamInNew) {
@ -1827,7 +1835,11 @@ SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) {
SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
ASSERT(index >= SYNC_INDEX_BEGIN); ASSERT(index >= SYNC_INDEX_BEGIN);
SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
ASSERT(index <= syncStartIndex);
if (index > syncStartIndex) {
syncNodeLog3("syncNodeGetPreIndex", pSyncNode);
ASSERT(0);
}
SyncIndex preIndex = index - 1; SyncIndex preIndex = index - 1;
return preIndex; return preIndex;
@ -1836,7 +1848,11 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) {
SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) {
ASSERT(index >= SYNC_INDEX_BEGIN); ASSERT(index >= SYNC_INDEX_BEGIN);
SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode);
ASSERT(index <= syncStartIndex);
if (index > syncStartIndex) {
syncNodeLog3("syncNodeGetPreTerm", pSyncNode);
ASSERT(0);
}
if (index == SYNC_INDEX_BEGIN) { if (index == SYNC_INDEX_BEGIN) {
return 0; return 0;
@ -1929,6 +1945,12 @@ void syncNodeLog2(char* s, SSyncNode* pObj) {
} }
} }
void syncNodeLog3(char* s, SSyncNode* pObj) {
char* serialized = syncNode2Str(pObj);
sTraceLong("syncNodeLog3 | len:%lu | %s | %s", strlen(serialized), s, serialized);
taosMemoryFree(serialized);
}
// ------ local funciton --------- // ------ local funciton ---------
// enqueue message ---- // enqueue message ----
static void syncNodeEqPingTimer(void* param, void* tmrId) { static void syncNodeEqPingTimer(void* param, void* tmrId) {

15
source/libs/sync/src/syncRaftCfg.c
View File

@ -53,7 +53,12 @@ int32_t raftCfgPersist(SRaftCfg *pRaftCfg) {
char buf[CONFIG_FILE_LEN] = {0}; char buf[CONFIG_FILE_LEN] = {0};
memset(buf, 0, sizeof(buf)); memset(buf, 0, sizeof(buf));
ASSERT(strlen(s) + 1 <= CONFIG_FILE_LEN);
if (strlen(s) + 1 > CONFIG_FILE_LEN) {
sError("too long config str:%s", s);
ASSERT(0);
}
snprintf(buf, sizeof(buf), "%s", s); snprintf(buf, sizeof(buf), "%s", s);
int64_t ret = taosWriteFile(pRaftCfg->pFile, buf, sizeof(buf)); int64_t ret = taosWriteFile(pRaftCfg->pFile, buf, sizeof(buf));
assert(ret == sizeof(buf)); assert(ret == sizeof(buf));
@ -96,14 +101,14 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) {
cJSON *pJson = syncCfg2Json(pSyncCfg); cJSON *pJson = syncCfg2Json(pSyncCfg);
char * serialized = cJSON_Print(pJson); char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
return serialized; return serialized;
} }
char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) {
int32_t len = 512; int32_t len = 512;
char * s = taosMemoryMalloc(len); char *s = taosMemoryMalloc(len);
memset(s, 0, len); memset(s, 0, len);
snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex);
@ -196,7 +201,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
char *raftCfg2Str(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) {
cJSON *pJson = raftCfg2Json(pRaftCfg); cJSON *pJson = raftCfg2Json(pRaftCfg);
char * serialized = cJSON_Print(pJson); char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson); cJSON_Delete(pJson);
return serialized; return serialized;
} }
@ -262,7 +267,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
} }
cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
ASSERT(code == 0); ASSERT(code == 0);

30
source/libs/sync/src/syncSnapshot.c
View File

@ -67,7 +67,7 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) {
bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; }
// begin send snapshot (current term, seq begin) // begin send snapshot (current term, seq begin)
void snapshotSenderStart(SSyncSnapshotSender *pSender) { void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader) {
ASSERT(!snapshotSenderIsStart(pSender)); ASSERT(!snapshotSenderIsStart(pSender));
pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN;
@ -75,8 +75,18 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) {
// open snapshot reader // open snapshot reader
ASSERT(pSender->pReader == NULL); ASSERT(pSender->pReader == NULL);
int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader)); pSender->pReader = pReader;
ASSERT(ret == 0); pSender->snapshot = snapshot;
/*
// open snapshot reader
ASSERT(pSender->pReader == NULL);
int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader));
ASSERT(ret == 0);
// get current snapshot info
pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot));
*/
if (pSender->pCurrentBlock != NULL) { if (pSender->pCurrentBlock != NULL) {
taosMemoryFree(pSender->pCurrentBlock); taosMemoryFree(pSender->pCurrentBlock);
@ -84,21 +94,7 @@ void snapshotSenderStart(SSyncSnapshotSender *pSender) {
pSender->blockLen = 0; pSender->blockLen = 0;
// get current snapshot info
pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot));
sTrace("snapshotSenderStart lastApplyIndex:%ld, lastApplyTerm:%lu, lastConfigIndex:%ld",
pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex);
if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) { if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) {
/*
SSyncRaftEntry *pEntry = NULL;
int32_t code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore,
pSender->snapshot.lastConfigIndex, &pEntry);
ASSERT(code == 0);
ASSERT(pEntry != NULL);
*/
SSyncRaftEntry *pEntry = SSyncRaftEntry *pEntry =
pSender->pSyncNode->pLogStore->getEntry(pSender->pSyncNode->pLogStore, pSender->snapshot.lastConfigIndex); pSender->pSyncNode->pLogStore->getEntry(pSender->pSyncNode->pLogStore, pSender->snapshot.lastConfigIndex);
ASSERT(pEntry != NULL); ASSERT(pEntry != NULL);

28
source/libs/sync/test/syncReconfigFinishTest.cpp
View File

@ -14,8 +14,8 @@ void logTest() {
sFatal("--- sync log test: fatal"); sFatal("--- sync log test: fatal");
} }
SSyncCfg* createSyncOldCfg() { SSyncCfg *createSyncOldCfg() {
SSyncCfg* pCfg = (SSyncCfg*)taosMemoryMalloc(sizeof(SSyncCfg)); SSyncCfg *pCfg = (SSyncCfg *)taosMemoryMalloc(sizeof(SSyncCfg));
memset(pCfg, 0, sizeof(SSyncCfg)); memset(pCfg, 0, sizeof(SSyncCfg));
pCfg->replicaNum = 3; pCfg->replicaNum = 3;
@ -28,8 +28,8 @@ SSyncCfg* createSyncOldCfg() {
return pCfg; return pCfg;
} }
SSyncCfg* createSyncNewCfg() { SSyncCfg *createSyncNewCfg() {
SSyncCfg* pCfg = (SSyncCfg*)taosMemoryMalloc(sizeof(SSyncCfg)); SSyncCfg *pCfg = (SSyncCfg *)taosMemoryMalloc(sizeof(SSyncCfg));
memset(pCfg, 0, sizeof(SSyncCfg)); memset(pCfg, 0, sizeof(SSyncCfg));
pCfg->replicaNum = 3; pCfg->replicaNum = 3;
@ -44,9 +44,9 @@ SSyncCfg* createSyncNewCfg() {
SyncReconfigFinish *createMsg() { SyncReconfigFinish *createMsg() {
SyncReconfigFinish *pMsg = syncReconfigFinishBuild(1234); SyncReconfigFinish *pMsg = syncReconfigFinishBuild(1234);
SSyncCfg* pOld = createSyncOldCfg(); SSyncCfg *pOld = createSyncOldCfg();
SSyncCfg* pNew = createSyncNewCfg(); SSyncCfg *pNew = createSyncNewCfg();
pMsg->oldCfg = *pOld; pMsg->oldCfg = *pOld;
pMsg->newCfg = *pNew; pMsg->newCfg = *pNew;
@ -60,18 +60,16 @@ SyncReconfigFinish *createMsg() {
return pMsg; return pMsg;
} }
void test1() { void test1() {
SyncReconfigFinish *pMsg = createMsg(); SyncReconfigFinish *pMsg = createMsg();
syncReconfigFinishLog2((char *)"test1:", pMsg); syncReconfigFinishLog2((char *)"test1:", pMsg);
syncReconfigFinishDestroy(pMsg); syncReconfigFinishDestroy(pMsg);
} }
void test2() { void test2() {
SyncReconfigFinish *pMsg = createMsg(); SyncReconfigFinish *pMsg = createMsg();
uint32_t len = pMsg->bytes; uint32_t len = pMsg->bytes;
char * serialized = (char *)taosMemoryMalloc(len); char * serialized = (char *)taosMemoryMalloc(len);
syncReconfigFinishSerialize(pMsg, serialized, len); syncReconfigFinishSerialize(pMsg, serialized, len);
SyncReconfigFinish *pMsg2 = syncReconfigFinishBuild(1000); SyncReconfigFinish *pMsg2 = syncReconfigFinishBuild(1000);
syncReconfigFinishDeserialize(serialized, len, pMsg2); syncReconfigFinishDeserialize(serialized, len, pMsg2);
@ -84,8 +82,8 @@ void test2() {
void test3() { void test3() {
SyncReconfigFinish *pMsg = createMsg(); SyncReconfigFinish *pMsg = createMsg();
uint32_t len; uint32_t len;
char * serialized = syncReconfigFinishSerialize2(pMsg, &len); char * serialized = syncReconfigFinishSerialize2(pMsg, &len);
SyncReconfigFinish *pMsg2 = syncReconfigFinishDeserialize2(serialized, len); SyncReconfigFinish *pMsg2 = syncReconfigFinishDeserialize2(serialized, len);
syncReconfigFinishLog2((char *)"test3: SyncReconfigFinishSerialize2 -> syncReconfigFinishDeserialize2 ", pMsg2); syncReconfigFinishLog2((char *)"test3: SyncReconfigFinishSerialize2 -> syncReconfigFinishDeserialize2 ", pMsg2);
@ -96,7 +94,7 @@ void test3() {
void test4() { void test4() {
SyncReconfigFinish *pMsg = createMsg(); SyncReconfigFinish *pMsg = createMsg();
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncReconfigFinish2RpcMsg(pMsg, &rpcMsg); syncReconfigFinish2RpcMsg(pMsg, &rpcMsg);
SyncReconfigFinish *pMsg2 = (SyncReconfigFinish *)taosMemoryMalloc(rpcMsg.contLen); SyncReconfigFinish *pMsg2 = (SyncReconfigFinish *)taosMemoryMalloc(rpcMsg.contLen);
syncReconfigFinishFromRpcMsg(&rpcMsg, pMsg2); syncReconfigFinishFromRpcMsg(&rpcMsg, pMsg2);
@ -109,7 +107,7 @@ void test4() {
void test5() { void test5() {
SyncReconfigFinish *pMsg = createMsg(); SyncReconfigFinish *pMsg = createMsg();
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
syncReconfigFinish2RpcMsg(pMsg, &rpcMsg); syncReconfigFinish2RpcMsg(pMsg, &rpcMsg);
SyncReconfigFinish *pMsg2 = syncReconfigFinishFromRpcMsg2(&rpcMsg); SyncReconfigFinish *pMsg2 = syncReconfigFinishFromRpcMsg2(&rpcMsg);
syncReconfigFinishLog2((char *)"test5: syncReconfigFinish2RpcMsg -> syncReconfigFinishFromRpcMsg2 ", pMsg2); syncReconfigFinishLog2((char *)"test5: syncReconfigFinish2RpcMsg -> syncReconfigFinishFromRpcMsg2 ", pMsg2);

16
source/libs/transport/inc/transLog.h
View File

@ -24,16 +24,16 @@ extern "C" {
#include "tlog.h" #include "tlog.h"
#include "ttrace.h" #include "ttrace.h"
#define tFatal(...) do {if (rpcDebugFlag & DEBUG_FATAL){ taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); }} while (0) #define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); }}
#define tError(...)do { if (rpcDebugFlag & DEBUG_ERROR){ taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); } } while(0) #define tError(...) { if (rpcDebugFlag & DEBUG_ERROR ){ taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); }}
#define tWarn(...) do { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); }} while(0) #define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); }}
#define tInfo(...) do { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); }} while(0) #define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); }}
#define tDebug(...) do {if (rpcDebugFlag & DEBUG_DEBUG){ taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); }} while(0) #define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); }}
#define tTrace(...) do {if (rpcDebugFlag & DEBUG_TRACE){ taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }} while(0) #define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }}
#define tDump(x, y) do {if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } } while(0) #define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } }
//#define tTR(param, ...) do { char buf[40] = {0};TRACE_TO_STR(trace, buf);tTrace("TRID: %s "param, buf, __VA_ARGS__);} while(0) //#define tTR(param, ...) do { char buf[40] = {0};TRACE_TO_STR(trace, buf);tTrace("TRID: %s "param, buf, __VA_ARGS__);} while(0)
#define tGTrace(param, ...) do { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", GTID: %s", __VA_ARGS__, buf);} while(0) #define tGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", gtid:%s", __VA_ARGS__, buf);}
// clang-format on // clang-format on
#ifdef __cplusplus #ifdef __cplusplus

26
source/util/src/thash.c
View File

@ -31,12 +31,12 @@
#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) #define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) #define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
#define FREE_HASH_NODE(_fp, _n) \ #define FREE_HASH_NODE(_fp, _n) \
do { \ do { \
if (_fp != NULL) { \ if (_fp != NULL) { \
(_fp)(_n); \ (_fp)(GET_HASH_NODE_DATA(_n)); \
} \ } \
taosMemoryFreeClear(_n); \ taosMemoryFreeClear(_n); \
} while (0); } while (0);
struct SHashNode { struct SHashNode {
@ -56,7 +56,7 @@ typedef struct SHashEntry {
} SHashEntry; } SHashEntry;
struct SHashObj { struct SHashObj {
SHashEntry ** hashList; SHashEntry **hashList;
size_t capacity; // number of slots size_t capacity; // number of slots
int64_t size; // number of elements in hash table int64_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function _hash_fn_t hashFp; // hash function
@ -65,7 +65,7 @@ struct SHashObj {
SRWLatch lock; // read-write spin lock SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type SHashLockTypeE type; // lock type
bool enableUpdate; // enable update bool enableUpdate; // enable update
SArray * pMemBlock; // memory block allocated for SHashEntry SArray *pMemBlock; // memory block allocated for SHashEntry
_hash_before_fn_t callbackFp; // function invoked before return the value to caller _hash_before_fn_t callbackFp; // function invoked before return the value to caller
}; };
@ -633,7 +633,7 @@ void taosHashTableResize(SHashObj *pHashObj) {
} }
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
void * pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity);
if (pNewEntryList == NULL) { if (pNewEntryList == NULL) {
// uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
return; return;
@ -642,7 +642,7 @@ void taosHashTableResize(SHashObj *pHashObj) {
pHashObj->hashList = pNewEntryList; pHashObj->hashList = pNewEntryList;
size_t inc = newCapacity - pHashObj->capacity; size_t inc = newCapacity - pHashObj->capacity;
void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); void *p = taosMemoryCalloc(inc, sizeof(SHashEntry));
for (int32_t i = 0; i < inc; ++i) { for (int32_t i = 0; i < inc; ++i) {
pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry));
@ -653,9 +653,9 @@ void taosHashTableResize(SHashObj *pHashObj) {
pHashObj->capacity = newCapacity; pHashObj->capacity = newCapacity;
for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
SHashEntry *pe = pHashObj->hashList[idx]; SHashEntry *pe = pHashObj->hashList[idx];
SHashNode * pNode; SHashNode *pNode;
SHashNode * pNext; SHashNode *pNext;
SHashNode * pPrev = NULL; SHashNode *pPrev = NULL;
if (pe->num == 0) { if (pe->num == 0) {
assert(pe->next == NULL); assert(pe->next == NULL);

44
tests/pytest/util/boundary.py Normal file
View File

@ -0,0 +1,44 @@
class DataBoundary:
def __init__(self):
self.TINYINT_BOUNDARY = [-128, 127]
self.SMALLINT_BOUNDARY = [-32768, 32767]
self.INT_BOUNDARY = [-2147483648, 2147483647]
self.BIGINT_BOUNDARY = [-9223372036854775808, 9223372036854775807]
self.UTINYINT_BOUNDARY = [0, 255]
self.USMALLINT_BOUNDARY = [0, 65535]
self.UINT_BOUNDARY = [0, 4294967295]
self.UBIGINT_BOUNDARY = [0, 18446744073709551615]
self.FLOAT_BOUNDARY = [-3.40E+38, 3.40E+38]
self.DOUBLE_BOUNDARY = [-1.7e+308, 1.7e+308]
self.BOOL_BOUNDARY = [True, False]
self.BINARY_MAX_LENGTH = 16374
self.NCHAR_MAX_LENGTH = 4093
self.DBNAME_MAX_LENGTH = 64
self.STBNAME_MAX_LENGTH = 192
self.TBNAME_MAX_LENGTH = 192
self.CHILD_TBNAME_MAX_LENGTH = 192
self.TAG_KEY_MAX_LENGTH = 64
self.COL_KEY_MAX_LENGTH = 64
self.MAX_TAG_COUNT = 128
self.MAX_TAG_COL_COUNT = 4096
self.mnodeShmSize = [6292480, 2147483647]
self.mnodeShmSize_default = 6292480
self.vnodeShmSize = [6292480, 2147483647]
self.vnodeShmSize_default = 31458304
self.DB_PARAM_BUFFER_CONFIG = {"create_name": "buffer", "query_name": "buffer", "vnode_json_key": "szBuf", "boundary": [3, 16384], "default": 96}
self.DB_PARAM_CACHELAST_CONFIG = {"create_name": "cachelast", "query_name": "cache_model", "vnode_json_key": "", "boundary": [0, 1, 2, 3], "default": 0}
self.DB_PARAM_COMP_CONFIG = {"create_name": "comp", "query_name": "compression", "vnode_json_key": "", "boundary": [0, 1, 2], "default": 2}
self.DB_PARAM_DURATION_CONFIG = {"create_name": "duration", "query_name": "duration", "vnode_json_key": "daysPerFile", "boundary": [1, 3650, '60m', '5256000m', '1h', '87600h', '1d', '3650d'], "default": "14400m"}
self.DB_PARAM_FSYNC_CONFIG = {"create_name": "fsync", "query_name": "fsync", "vnode_json_key": "", "boundary": [0, 180000], "default": 3000}
self.DB_PARAM_KEEP_CONFIG = {"create_name": "keep", "query_name": "fsync", "vnode_json_key": "", "boundary": [1, 365000,'1440m','525600000m','24h','8760000h','1d','365000d'], "default": "5256000m,5256000m,5256000m"}
self.DB_PARAM_MAXROWS_CONFIG = {"create_name": "maxrows", "query_name": "maxrows", "vnode_json_key": "maxRows", "boundary": [200, 10000], "default": 4096}
self.DB_PARAM_MINROWS_CONFIG = {"create_name": "minrows", "query_name": "minrows", "vnode_json_key": "minRows", "boundary": [10, 1000], "default": 100}
self.DB_PARAM_NTABLES_CONFIG = {"create_name": "ntables", "query_name": "ntables", "vnode_json_key": "", "boundary": 0, "default": 0}
self.DB_PARAM_PAGES_CONFIG = {"create_name": "pages", "query_name": "pages", "vnode_json_key": "szCache", "boundary": [64], "default": 256}
self.DB_PARAM_PAGESIZE_CONFIG = {"create_name": "pagesize", "query_name": "pagesize", "vnode_json_key": "szPage", "boundary": [1, 16384], "default": 4}
self.DB_PARAM_PRECISION_CONFIG = {"create_name": "precision", "query_name": "precision", "vnode_json_key": "", "boundary": ['ms', 'us', 'ns'], "default": "ms"}
self.DB_PARAM_REPLICA_CONFIG = {"create_name": "replica", "query_name": "replica", "vnode_json_key": "", "boundary": [1], "default": 1}
self.DB_PARAM_SINGLE_STABLE_CONFIG = {"create_name": "single_stable", "query_name": "single_stable_model", "vnode_json_key": "", "boundary": [0, 1], "default": 0}
self.DB_PARAM_STRICT_CONFIG = {"create_name": "strict", "query_name": "strict", "vnode_json_key": "", "boundary": {"no_strict": 0, "strict": 1}, "default": "no_strict"}
self.DB_PARAM_VGROUPS_CONFIG = {"create_name": "vgroups", "query_name": "vgroups", "vnode_json_key": "", "boundary": [1, 32], "default": 2}
self.DB_PARAM_WAL_CONFIG = {"create_name": "wal", "query_name": "wal", "vnode_json_key": "", "boundary": [1, 2], "default": 1}

471
tests/pytest/util/common.py
View File

@ -17,7 +17,7 @@ import string
import requests import requests
import time import time
import socket import socket
from .boundary import DataBoundary
import taos import taos
from util.log import * from util.log import *
from util.sql import * from util.sql import *
@ -25,9 +25,45 @@ from util.cases import *
from util.dnodes import * from util.dnodes import *
from util.common import * from util.common import *
class TDCom: class TDCom:
def init(self, conn, logSql): def __init__(self):
tdSql.init(conn.cursor(), logSql) self.sml_type = None
self.env_setting = None
self.smlChildTableName_value = None
self.defaultJSONStrType_value = None
self.smlTagNullName_value = None
self.default_varchar_length = 256
self.default_nchar_length = 256
self.default_varchar_datatype = "letters"
self.default_nchar_datatype = "letters"
self.default_tagname_prefix = "t"
self.default_colname_prefix = "c"
self.default_stbname_prefix = "stb"
self.default_ctbname_prefix = "ctb"
self.default_tbname_prefix = "tb"
self.default_tag_index_start_num = 1
self.default_column_index_start_num = 1
self.default_stbname_index_start_num = 1
self.default_ctbname_index_start_num = 1
self.default_tbname_index_start_num = 1
self.default_tagts_name = "ts"
self.default_colts_name = "ts"
self.dbname = "test"
self.stb_name = "stb"
self.ctb_name = "ctb"
self.tb_name = "tb"
self.need_tagts = False
self.tag_type_str = ""
self.column_type_str = ""
self.columns_str = None
self.ts_value = None
self.tag_value_list = list()
self.column_value_list = list()
self.full_type_list = ["tinyint", "smallint", "int", "bigint", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned", "float", "double", "binary", "nchar", "bool"]
self.white_list = ["statsd", "node_exporter", "collectd", "icinga2", "tcollector", "information_schema", "performance_schema"]
self.Boundary = DataBoundary()
# def init(self, conn, logSql):
# # tdSql.init(conn.cursor(), logSql)
def preDefine(self): def preDefine(self):
header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='}
@ -113,6 +149,59 @@ class TDCom:
def dateToTs(self, datetime_input): def dateToTs(self, datetime_input):
return int(time.mktime(time.strptime(datetime_input, "%Y-%m-%d %H:%M:%S.%f"))) return int(time.mktime(time.strptime(datetime_input, "%Y-%m-%d %H:%M:%S.%f")))
def genTs(self, precision="ms", ts="", protype="taosc", ns_tag=None):
"""
protype = "taosc" or "restful"
gen ts and datetime
"""
if precision == "ns":
if ts == "" or ts is None:
ts = time.time_ns()
else:
ts = ts
if ns_tag is None:
dt = ts
else:
dt = datetime.fromtimestamp(ts // 1000000000)
dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000000000)).zfill(9)
if protype == "restful":
dt = datetime.fromtimestamp(ts // 1000000000)
dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000000000)).zfill(9)
else:
if ts == "" or ts is None:
ts = time.time()
else:
ts = ts
if precision == "ms" or precision is None:
ts = int(round(ts * 1000))
dt = datetime.fromtimestamp(ts // 1000)
if protype == "taosc":
dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000)).zfill(3) + '000'
elif protype == "restful":
dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000)).zfill(3)
else:
pass
elif precision == "us":
ts = int(round(ts * 1000000))
dt = datetime.fromtimestamp(ts // 1000000)
dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000000)).zfill(6)
return ts, dt
def get_long_name(self, length=10, mode="letters"):
"""
generate long name
mode could be numbers/letters/letters_mixed/mixed
"""
if mode == "numbers":
population = string.digits
elif mode == "letters":
population = string.ascii_letters.lower()
elif mode == "letters_mixed":
population = string.ascii_letters.upper() + string.ascii_letters.lower()
else:
population = string.ascii_letters.lower() + string.digits
return "".join(random.choices(population, k=length))
def getLongName(self, len, mode = "mixed"): def getLongName(self, len, mode = "mixed"):
""" """
generate long name generate long name
@ -177,84 +266,92 @@ class TDCom:
def close(self): def close(self):
self.cursor.close() self.cursor.close()
def create_database(self,tsql, dbName='test',dropFlag=1,precision="ms", **kwargs): ########################################################################################################################################
# new common API
########################################################################################################################################
def create_database(self,tsql, dbName='test',dropFlag=1,**kwargs):
if dropFlag == 1: if dropFlag == 1:
tsql.execute("drop database if exists %s"%(dbName)) tsql.execute("drop database if exists %s"%(dbName))
''' '''
vgroups replica precision strict wal fsync comp cachelast single_stable buffer pagesize pages minrows maxrows duration keep retentions vgroups replica precision strict wal fsync comp cachelast single_stable buffer pagesize pages minrows maxrows duration keep retentions
''' '''
sqlString = f'create database if not exists {dbName} precision "{precision}" vgroups 4' sqlString = f'create database if not exists {dbName} '
dbParams = ""
if len(kwargs) > 0: if len(kwargs) > 0:
dbParams = ""
for param, value in kwargs.items(): for param, value in kwargs.items():
dbParams += f'{param} {value} ' if param == "precision":
dbParams += f'{param} "{value}" '
else:
dbParams += f'{param} {value} '
sqlString += f'{dbParams}' sqlString += f'{dbParams}'
tdLog.debug("create db sql: %s"%sqlString)
tsql.execute(sqlString) tsql.execute(sqlString)
tdLog.debug("complete to create database %s"%(dbName)) tdLog.debug("complete to create database %s"%(dbName))
return return
def create_stable(self,tsql, dbName,stbName,columnDict,tagDict): # def create_stable(self,tsql, dbName,stbName,column_elm_list=None, tag_elm_list=None):
colSchema = '' # colSchema = ''
for i in range(columnDict['int']): # for i in range(columnDict['int']):
colSchema += ', c%d int'%i # colSchema += ', c%d int'%i
tagSchema = '' # tagSchema = ''
for i in range(tagDict['int']): # for i in range(tagDict['int']):
if i > 0: # if i > 0:
tagSchema += ',' # tagSchema += ','
tagSchema += 't%d int'%i # tagSchema += 't%d int'%i
tsql.execute("create table if not exists %s.%s (ts timestamp %s) tags(%s)"%(dbName, stbName, colSchema, tagSchema)) # tsql.execute("create table if not exists %s.%s (ts timestamp %s) tags(%s)"%(dbName, stbName, colSchema, tagSchema))
tdLog.debug("complete to create %s.%s" %(dbName, stbName)) # tdLog.debug("complete to create %s.%s" %(dbName, stbName))
return # return
def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict): # def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict):
tsql.execute("use %s" %dbName) # tsql.execute("use %s" %dbName)
tagsValues = '' # tagsValues = ''
for i in range(tagDict['int']): # for i in range(tagDict['int']):
if i > 0: # if i > 0:
tagsValues += ',' # tagsValues += ','
tagsValues += '%d'%i # tagsValues += '%d'%i
pre_create = "create table" # pre_create = "create table"
sql = pre_create # sql = pre_create
#tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) # #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname))
for i in range(ctbNum): # for i in range(ctbNum):
sql += " %s_%d using %s tags(%s)"%(stbName,i,stbName,tagsValues) # sql += " %s_%d using %s tags(%s)"%(stbName,i,stbName,tagsValues)
if (i > 0) and (i%100 == 0): # if (i > 0) and (i%100 == 0):
tsql.execute(sql) # tsql.execute(sql)
sql = pre_create # sql = pre_create
if sql != pre_create: # if sql != pre_create:
tsql.execute(sql) # tsql.execute(sql)
tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) # tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName))
return # return
def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): # def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0):
tdLog.debug("start to insert data ............") # tdLog.debug("start to insert data ............")
tsql.execute("use %s" %dbName) # tsql.execute("use %s" %dbName)
pre_insert = "insert into " # pre_insert = "insert into "
sql = pre_insert # sql = pre_insert
if startTs == 0: # if startTs == 0:
t = time.time() # t = time.time()
startTs = int(round(t * 1000)) # startTs = int(round(t * 1000))
#tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) # #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
for i in range(ctbNum): # for i in range(ctbNum):
sql += " %s_%d values "%(stbName,i) # sql += " %s_%d values "%(stbName,i)
for j in range(rowsPerTbl): # for j in range(rowsPerTbl):
sql += "(%d, %d, %d)"%(startTs + j, j, j) # sql += "(%d, %d, %d)"%(startTs + j, j, j)
if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): # if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
tsql.execute(sql) # tsql.execute(sql)
if j < rowsPerTbl - 1: # if j < rowsPerTbl - 1:
sql = "insert into %s_%d values " %(stbName,i) # sql = "insert into %s_%d values " %(stbName,i)
else: # else:
sql = "insert into " # sql = "insert into "
#end sql # #end sql
if sql != pre_insert: # if sql != pre_insert:
#print("insert sql:%s"%sql) # #print("insert sql:%s"%sql)
tsql.execute(sql) # tsql.execute(sql)
tdLog.debug("insert data ............ [OK]") # tdLog.debug("insert data ............ [OK]")
return # return
def getBuildPath(self): def getBuildPath(self):
selfPath = os.path.dirname(os.path.realpath(__file__)) selfPath = os.path.dirname(os.path.realpath(__file__))
@ -295,4 +392,250 @@ class TDCom:
newTdSql.init(cur, False) newTdSql.init(cur, False)
return newTdSql return newTdSql
################################################################################################################
# port from the common.py of new test frame
################################################################################################################
def gen_default_tag_str(self):
default_tag_str = ""
for tag_type in self.full_type_list:
if tag_type.lower() not in ["varchar", "binary", "nchar"]:
default_tag_str += f" {self.default_tagname_prefix}{self.default_tag_index_start_num} {tag_type},"
else:
if tag_type.lower() in ["varchar", "binary"]:
default_tag_str += f" {self.default_tagname_prefix}{self.default_tag_index_start_num} {tag_type}({self.default_varchar_length}),"
else:
default_tag_str += f" {self.default_tagname_prefix}{self.default_tag_index_start_num} {tag_type}({self.default_nchar_length}),"
self.default_tag_index_start_num += 1
if self.need_tagts:
default_tag_str = self.default_tagts_name + " timestamp," + default_tag_str
return default_tag_str[:-1].lstrip()
def gen_default_column_str(self):
self.default_column_index_start_num = 1
default_column_str = ""
for col_type in self.full_type_list:
if col_type.lower() not in ["varchar", "binary", "nchar"]:
default_column_str += f" {self.default_colname_prefix}{self.default_column_index_start_num} {col_type},"
else:
if col_type.lower() in ["varchar", "binary"]:
default_column_str += f" {self.default_colname_prefix}{self.default_column_index_start_num} {col_type}({self.default_varchar_length}),"
else:
default_column_str += f" {self.default_colname_prefix}{self.default_column_index_start_num} {col_type}({self.default_nchar_length}),"
self.default_column_index_start_num += 1
default_column_str = self.default_colts_name + " timestamp," + default_column_str
return default_column_str[:-1].lstrip()
def gen_tag_type_str(self, tagname_prefix, tag_elm_list):
tag_index_start_num = 1
tag_type_str = ""
if tag_elm_list is None:
tag_type_str = self.gen_default_tag_str()
else:
for tag_elm in tag_elm_list:
if "count" in tag_elm:
total_count = int(tag_elm["count"])
else:
total_count = 1
if total_count > 0:
for _ in range(total_count):
tag_type_str += f'{tagname_prefix}{tag_index_start_num} {tag_elm["type"]}, '
if tag_elm["type"] in ["varchar", "binary", "nchar"]:
tag_type_str = tag_type_str.rstrip()[:-1] + f'({tag_elm["len"]}), '
tag_index_start_num += 1
else:
continue
tag_type_str = tag_type_str.rstrip()[:-1]
return tag_type_str
def gen_column_type_str(self, colname_prefix, column_elm_list):
column_index_start_num = 1
column_type_str = ""
if column_elm_list is None:
column_type_str = self.gen_default_column_str()
else:
for column_elm in column_elm_list:
if "count" in column_elm:
total_count = int(column_elm["count"])
else:
total_count = 1
if total_count > 0:
for _ in range(total_count):
column_type_str += f'{colname_prefix}{column_index_start_num} {column_elm["type"]}, '
if column_elm["type"] in ["varchar", "binary", "nchar"]:
column_type_str = column_type_str.rstrip()[:-1] + f'({column_elm["len"]}), '
column_index_start_num += 1
else:
continue
column_type_str = self.default_colts_name + " timestamp, " + column_type_str.rstrip()[:-1]
return column_type_str
def gen_random_type_value(self, type_name, binary_length, binary_type, nchar_length, nchar_type):
if type_name.lower() == "tinyint":
return random.randint(self.Boundary.TINYINT_BOUNDARY[0], self.Boundary.TINYINT_BOUNDARY[1])
elif type_name.lower() == "smallint":
return random.randint(self.Boundary.SMALLINT_BOUNDARY[0], self.Boundary.SMALLINT_BOUNDARY[1])
elif type_name.lower() == "int":
return random.randint(self.Boundary.INT_BOUNDARY[0], self.Boundary.INT_BOUNDARY[1])
elif type_name.lower() == "bigint":
return random.randint(self.Boundary.BIGINT_BOUNDARY[0], self.Boundary.BIGINT_BOUNDARY[1])
elif type_name.lower() == "tinyint unsigned":
return random.randint(self.Boundary.UTINYINT_BOUNDARY[0], self.Boundary.UTINYINT_BOUNDARY[1])
elif type_name.lower() == "smallint unsigned":
return random.randint(self.Boundary.USMALLINT_BOUNDARY[0], self.Boundary.USMALLINT_BOUNDARY[1])
elif type_name.lower() == "int unsigned":
return random.randint(self.Boundary.UINT_BOUNDARY[0], self.Boundary.UINT_BOUNDARY[1])
elif type_name.lower() == "bigint unsigned":
return random.randint(self.Boundary.UBIGINT_BOUNDARY[0], self.Boundary.UBIGINT_BOUNDARY[1])
elif type_name.lower() == "float":
return random.uniform(self.Boundary.FLOAT_BOUNDARY[0], self.Boundary.FLOAT_BOUNDARY[1])
elif type_name.lower() == "double":
return random.uniform(self.Boundary.FLOAT_BOUNDARY[0], self.Boundary.FLOAT_BOUNDARY[1])
elif type_name.lower() == "binary":
return f'{self.get_long_name(binary_length, binary_type)}'
elif type_name.lower() == "varchar":
return self.get_long_name(binary_length, binary_type)
elif type_name.lower() == "nchar":
return self.get_long_name(nchar_length, nchar_type)
elif type_name.lower() == "bool":
return random.choice(self.Boundary.BOOL_BOUNDARY)
elif type_name.lower() == "timestamp":
return self.genTs()[0]
else:
pass
def gen_tag_value_list(self, tag_elm_list):
tag_value_list = list()
if tag_elm_list is None:
tag_value_list = list(map(lambda i: self.gen_random_type_value(i, self.default_varchar_length, self.default_varchar_datatype, self.default_nchar_length, self.default_nchar_datatype), self.full_type_list))
else:
for tag_elm in tag_elm_list:
if "count" in tag_elm:
total_count = int(tag_elm["count"])
else:
total_count = 1
if total_count > 0:
for _ in range(total_count):
if tag_elm["type"] in ["varchar", "binary", "nchar"]:
tag_value_list.append(self.gen_random_type_value(tag_elm["type"], tag_elm["len"], self.default_varchar_datatype, tag_elm["len"], self.default_nchar_datatype))
else:
tag_value_list.append(self.gen_random_type_value(tag_elm["type"], "", "", "", ""))
else:
continue
return tag_value_list
def gen_column_value_list(self, column_elm_list, ts_value=None):
if ts_value is None:
ts_value = self.genTs()[0]
column_value_list = list()
if column_elm_list is None:
column_value_list = list(map(lambda i: self.gen_random_type_value(i, self.default_varchar_length, self.default_varchar_datatype, self.default_nchar_length, self.default_nchar_datatype), self.full_type_list))
else:
for column_elm in column_elm_list:
if "count" in column_elm:
total_count = int(column_elm["count"])
else:
total_count = 1
if total_count > 0:
for _ in range(total_count):
if column_elm["type"] in ["varchar", "binary", "nchar"]:
column_value_list.append(self.gen_random_type_value(column_elm["type"], column_elm["len"], self.default_varchar_datatype, column_elm["len"], self.default_nchar_datatype))
else:
column_value_list.append(self.gen_random_type_value(column_elm["type"], "", "", "", ""))
else:
continue
column_value_list = [self.ts_value] + self.column_value_list
return column_value_list
def create_stable(self, tsql, dbname=None, stbname="stb", column_elm_list=None, tag_elm_list=None,
count=1, default_stbname_prefix="stb", **kwargs):
colname_prefix = 'c'
tagname_prefix = 't'
stbname_index_start_num = 1
stb_params = ""
if len(kwargs) > 0:
for param, value in kwargs.items():
stb_params += f'{param} "{value}" '
column_type_str = self.gen_column_type_str(colname_prefix, column_elm_list)
tag_type_str = self.gen_tag_type_str(tagname_prefix, tag_elm_list)
if int(count) <= 1:
create_stable_sql = f'create table {dbname}.{stbname} ({column_type_str}) tags ({tag_type_str}) {stb_params};'
tdLog.info("create stb sql: %s"%create_stable_sql)
tsql.execute(create_stable_sql)
else:
for _ in range(count):
create_stable_sql = f'create table {dbname}.{default_stbname_prefix}{stbname_index_start_num} ({column_type_str}) tags ({tag_type_str}) {stb_params};'
stbname_index_start_num += 1
tsql.execute(create_stable_sql)
def create_ctable(self, tsql, dbname=None, stbname=None, tag_elm_list=None, count=1, default_ctbname_prefix="ctb", **kwargs):
ctbname_index_start_num = 0
ctb_params = ""
if len(kwargs) > 0:
for param, value in kwargs.items():
ctb_params += f'{param} "{value}" '
tag_value_list = self.gen_tag_value_list(tag_elm_list)
tag_value_str = ""
# tag_value_str = ", ".join(str(v) for v in self.tag_value_list)
for tag_value in tag_value_list:
if isinstance(tag_value, str):
tag_value_str += f'"{tag_value}", '
else:
tag_value_str += f'{tag_value}, '
tag_value_str = tag_value_str.rstrip()[:-1]
if int(count) <= 1:
create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};'
tsql.execute(create_ctable_sql)
else:
for _ in range(count):
create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};'
ctbname_index_start_num += 1
tdLog.info("create ctb sql: %s"%create_ctable_sql)
tsql.execute(create_ctable_sql)
def create_table(self, tsql, dbname=None, tbname="ntb", column_elm_list=None, count=1, **kwargs):
tbname_index_start_num = 1
tbname_prefix="ntb"
tb_params = ""
if len(kwargs) > 0:
for param, value in kwargs.items():
tb_params += f'{param} "{value}" '
column_type_str = self.gen_column_type_str(tbname_prefix, column_elm_list)
if int(count) <= 1:
create_table_sql = f'create table {dbname}.{tbname} ({column_type_str}) {tb_params};'
tsql.execute(create_table_sql)
else:
for _ in range(count):
create_table_sql = f'create table {dbname}.{tbname_prefix}{tbname_index_start_num} ({column_type_str}) {tb_params};'
tbname_index_start_num += 1
tsql.execute(create_table_sql)
def insert_rows(self, tsql, dbname=None, tbname=None, column_ele_list=None, start_ts_value=None, count=1):
if start_ts_value is None:
start_ts_value = self.genTs()[0]
column_value_list = self.gen_column_value_list(column_ele_list, start_ts_value)
# column_value_str = ", ".join(str(v) for v in self.column_value_list)
column_value_str = ""
for column_value in column_value_list:
if isinstance(column_value, str):
column_value_str += f'"{column_value}", '
else:
column_value_str += f'{column_value}, '
column_value_str = column_value_str.rstrip()[:-1]
if int(count) <= 1:
insert_sql = f'insert into {self.tb_name} values ({column_value_str});'
tsql.execute(insert_sql)
else:
for num in range(count):
column_value_list = self.gen_column_value_list(column_ele_list, f'{start_ts_value}+{num}s')
column_value_str = ", ".join(str(v) for v in column_value_list)
insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});'
tsql.execute(insert_sql)
tdCom = TDCom() tdCom = TDCom()

150
tests/system-test/2-query/twa.py Normal file
View File

@ -0,0 +1,150 @@
from util.log import *
from util.cases import *
from util.sql import *
import numpy as np
import random ,os ,sys
import platform
import math
class TDTestCase:
updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 ,
"jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143,
"wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143,
"maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 }
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.vnode_disbutes = None
self.ts = 1537146000000
self.tb_nums = 20
self.row_nums = 100
self.time_step = 1000
def prepare_datas_of_distribute(self):
# prepate datas for 20 tables distributed at different vgroups
tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5")
tdSql.execute(" use testdb ")
tdSql.execute(
'''create table stb1
(ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp)
tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32))
'''
)
for i in range(self.tb_nums):
tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )')
ts = self.ts
for j in range(self.row_nums):
ts+=j*self.time_step
tdSql.execute(
f"insert into ct{i+1} values({ts}, 1, 11111, 111, 1, 1.11, 11.11, 2, 'binary{j}', 'nchar{j}', now()+{1*j}a )"
)
tdSql.execute("insert into ct1 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct1 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdSql.execute("insert into ct1 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ")
tdLog.info(" prepare data for distributed_aggregate done! ")
def twa_support_types(self):
tdSql.query("desc stb1 ")
schema_list = tdSql.queryResult
for col_type in schema_list:
if col_type[1] in ["TINYINT" ,"SMALLINT","BIGINT" ,"INT","FLOAT","DOUBLE"]:
tdSql.query(f" select twa({col_type[0]}) from stb1 partition by tbname ")
else:
tdSql.error(f" select twa({col_type[0]}) from stb1 partition by tbname ")
def check_distribute_datas(self):
# get vgroup_ids of all
tdSql.query("show vgroups ")
vgroups = tdSql.queryResult
vnode_tables={}
for vgroup_id in vgroups:
vnode_tables[vgroup_id[0]]=[]
# check sub_table of per vnode ,make sure sub_table has been distributed
tdSql.query("show tables like 'ct%'")
table_names = tdSql.queryResult
tablenames = []
for table_name in table_names:
vnode_tables[table_name[6]].append(table_name[0])
self.vnode_disbutes = vnode_tables
count = 0
for k ,v in vnode_tables.items():
if len(v)>=2:
count+=1
if count < 2:
tdLog.exit(" the datas of all not satisfy sub_table has been distributed ")
def distribute_twa_query(self):
# basic filter
tdSql.query(" select twa(c1) from ct1 ")
tdSql.checkData(0,0,1.000000000)
tdSql.query(" select twa(c1) from stb1 partition by tbname ")
tdSql.checkRows(self.tb_nums)
tdSql.checkData(0,0,1.000000000)
tdSql.query(" select twa(c2) from stb1 group by tbname ")
tdSql.checkRows(self.tb_nums)
tdSql.checkData(0,0,11111.000000000)
tdSql.query("select twa(c1+c2) from stb1 partition by tbname ")
tdSql.checkData(0,0,11112.000000000)
tdSql.query("select twa(c1) from stb1 partition by t1")
tdSql.checkRows(self.tb_nums)
tdSql.checkData(0,0,1.000000000)
# union all
tdSql.query(" select twa(c1) from stb1 partition by tbname union all select twa(c1) from stb1 partition by tbname ")
tdSql.checkRows(40)
tdSql.checkData(0,0,1.000000000)
# join
tdSql.execute(" create database if not exists db ")
tdSql.execute(" use db ")
tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ")
tdSql.execute(" create table tb1 using st tags(1) ")
tdSql.execute(" create table tb2 using st tags(2) ")
for i in range(10):
ts = i*10 + self.ts
tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)")
tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)")
tdSql.query(" select twa(tb1.c1), twa(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts ")
tdSql.checkRows(1)
tdSql.checkData(0,0,4.500000000)
tdSql.checkData(0,1,4.500000000)
# group by
tdSql.execute(" use testdb ")
# mixup with other functions
tdSql.query(" select twa(c1),twa(c2),max(c1),elapsed(ts) from stb1 ")
tdSql.checkData(0,0,1.000000000)
tdSql.checkData(0,1,11111.000000000)
tdSql.checkData(0,2,1)
def run(self):
self.prepare_datas_of_distribute()
self.check_distribute_datas()
self.twa_support_types()
self.distribute_twa_query()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

96
tests/system-test/7-tmq/stbFilter.py Normal file
View File

@ -0,0 +1,96 @@
import taos
import sys
import time
import socket
import os
import threading
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.common import *
sys.path.append("./7-tmq")
from tmqCommon import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to excute {__file__}")
tdSql.init(conn.cursor())
#tdSql.init(conn.cursor(), logSql) # output sql.txt file
def tmqCase1(self):
tdLog.printNoPrefix("======== test case 1: ")
paraDict = {'dbName': 'db1',
'dropFlag': 1,
'event': '',
'vgroups': 4,
'stbName': 'stb',
'colPrefix': 'c',
'tagPrefix': 't',
'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}],
'ctbPrefix': 'ctb',
'ctbNum': 10,
'rowsPerTbl': 10000,
'batchNum': 100,
'startTs': 1640966400000, # 2022-01-01 00:00:00.000
'pollDelay': 10,
'showMsg': 1,
'showRow': 1}
tmqCom.initConsumerTable()
tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"])
tdLog.info("create stb")
tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
tdLog.info("create ctb")
tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
tdLog.info("insert data")
tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
tdLog.info("create topics from db")
topicName = 'topic_%s_%s'%(paraDict['dbName'], paraDict['stbName'])
tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s where c1 %% 4 == 0" %(topicName, paraDict['dbName'], paraDict['stbName']))
tdLog.info("insert consume info to consume processor")
consumerId = 0
expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"]
topicList = topicName
ifcheckdata = 0
ifManualCommit = 1
keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'
tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit)
tdLog.info("start consume processor")
tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow'])
tdLog.info("wait the consume result")
expectRows = 1
resultList = tmqCom.selectConsumeResult(expectRows)
totalConsumeRows = 0
for i in range(expectRows):
totalConsumeRows += resultList[i]
if totalConsumeRows != expectrowcnt / 4:
tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt / 4))
tdLog.exit("tmq consume rows error!")
time.sleep(10)
tdSql.query("drop topic %s"%topicName)
tdLog.printNoPrefix("======== test case 1 end ...... ")
def run(self):
tdSql.prepare()
self.tmqCase1()
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
event = threading.Event()
tdCases.addLinux(__file__, TDTestCase())
tdCases.addWindows(__file__, TDTestCase())

60
tests/system-test/7-tmq/tmqCommon.py
View File

@ -92,13 +92,65 @@ class TMQCom:
tdLog.info(shellCmd) tdLog.info(shellCmd)
os.system(shellCmd) os.system(shellCmd)
def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'):
while 1:
tdSql.query("select * from %s.notifyinfo"%cdbName)
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0):
break
else:
time.sleep(0.1)
return
def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'):
while 1:
tdSql.query("select * from %s.notifyinfo"%cdbName)
#tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
if tdSql.getRows() == 2 :
print(tdSql.getData(0, 1), tdSql.getData(1, 1))
if tdSql.getData(1, 1) == 1:
break
time.sleep(0.1)
return
def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
tdLog.debug("start to insert data ............")
tsql.execute("use %s" %dbName)
pre_insert = "insert into "
sql = pre_insert
t = time.time()
startTs = int(round(t * 1000))
#tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
for i in range(ctbNum):
sql += " %s%d values "%(stbName,i)
for j in range(rowsPerTbl):
sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
tsql.execute(sql)
if j < rowsPerTbl - 1:
sql = "insert into %s%d values " %(stbName,i)
else:
sql = "insert into "
#end sql
if sql != pre_insert:
#print("insert sql:%s"%sql)
tsql.execute(sql)
tdLog.debug("insert data ............ [OK]")
return
def syncCreateDbStbCtbInsertData(self, tsql, paraDict): def syncCreateDbStbCtbInsertData(self, tsql, paraDict):
tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"], paraDict['precision']) tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"])
tdCom.create_stable(tsql, paraDict["dbName"],paraDict["stbName"], paraDict["columnDict"], paraDict["tagDict"]) tdCom.create_stable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema'])
tdCom.create_ctables(tsql, paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["tagDict"]) tdCom.create_ctable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix'])
if "event" in paraDict and type(paraDict['event']) == type(threading.Event()): if "event" in paraDict and type(paraDict['event']) == type(threading.Event()):
paraDict["event"].set() paraDict["event"].set()
tdCom.insert_data(tsql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"])
ctbPrefix = paraDict['ctbPrefix']
ctbNum = paraDict["ctbNum"]
for i in range(ctbNum):
tbName = '%s%s'%(ctbPrefix,i)
tdCom.insert_rows(tsql,dbname=paraDict["dbName"],tbname=tbName,start_ts_value=paraDict['startTs'],count=paraDict['rowsPerTbl'])
return return
def threadFunction(self, **paraDict): def threadFunction(self, **paraDict):

1
tests/system-test/fulltest.sh
View File

@ -108,6 +108,7 @@ python3 ./test.py -f 2-query/distribute_agg_spread.py
python3 ./test.py -f 2-query/distribute_agg_apercentile.py python3 ./test.py -f 2-query/distribute_agg_apercentile.py
python3 ./test.py -f 2-query/distribute_agg_avg.py python3 ./test.py -f 2-query/distribute_agg_avg.py
python3 ./test.py -f 2-query/distribute_agg_stddev.py python3 ./test.py -f 2-query/distribute_agg_stddev.py
python3 ./test.py -f 2-query/twa.py
python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py
python3 ./test.py -f 6-cluster/5dnode2mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py