xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[TD-14388]<feature>: Support value type parameter processing in scalar function

This commit is contained in:
Ganlin Zhao 2022-03-29 18:58:36 +08:00
parent ab831ee814 7606fa91fa
commit 370218220a
82 changed files with 5500 additions and 4048 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.clang-format
View File

@ -5,7 +5,6 @@ AccessModifierOffset: -1
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: true
AlignConsecutiveMacros: true
AlignEscapedNewlinesLeft: true
AlignOperands: true
AlignTrailingComments: true

5
example/src/tmq.c
View File

@ -80,8 +80,9 @@ int32_t create_topic() {
}
taos_free_result(pRes);
const char* sql = "select * from tu1";
pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
/*const char* sql = "select * from tu1";*/
/*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1");
if (taos_errno(pRes) != 0) {
printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
return -1;

4
example/src/tstream.c
View File

@ -20,7 +20,7 @@
#include "taos.h"
int32_t init_env() {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
if (pConn == NULL) {
return -1;
}
@ -65,7 +65,7 @@ int32_t init_env() {
int32_t create_stream() {
printf("create stream\n");
TAOS_RES* pRes;
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
if (pConn == NULL) {
return -1;
}

78
include/client/consumer/consumer.h
View File

@ -1,78 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_CONSUMER_H_
#define _TD_CONSUMER_H_
#include "tlist.h"
#include "tarray.h"
#include "hash.h"
#ifdef __cplusplus
extern "C" {
#endif
//consumer handle
struct tmq_consumer_t;
typedef struct tmq_consumer_t tmq_consumer_t;
//consumer config
struct tmq_consumer_config_t;
typedef struct tmq_consumer_config_t tmq_consumer_config_t;
//response err
struct tmq_resp_err_t;
typedef struct tmq_resp_err_t tmq_resp_err_t;
struct tmq_message_t;
typedef struct tmq_message_t tmq_message_t;
struct tmq_col_batch_t;
typedef struct tmq_col_batch_t tmq_col_batch_t;
//get content of message
tmq_col_batch_t* tmq_get_msg_col_by_idx(tmq_message_t*, int32_t col_id);
tmq_col_batch_t* tmq_get_msg_col_by_name(tmq_message_t*, const char*);
//consumer config
int32_t tmq_conf_set(tmq_consumer_config_t* , const char* config_key, const char* config_value, char* errstr, int32_t errstr_cap);
//consumer initialization
//resouces are supposed to be free by users by calling tmq_consumer_destroy
tmq_consumer_t* tmq_consumer_new(tmq_consumer_config_t* , char* errstr, int32_t errstr_cap);
//subscribe
tmq_resp_err_t tmq_subscribe(tmq_consumer_t*, const SList*);
tmq_resp_err_t tmq_unsubscribe(tmq_consumer_t*);
//consume
//resouces are supposed to be free by users by calling tmq_message_destroy
tmq_message_t* tmq_consume_poll(tmq_consumer_t*, int64_t blocking_time);
//destroy message and free memory
void tmq_message_destroy(tmq_message_t*);
//close consumer
int32_t tmq_consumer_close(tmq_consumer_t*);
//destroy consumer
void tmq_consumer_destroy(tmq_message_t*);
#ifdef __cplusplus
}
#endif
#endif /*_TD_CONSUMER_H_*/

27
include/client/stream/stream.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_STREAM_H_
#define _TD_STREAM_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_STREAM_H_*/

40
include/client/taos.h
View File

@ -26,9 +26,11 @@ extern "C" {
typedef void TAOS;
typedef void TAOS_STMT;
typedef void TAOS_RES;
typedef void TAOS_STREAM;
typedef void TAOS_SUB;
typedef void **TAOS_ROW;
#if 0
typedef void TAOS_STREAM;
#endif
typedef void TAOS_SUB;
// Data type definition
#define TSDB_DATA_TYPE_NULL 0 // 1 bytes
@ -149,6 +151,7 @@ DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt);
DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
@ -179,15 +182,20 @@ DLL_EXPORT int taos_errno(TAOS_RES *tres);
DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
// Shuduo: temporary enable for app build
#if 1
typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp,
void *param, int interval);
DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
#endif
#if 0
DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
int64_t stime, void *param, void (*callback)(void *));
DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
#endif
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
@ -215,8 +223,12 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
// will be removed in 3.0
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
// will replace last one
DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
/* ------------------------TMQ CONSUMER INTERFACE------------------------ */
@ -255,20 +267,26 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message);
DLL_EXPORT void *tmq_get_topic_schema(tmq_t *tmq, const char *topic);
DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message);
DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message);
DLL_EXPORT int32_t tmq_get_vgroup_id(tmq_message_t *message);
DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message);
DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message);
DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
DLL_EXPORT int32_t tmq_field_count(tmq_t *tmq, const char *topic);
DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message);
/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
#if 0
DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
#endif
DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
/* -------------------------------- OTHER -------------------------------- */
/* ------------------------------ TMQ END -------------------------------- */
#if 1 // Shuduo: temporary enable for app build
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
#endif
#ifdef __cplusplus
}

56
include/common/tmsg.h
View File

@ -2215,23 +2215,6 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
return buf;
}
typedef struct {
int64_t uid;
int32_t numOfRows;
char* colData;
} SMqTbData;
typedef struct {
char topicName[TSDB_TOPIC_FNAME_LEN];
int64_t committedOffset;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t bodyLen;
int32_t numOfTb;
SMqTbData* tbData;
} SMqTopicData;
typedef struct {
int8_t mqMsgType;
int32_t code;
@ -2259,8 +2242,11 @@ typedef struct {
} SMqSubVgEp;
typedef struct {
char topic[TSDB_TOPIC_FNAME_LEN];
SArray* vgs; // SArray<SMqSubVgEp>
char topic[TSDB_TOPIC_FNAME_LEN];
int8_t isSchemaAdaptive;
SArray* vgs; // SArray<SMqSubVgEp>
int32_t numOfFields;
TAOS_FIELD* fields;
} SMqSubTopicEp;
typedef struct {
@ -2281,32 +2267,6 @@ typedef struct {
SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp;
typedef struct {
int32_t curBlock;
int32_t curRow;
void** uData;
} SMqRowIter;
struct tmq_message_t {
SMqPollRsp msg;
void* vg;
SMqRowIter iter;
};
#if 0
struct tmq_message_t {
SMqRspHead head;
union {
SMqPollRsp consumeRsp;
SMqCMGetSubEpRsp getEpRsp;
};
void* extra;
int32_t curBlock;
int32_t curRow;
void** uData;
};
#endif
static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
@ -2331,17 +2291,21 @@ static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) {
static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pTopicEp->topic);
tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive);
int32_t sz = taosArrayGetSize(pTopicEp->vgs);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i);
tlen += tEncodeSMqSubVgEp(buf, pVgEp);
}
tlen += taosEncodeFixedI32(buf, pTopicEp->numOfFields);
// tlen += taosEncodeBinary(buf, pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
return tlen;
}
static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) {
buf = taosDecodeStringTo(buf, pTopicEp->topic);
buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp));
@ -2353,6 +2317,8 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
buf = tDecodeSMqSubVgEp(buf, &vgEp);
taosArrayPush(pTopicEp->vgs, &vgEp);
}
buf = taosDecodeFixedI32(buf, &pTopicEp->numOfFields);
// buf = taosDecodeBinary(buf, (void**)&pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
return buf;
}

27
include/common/tmsgcb.h
View File

@ -25,6 +25,7 @@ extern "C" {
typedef struct SRpcMsg SRpcMsg;
typedef struct SEpSet SEpSet;
typedef struct SMgmtWrapper SMgmtWrapper;
typedef enum {
QUERY_QUEUE,
FETCH_QUEUE,
@ -38,24 +39,30 @@ typedef enum {
typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, SEpSet* epSet, SRpcMsg* pReq);
typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq);
typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, SRpcMsg* pRsp);
typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp);
typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type);
typedef struct {
SMgmtWrapper* pWrapper;
PutToQueueFp queueFps[QUEUE_MAX];
GetQueueSizeFp qsizeFp;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
SMgmtWrapper* pWrapper;
PutToQueueFp queueFps[QUEUE_MAX];
GetQueueSizeFp qsizeFp;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
ReleaseHandleFp releaseHandleFp;
} SMsgCb;
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq);
int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq);
void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp);
void tmsgSendRsp(const SMsgCb* pMsgCb, const SRpcMsg* pRsp);
void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
void tmsgReleaseHandle(const SMsgCb* pMsgCb, void* handle, int8_t type);
#ifdef __cplusplus
}

24
include/common/ttokendef.h
View File

@ -139,18 +139,18 @@
#define TK_TOPIC 121
#define TK_AS 122
#define TK_NK_BOOL 123
#define TK_NK_VARIABLE 124
#define TK_NK_UNDERLINE 125
#define TK_ROWTS 126
#define TK_TBNAME 127
#define TK_QSTARTTS 128
#define TK_QENDTS 129
#define TK_WSTARTTS 130
#define TK_WENDTS 131
#define TK_WDURATION 132
#define TK_BETWEEN 133
#define TK_IS 134
#define TK_NULL 135
#define TK_NULL 124
#define TK_NK_VARIABLE 125
#define TK_NK_UNDERLINE 126
#define TK_ROWTS 127
#define TK_TBNAME 128
#define TK_QSTARTTS 129
#define TK_QENDTS 130
#define TK_WSTARTTS 131
#define TK_WENDTS 132
#define TK_WDURATION 133
#define TK_BETWEEN 134
#define TK_IS 135
#define TK_NK_LT 136
#define TK_NK_GT 137
#define TK_NK_LE 138

7
include/libs/executor/executor.h
View File

@ -20,8 +20,8 @@
extern "C" {
#endif
#include "tcommon.h"
#include "query.h"
#include "tcommon.h"
typedef void* qTaskInfo_t;
typedef void* DataSinkHandle;
@ -36,7 +36,7 @@ typedef struct SReadHandle {
#define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
#define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
/**
* Create the exec task for streaming mode
* @param pMsg
@ -62,7 +62,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type);
* @param type
* @return
*/
int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, void** pBlocks, size_t numOfBlocks, int32_t type);
int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type);
/**
* Update the table id list, add or remove.
@ -167,7 +167,6 @@ int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
*/
void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle);
void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
#ifdef __cplusplus

1
include/libs/nodes/nodes.h
View File

@ -121,6 +121,7 @@ typedef enum ENodeType {
QUERY_NODE_LOGIC_PLAN_EXCHANGE,
QUERY_NODE_LOGIC_PLAN_WINDOW,
QUERY_NODE_LOGIC_PLAN_SORT,
QUERY_NODE_LOGIC_PLAN_PARTITION,
QUERY_NODE_LOGIC_SUBPLAN,
QUERY_NODE_LOGIC_PLAN,

25
include/libs/nodes/plannodes.h
View File

@ -66,7 +66,11 @@ typedef struct SAggLogicNode {
typedef struct SProjectLogicNode {
SLogicNode node;
SNodeList* pProjections;
char stmtName[TSDB_TABLE_NAME_LEN];
char stmtName[TSDB_TABLE_NAME_LEN];
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
} SProjectLogicNode;
typedef struct SVnodeModifLogicNode {
@ -106,6 +110,11 @@ typedef struct SSortLogicNode {
SNodeList* pSortKeys;
} SSortLogicNode;
typedef struct SPartitionLogicNode {
SLogicNode node;
SNodeList* pPartitionKeys;
} SPartitionLogicNode;
typedef enum ESubplanType {
SUBPLAN_TYPE_MERGE = 1,
SUBPLAN_TYPE_PARTIAL,
@ -150,7 +159,8 @@ typedef struct SDataBlockDescNode {
ENodeType type;
int16_t dataBlockId;
SNodeList* pSlots;
int32_t resultRowSize;
int32_t totalRowSize;
int32_t outputRowSize;
int16_t precision;
} SDataBlockDescNode;
@ -195,6 +205,10 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode;
typedef struct SProjectPhysiNode {
SPhysiNode node;
SNodeList* pProjections;
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
} SProjectPhysiNode;
typedef struct SJoinPhysiNode {
@ -283,10 +297,17 @@ typedef struct SSubplan {
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
} SSubplan;
typedef enum EQueryMode {
QUERY_MODE_NORMAL = 1,
QUERY_MODE_EXPLAIN,
QUERY_MODE_EXPLAIN_AN
} EQueryMode;
typedef struct SQueryPlan {
ENodeType type;
uint64_t queryId;
int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
} SQueryPlan;

4
include/libs/nodes/querynodes.h
View File

@ -180,8 +180,8 @@ typedef struct SOrderByExprNode {
typedef struct SLimitNode {
ENodeType type; // QUERY_NODE_LIMIT
uint64_t limit;
uint64_t offset;
int64_t limit;
int64_t offset;
} SLimitNode;
typedef struct SStateWindowNode {

5
include/libs/stream/tstream.h
View File

@ -30,6 +30,11 @@ enum {
STREAM_TASK_STATUS__STOP,
};
enum {
STREAM_CREATED_BY__USER = 1,
STREAM_CREATED_BY__SMA,
};
#if 0
// pipe -> fetch/pipe queue
// merge -> merge queue

30
include/libs/transport/trpc.h
View File

@ -49,11 +49,13 @@ typedef struct SRpcMsg {
} SRpcMsg;
typedef struct {
char user[TSDB_USER_LEN];
SRpcMsg rpcMsg;
int32_t rspLen;
void * pRsp;
void * pNode;
char user[TSDB_USER_LEN];
uint32_t clientIp;
uint16_t clientPort;
SRpcMsg rpcMsg;
int32_t rspLen;
void *pRsp;
void *pNode;
} SNodeMsg;
typedef struct SRpcInit {
@ -105,20 +107,20 @@ void rpcClose(void *);
void * rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void * rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
// Because taosd supports multi-process mode
// These functions should not be used on the server side
// Please use tmsg<xx> functions, which are defined in tmsgcb.h
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
void rpcReleaseHandle(void *handle, int8_t type); // just release client conn to rpc instance, no close sock
// These functions will not be called in the child process
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(int64_t rid);
void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
// just release client conn to rpc instance, no close sock
void rpcReleaseHandle(void *handle, int8_t type); //
void rpcRefHandle(void *handle, int8_t type);
void rpcUnrefHandle(void *handle, int8_t type);
#ifdef __cplusplus
}

12
include/util/tprocess.h
View File

@ -22,11 +22,14 @@
extern "C" {
#endif
typedef enum { PROC_REQ, PROC_RSP, PROC_REGISTER } ProcFuncType;
typedef struct SProcQueue SProcQueue;
typedef struct SProcObj SProcObj;
typedef void *(*ProcMallocFp)(int32_t contLen);
typedef void *(*ProcFreeFp)(void *pCont);
typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
ProcFuncType ftype);
typedef struct {
int32_t childQueueSize;
@ -52,9 +55,10 @@ int32_t taosProcRun(SProcObj *pProc);
void taosProcStop(SProcObj *pProc);
bool taosProcIsChild(SProcObj *pProc);
int32_t taosProcChildId(SProcObj *pProc);
int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
ProcFuncType ftype);
int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
ProcFuncType ftype);
#ifdef __cplusplus
}

231
source/client/src/clientMain.c
View File

@ -1,12 +1,27 @@
#include "os.h"
#include "tref.h"
#include "trpc.h"
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "query.h"
#include "tmsg.h"
#include "tglobal.h"
#include "catalog.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "version.h"
#define TSC_VAR_NOT_RELEASE 1
@ -24,7 +39,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
}
}
int ret = taos_options_imp(option, (const char*)arg);
int ret = taos_options_imp(option, (const char *)arg);
atomic_store_32(&lock, 0);
return ret;
}
@ -69,13 +84,13 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
return taos_connect_internal(ip, user, pass, NULL, db, port);
}
void taos_close(TAOS* taos) {
void taos_close(TAOS *taos) {
if (taos == NULL) {
return;
}
STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
/*taosRemoveRef(clientConnRefPool, pTscObj->id);*/
}
@ -85,48 +100,46 @@ int taos_errno(TAOS_RES *tres) {
return terrno;
}
return ((SRequestObj*) tres)->code;
return ((SRequestObj *)tres)->code;
}
const char *taos_errstr(TAOS_RES *res) {
SRequestObj *pRequest = (SRequestObj *) res;
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest == NULL) {
return (const char*) tstrerror(terrno);
return (const char *)tstrerror(terrno);
}
if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) {
return pRequest->msgBuf;
} else {
return (const char*)tstrerror(pRequest->code);
return (const char *)tstrerror(pRequest->code);
}
}
void taos_free_result(TAOS_RES *res) {
SRequestObj* pRequest = (SRequestObj*) res;
SRequestObj *pRequest = (SRequestObj *)res;
destroyRequest(pRequest);
}
int taos_field_count(TAOS_RES *res) {
int taos_field_count(TAOS_RES *res) {
if (res == NULL) {
return 0;
}
SRequestObj* pRequest = (SRequestObj*) res;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SRequestObj *pRequest = (SRequestObj *)res;
SReqResultInfo *pResInfo = &pRequest->body.resInfo;
return pResInfo->numOfCols;
}
int taos_num_fields(TAOS_RES *res) {
return taos_field_count(res);
}
int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
if (taos_num_fields(res) == 0) {
return NULL;
}
SReqResultInfo* pResInfo = &(((SRequestObj*) res)->body.resInfo);
SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo);
return pResInfo->fields;
}
@ -135,7 +148,7 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) {
return NULL;
}
return taos_query_l(taos, sql, (int32_t) strlen(sql));
return taos_query_l(taos, sql, (int32_t)strlen(sql));
}
TAOS_ROW taos_fetch_row(TAOS_RES *res) {
@ -143,18 +156,16 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL;
}
SRequestObj *pRequest = (SRequestObj *) res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS ||
taos_num_fields(res) == 0) {
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
return NULL;
}
return doFetchRow(pRequest, true);
}
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
int32_t len = 0;
for (int i = 0; i < num_fields; ++i) {
if (i > 0) {
@ -213,7 +224,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE);
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY) {
assert(charLen <= fields[i].bytes && charLen >= 0);
} else {
@ -238,31 +249,44 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
return len;
}
int* taos_fetch_lengths(TAOS_RES *res) {
int *taos_fetch_lengths(TAOS_RES *res) {
if (res == NULL) {
return NULL;
}
return ((SRequestObj*) res)->body.resInfo.length;
return ((SRequestObj *)res)->body.resInfo.length;
}
// todo intergrate with tDataTypes
const char *taos_data_type(int type) {
switch (type) {
case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL";
case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL";
case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT";
case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT";
case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT";
case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT";
case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT";
case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE";
case TSDB_DATA_TYPE_VARCHAR: return "TSDB_DATA_TYPE_VARCHAR";
// case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_VARCHAR";
case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP";
case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR";
case TSDB_DATA_TYPE_JSON: return "TSDB_DATA_TYPE_JSON";
default: return "UNKNOWN";
case TSDB_DATA_TYPE_NULL:
return "TSDB_DATA_TYPE_NULL";
case TSDB_DATA_TYPE_BOOL:
return "TSDB_DATA_TYPE_BOOL";
case TSDB_DATA_TYPE_TINYINT:
return "TSDB_DATA_TYPE_TINYINT";
case TSDB_DATA_TYPE_SMALLINT:
return "TSDB_DATA_TYPE_SMALLINT";
case TSDB_DATA_TYPE_INT:
return "TSDB_DATA_TYPE_INT";
case TSDB_DATA_TYPE_BIGINT:
return "TSDB_DATA_TYPE_BIGINT";
case TSDB_DATA_TYPE_FLOAT:
return "TSDB_DATA_TYPE_FLOAT";
case TSDB_DATA_TYPE_DOUBLE:
return "TSDB_DATA_TYPE_DOUBLE";
case TSDB_DATA_TYPE_VARCHAR:
return "TSDB_DATA_TYPE_VARCHAR";
// case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_VARCHAR";
case TSDB_DATA_TYPE_TIMESTAMP:
return "TSDB_DATA_TYPE_TIMESTAMP";
case TSDB_DATA_TYPE_NCHAR:
return "TSDB_DATA_TYPE_NCHAR";
case TSDB_DATA_TYPE_JSON:
return "TSDB_DATA_TYPE_JSON";
default:
return "UNKNOWN";
}
}
@ -273,8 +297,8 @@ int taos_affected_rows(TAOS_RES *res) {
return 0;
}
SRequestObj* pRequest = (SRequestObj*) res;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SRequestObj *pRequest = (SRequestObj *)res;
SReqResultInfo *pResInfo = &pRequest->body.resInfo;
return pResInfo->numOfRows;
}
@ -295,8 +319,8 @@ int taos_select_db(TAOS *taos, const char *db) {
char sql[256] = {0};
snprintf(sql, tListLen(sql), "use %s", db);
TAOS_RES* pRequest = taos_query(taos, sql);
int32_t code = taos_errno(pRequest);
TAOS_RES *pRequest = taos_query(taos, sql);
int32_t code = taos_errno(pRequest);
taos_free_result(pRequest);
return code;
@ -307,61 +331,57 @@ void taos_stop_query(TAOS_RES *res) {
return;
}
SRequestObj* pRequest = (SRequestObj*) res;
int32_t numOfFields = taos_num_fields(pRequest);
SRequestObj *pRequest = (SRequestObj *)res;
int32_t numOfFields = taos_num_fields(pRequest);
// It is not a query, no need to stop.
if (numOfFields == 0) {
return;
}
// scheduleCancelJob(pRequest->body.pQueryJob);
// scheduleCancelJob(pRequest->body.pQueryJob);
}
bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
SRequestObj* pRequestObj = res;
SReqResultInfo* pResultInfo = &pRequestObj->body.resInfo;
SRequestObj *pRequestObj = res;
SReqResultInfo *pResultInfo = &pRequestObj->body.resInfo;
if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) {
return true;
}
SResultColumn* pCol = &pRequestObj->body.resInfo.pCol[col];
SResultColumn *pCol = &pRequestObj->body.resInfo.pCol[col];
return colDataIsNull_f(pCol->nullbitmap, row);
}
int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
if (res == NULL) {
return 0;
}
SRequestObj *pRequest = (SRequestObj *) res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS ||
taos_num_fields(res) == 0) {
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
return 0;
}
doFetchRow(pRequest, false);
// TODO refactor
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
pResultInfo->current = pResultInfo->numOfRows;
*rows = pResultInfo->row;
return pResultInfo->numOfRows;
}
int taos_validate_sql(TAOS *taos, const char *sql) {
return true;
}
int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
const char *taos_get_server_info(TAOS *taos) {
if (taos == NULL) {
return NULL;
}
STscObj* pTscObj = (STscObj*) taos;
STscObj *pTscObj = (STscObj *)taos;
return pTscObj->ver;
}
@ -387,52 +407,62 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
// TODO
}
TAOS_STMT* taos_stmt_init(TAOS* taos) {
// TODO
return NULL;
TAOS_STMT *taos_stmt_init(TAOS *taos) {
// TODO
return NULL;
}
int taos_stmt_close(TAOS_STMT* stmt) {
// TODO
return -1;
int taos_stmt_close(TAOS_STMT *stmt) {
// TODO
return -1;
}
int taos_stmt_execute(TAOS_STMT* stmt) {
// TODO
return -1;
int taos_stmt_execute(TAOS_STMT *stmt) {
// TODO
return -1;
}
char *taos_stmt_errstr(TAOS_STMT *stmt) {
// TODO
return NULL;
// TODO
return NULL;
}
int taos_stmt_affected_rows(TAOS_STMT* stmt) {
int taos_stmt_affected_rows(TAOS_STMT *stmt) {
// TODO
return -1;
}
TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
// TODO
return NULL;
}
int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) {
// TODO
return -1;
}
int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
// TODO
return -1;
}
int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) {
// TODO
return -1;
}
int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
// TODO
return -1;
}
int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
// TODO
return -1;
}
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
// TODO
return NULL;
}
int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) {
// TODO
return -1;
}
int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
// TODO
return -1;
}
int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) {
// TODO
return -1;
}
int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) {
int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
// TODO
return -1;
}
@ -442,6 +472,11 @@ int taos_stmt_add_batch(TAOS_STMT* stmt) {
return -1;
}
TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
// TODO
return NULL;
}
int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) {
// TODO
return -1;

40
source/client/src/tmq.c
View File

@ -26,6 +26,18 @@
#include "tqueue.h"
#include "tref.h"
typedef struct {
int32_t curBlock;
int32_t curRow;
void** uData;
} SMqRowIter;
struct tmq_message_t {
SMqPollRsp msg;
void* vg;
SMqRowIter iter;
};
struct tmq_list_t {
SArray container;
};
@ -99,13 +111,14 @@ typedef struct {
typedef struct {
// subscribe info
int32_t sqlLen;
char* sql;
char* topicName;
int64_t topicId;
int32_t nextVgIdx;
SArray* vgs; // SArray<SMqClientVg>
SSchemaWrapper schema;
int32_t sqlLen;
char* sql;
char* topicName;
int64_t topicId;
SArray* vgs; // SArray<SMqClientVg>
int8_t isSchemaAdaptive;
int32_t numOfFields;
TAOS_FIELD* fields;
} SMqClientTopic;
typedef struct {
@ -130,11 +143,11 @@ typedef struct {
} SMqPollCbParam;
typedef struct {
tmq_t* tmq;
/*SMqClientVg* pVg;*/
tmq_t* tmq;
int32_t async;
tsem_t rspSem;
tmq_resp_err_t rspErr;
/*SMqClientVg* pVg;*/
} SMqCommitCbParam;
tmq_conf_t* tmq_conf_new() {
@ -471,7 +484,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
tNameExtractFullName(&name, topicFname);
tscDebug("subscribe topic: %s", topicFname);
SMqClientTopic topic = {
.nextVgIdx = 0, .sql = NULL, .sqlLen = 0, .topicId = 0, .topicName = topicFname, .vgs = NULL};
.sql = NULL,
.sqlLen = 0,
.topicId = 0,
.topicName = topicFname,
.vgs = NULL,
};
topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
taosArrayPush(tmq->clientTopics, &topic);
taosArrayPush(req.topicNames, &topicFname);
@ -615,6 +633,7 @@ _return:
return pRequest;
}
#if 0
TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos;
SRequestObj* pRequest = NULL;
@ -700,6 +719,7 @@ _return:
return pRequest;
}
#endif
static char* formatTimestamp(char* buf, int64_t val, int precision) {
time_t tt;

14
source/client/test/tmqTest.cpp
View File

@ -35,7 +35,7 @@ int main(int argc, char** argv) {
TEST(testCase, driverInit_Test) {
// taosInitGlobalCfg();
// taos_init();
// taos_init();
}
TEST(testCase, create_topic_ctb_Test) {
@ -55,8 +55,9 @@ TEST(testCase, create_topic_ctb_Test) {
taos_free_result(pRes);
char* sql = "select * from tu";
pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
// char* sql = "select * from tu";
// pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from tu");
taos_free_result(pRes);
taos_close(pConn);
}
@ -69,7 +70,7 @@ TEST(testCase, create_topic_stb_Test) {
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
}
//taos_free_result(pRes);
// taos_free_result(pRes);
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == nullptr);
@ -79,8 +80,9 @@ TEST(testCase, create_topic_stb_Test) {
taos_free_result(pRes);
char* sql = "select * from st1";
pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
// char* sql = "select * from st1";
// pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from st1");
taos_free_result(pRes);
taos_close(pConn);
}

12
source/common/src/tmsgcb.c
View File

@ -24,7 +24,7 @@ int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype);
}
int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq) {
int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) {
return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
}
@ -32,4 +32,12 @@ int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) {
return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq);
}
void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); }
void tmsgSendRsp(const SMsgCb* pMsgCb, const SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); }
void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {
(*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg);
}
void tmsgReleaseHandle(const SMsgCb* pMsgCb, void* handle, int8_t type) {
(*pMsgCb->releaseHandleFp)(pMsgCb->pWrapper, handle, type);
}

1
source/dnode/mgmt/bnode/src/bmInt.c
View File

@ -24,6 +24,7 @@ static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

5
source/dnode/mgmt/container/inc/dnd.h
View File

@ -137,8 +137,9 @@ void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc
void dndSendMonitorReport(SDnode *pDnode);
int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg);
void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);

1
source/dnode/mgmt/container/inc/dndInt.h
View File

@ -56,6 +56,7 @@ void dndCleanupServer(SDnode *pDnode);
int32_t dndInitClient(SDnode *pDnode);
void dndCleanupClient(SDnode *pDnode);
int32_t dndInitMsgHandle(SDnode *pDnode);
void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
#ifdef __cplusplus
}

27
source/dnode/mgmt/container/src/dndExec.c
View File

@ -116,11 +116,12 @@ static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
}
}
static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) {
static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
ProcFuncType ftype) {
SRpcMsg *pRpc = &pMsg->rpcMsg;
pRpc->pCont = pCont;
dTrace("msg:%p, get from child process queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pRpc->msgType),
pRpc->handle, pRpc->ahandle);
dTrace("msg:%p, get from child queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pRpc->msgType), pRpc->handle,
pRpc->ahandle);
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
int32_t code = (*msgFp)(pWrapper, pMsg);
@ -138,13 +139,21 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t
}
}
static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, int32_t msgLen, void *pCont, int32_t contLen) {
pRpc->pCont = pCont;
dTrace("msg:%p, get from parent process queue, type:%s handle:%p app:%p", pRpc, TMSG_INFO(pRpc->msgType),
pRpc->handle, pRpc->ahandle);
static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
ProcFuncType ftype) {
pMsg->pCont = pCont;
dTrace("msg:%p, get from parent queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pMsg->msgType), pMsg->handle,
pMsg->ahandle);
dndSendRsp(pWrapper, pRpc);
taosMemoryFree(pRpc);
switch (ftype) {
case PROC_REGISTER:
rpcRegisterBrokenLinkArg(pMsg);
break;
default:
dndSendRpcRsp(pWrapper, pMsg);
break;
}
taosMemoryFree(pMsg);
}
static int32_t dndRunInMultiProcess(SDnode *pDnode) {

4
source/dnode/mgmt/container/src/dndMsg.c
View File

@ -42,6 +42,8 @@ static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
}
memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN);
pMsg->clientIp = connInfo.clientIp;
pMsg->clientPort = connInfo.clientPort;
memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
return 0;
}
@ -66,7 +68,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
} else if (pWrapper->procType == PROC_PARENT) {
dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
pRpc->ahandle, pMsg->user);
code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen);
code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ);
} else {
dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle,
pMsg->user);

24
source/dnode/mgmt/container/src/dndTransport.c
View File

@ -309,7 +309,7 @@ int32_t dndInitMsgHandle(SDnode *pDnode) {
return 0;
}
static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *pReq) {
if (pMgmt->clientRpc == NULL) {
terrno = TSDB_CODE_DND_OFFLINE;
return -1;
@ -319,7 +319,7 @@ static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
return 0;
}
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) {
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) {
if (pWrapper->procType == PROC_CHILD) {
} else {
SDnode *pDnode = pWrapper->pDnode;
@ -348,7 +348,7 @@ int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
}
}
static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
if (pRsp->code == TSDB_CODE_APP_NOT_READY) {
SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE);
if (pDnodeWrapper != NULL) {
@ -362,11 +362,11 @@ static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
}
}
void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
if (pWrapper->procType == PROC_CHILD) {
int32_t code = -1;
do {
code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen);
code = taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP);
if (code != 0) {
taosMsleep(10);
}
@ -375,3 +375,17 @@ void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
dndSendRpcRsp(pWrapper, pRsp);
}
}
void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
if (pWrapper->procType == PROC_CHILD) {
int32_t code = -1;
do {
code = taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REGISTER);
if (code != 0) {
taosMsleep(10);
}
} while (code != 0);
} else {
rpcRegisterBrokenLinkArg(pMsg);
}
}

2
source/dnode/mgmt/dnode/inc/dm.h
View File

@ -29,7 +29,7 @@ void dmInitMsgHandles(SMgmtWrapper *pWrapper);
void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg);
#ifdef __cplusplus
}

2
source/dnode/mgmt/dnode/src/dmInt.c
View File

@ -54,7 +54,7 @@ void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqd
taosRUnLockLatch(&pMgmt->latch);
}
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) {
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pReq) {
SDnode *pDnode = pMgmt->pDnode;
SEpSet epSet = {0};

1
source/dnode/mgmt/mnode/src/mmInt.c
View File

@ -52,6 +52,7 @@ static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

1
source/dnode/mgmt/qnode/src/qmInt.c
View File

@ -27,6 +27,7 @@ static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

1
source/dnode/mgmt/snode/src/smInt.c
View File

@ -24,6 +24,7 @@ static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

1
source/dnode/mgmt/vnode/src/vmInt.c
View File

@ -137,6 +137,7 @@ static void *vmOpenVnodeFunc(void *param) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
SVnode *pImpl = vnodeOpen(pCfg->path, &cfg);
if (pImpl == NULL) {

1
source/dnode/mgmt/vnode/src/vmMsg.c
View File

@ -91,6 +91,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
vnodeCfg.msgCb = msgCb;
vnodeCfg.pTfs = pMgmt->pTfs;

6
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -731,10 +731,10 @@ typedef struct {
int32_t vgNum;
SRWLatch lock;
int8_t status;
int8_t sourceType;
int8_t sinkType;
// int32_t sqlLen;
int32_t sinkVgId; // 0 for automatic
int8_t createdBy; // STREAM_CREATED_BY__USER or SMA
int32_t fixedSinkVgId; // 0 for shuffle
int64_t smaId; // 0 for unused
char* sql;
char* logicalPlan;
char* physicalPlan;

2
source/dnode/mnode/impl/inc/mndScheduler.h
View File

@ -27,7 +27,7 @@ void mndCleanupScheduler(SMnode* pMnode);
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId);
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
#ifdef __cplusplus
}

2
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -31,7 +31,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId);
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans);
#ifdef __cplusplus
}

10
source/dnode/mnode/impl/src/mndDef.c
View File

@ -26,8 +26,11 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
/*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
// TODO encode tasks
if (pObj->tasks) {
@ -69,8 +72,11 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;
/*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
pObj->tasks = NULL;
int32_t sz;

312
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -20,138 +20,178 @@
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
//!!!! Note: only APPEND columns in below tables, NO insert !!!!
static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "wallevel", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
// {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update
};
static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userIdxSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userStbsSchema[] = {{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY},
{.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema dnodesSchema[] = {
{.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema mnodesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema modulesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema qnodesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema userDBSchema[] = {
{.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "wallevel", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
// {.name = "update", .bytes = 1, .type =
// TSDB_DATA_TYPE_TINYINT}, // disable update
};
static const SInfosTableSchema userFuncSchema[] = {
{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userIdxSchema[] = {
{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userStbsSchema[] = {
{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userStreamsSchema[] = {
{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY},
{.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userTblsSchema[] = {
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY},
{.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE},
{.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userUsersSchema[] = {{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema vgroupsSchema[] = {{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userTblDistSchema[] = {
{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY},
{.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE},
{.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userUsersSchema[] = {
{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema vgroupsSchema[] = {
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
{TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
{TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
{TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
{TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
{TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
{TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
{TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
{TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
};
// TODO put into perf schema
static const SInfosTableSchema topicSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
//connection/application/
static const SInfosTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// ep
// up time
// topics
};
static const SInfosTableSchema subscribeSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
{TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
{TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
{TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
{TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
{TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
{TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
{TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
{TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
};
// connection/application/
int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) {
@ -161,7 +201,7 @@ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, S
for (int32_t i = 0; i < colNum; ++i) {
strcpy(schema[i].name, pSrc[i].name);
schema[i].type = pSrc[i].type;
schema[i].colId = i + 1;
schema[i].bytes = pSrc[i].bytes;
@ -182,11 +222,11 @@ int32_t mndInsInitMeta(SHashObj *hash) {
for (int32_t i = 0; i < tListLen(infosMeta); ++i) {
strcpy(meta.tbName, infosMeta[i].name);
meta.numOfColumns = infosMeta[i].colNum;
if (mndInitInfosTableSchema(infosMeta[i].schema, infosMeta[i].colNum, &meta.pSchemas)) {
return -1;
}
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -210,7 +250,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
}
*pRsp = *meta;
pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -237,13 +277,13 @@ void mndCleanupInfos(SMnode *pMnode) {
if (NULL == pMnode->infosMeta) {
return;
}
void *pIter = taosHashIterate(pMnode->infosMeta, NULL);
while (pIter) {
STableMetaRsp *meta = (STableMetaRsp *)pIter;
taosMemoryFreeClear(meta->pSchemas);
pIter = taosHashIterate(pMnode->infosMeta, pIter);
}
@ -251,5 +291,3 @@ void mndCleanupInfos(SMnode *pMnode) {
pMnode->infosMeta = NULL;
}

31
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -44,7 +44,8 @@ typedef struct {
SQueryDesc *pQueries;
} SConnObj;
static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime);
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid,
const char *app, int64_t startTime);
static void mndFreeConn(SConnObj *pConn);
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
@ -94,7 +95,8 @@ void mndCleanupProfile(SMnode *pMnode) {
}
}
static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime) {
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid,
const char *app, int64_t startTime) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1);
@ -104,8 +106,8 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
SConnObj connObj = {.id = connId,
.appStartTimeMs = startTime,
.pid = pid,
.ip = pInfo->clientIp,
.port = pInfo->clientPort,
.ip = ip,
.port = port,
.killed = 0,
.loginTimeMs = taosGetTimestampMs(),
.lastAccessTimeMs = 0,
@ -114,17 +116,17 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
.pQueries = NULL};
connObj.lastAccessTimeMs = connObj.loginTimeMs;
tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN);
tstrncpy(connObj.user, user, TSDB_USER_LEN);
tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN);
int32_t keepTime = tsShellActivityTimer * 3;
SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000);
if (pConn == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("conn:%d, failed to put into cache since %s, user:%s", connId, pInfo->user, terrstr());
mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr());
return NULL;
} else {
mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, pInfo->user);
mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, user);
return pConn;
}
}
@ -184,20 +186,14 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
SConnObj *pConn = NULL;
int32_t code = -1;
SConnectReq connReq = {0};
char ip[30] = {0};
if (tDeserializeSConnectReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &connReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto CONN_OVER;
}
SRpcConnInfo info = {0};
if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) {
mError("user:%s, failed to login while get connection info since %s", pReq->user, terrstr());
goto CONN_OVER;
}
char ip[30];
taosIp2String(info.clientIp, ip);
taosIp2String(pReq->clientIp, ip);
pUser = mndAcquireUser(pMnode, pReq->user);
if (pUser == NULL) {
@ -216,7 +212,8 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
}
}
pConn = mndCreateConn(pMnode, &info, connReq.pid, connReq.app, connReq.startTime);
pConn =
mndCreateConn(pMnode, pReq->user, pReq->clientIp, pReq->clientPort, connReq.pid, connReq.app, connReq.startTime);
if (pConn == NULL) {
mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr());
goto CONN_OVER;
@ -241,7 +238,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
pReq->rspLen = contLen;
pReq->pRsp = pRsp;
mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app);
mDebug("user:%s, login from %s, conn:%d, app:%s", pReq->user, ip, pConn->id, connReq.app);
code = 0;

17
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -14,15 +14,15 @@
*/
#include "mndQuery.h"
#include "mndMnode.h"
#include "executor.h"
#include "mndMnode.h"
#include "qworker.h"
int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
mTrace("message in query queue is processing");
SMnode *pMnode = pReq->pNode;
SMnode *pMnode = pReq->pNode;
SReadHandle handle = {0};
mTrace("msg:%p, in query queue is processing", pReq);
switch (pReq->rpcMsg.msgType) {
case TDMT_VND_QUERY:
return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, &pReq->rpcMsg);
@ -35,9 +35,9 @@ int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
}
int32_t mndProcessFetchMsg(SNodeMsg *pReq) {
mTrace("message in fetch queue is processing");
SMnode *pMnode = pReq->pNode;
mTrace("msg:%p, in fetch queue is processing", pReq);
switch (pReq->rpcMsg.msgType) {
case TDMT_VND_FETCH:
return qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg);
@ -52,9 +52,9 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) {
}
int32_t mndInitQuery(SMnode *pMnode) {
int32_t code = qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb);
if (code) {
return code;
if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) {
mError("failed to init qworker in mnode since %s", terrstr());
return -1;
}
mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg);
@ -67,4 +67,3 @@ int32_t mndInitQuery(SMnode *pMnode) {
}
void mndCleanupQuery(SMnode *pMnode) { qWorkerDestroy((void **)&pMnode->pQuery); }

118
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -119,7 +119,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) {
return pVgroup;
}
int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId) {
int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SSdb* pSdb = pMnode->pSdb;
void* pIter = NULL;
SArray* tasks = taosArrayGetP(pStream->tasks, 0);
@ -151,9 +151,9 @@ int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream,
pTask->execType = TASK_EXEC__NONE;
// sink
if (smaId != -1) {
if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
pTask->sinkType = TASK_SINK__SMA;
pTask->smaSink.smaId = smaId;
pTask->smaSink.smaId = pStream->smaId;
} else {
pTask->sinkType = TASK_SINK__TABLE;
}
@ -166,7 +166,45 @@ int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream,
return 0;
}
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId) {
int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
ASSERT(pStream->fixedSinkVgId != 0);
SArray* tasks = taosArrayGetP(pStream->tasks, 0);
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
taosArrayPush(tasks, &pTask);
pTask->nodeId = pStream->fixedSinkVgId;
SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId);
if (pVgroup == NULL) {
return -1;
}
pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
// source
pTask->sourceType = TASK_SOURCE__MERGE;
// exec
pTask->execType = TASK_EXEC__NONE;
// sink
if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
pTask->sinkType = TASK_SINK__SMA;
pTask->smaSink.smaId = pStream->smaId;
} else {
pTask->sinkType = TASK_SINK__TABLE;
}
//
// dispatch
pTask->dispatchType = TASK_DISPATCH__NONE;
mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId);
return 0;
}
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SSdb* pSdb = pMnode->pSdb;
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
if (pPlan == NULL) {
@ -185,7 +223,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
taosArrayPush(pStream->tasks, &taskOneLevel);
// add extra sink
hasExtraSink = true;
mndAddSinkToStream(pMnode, pTrans, pStream, smaId);
if (pStream->fixedSinkVgId == 0) {
mndAddShuffledSinkToStream(pMnode, pTrans, pStream);
} else {
mndAddFixedSinkToStream(pMnode, pTrans, pStream);
}
}
for (int32_t level = 0; level < totLevel; level++) {
@ -221,12 +263,14 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
pTask->sinkType = TASK_SINK__SHOW;
pTask->showSink.reserved = 0;
if (!hasExtraSink) {
if (smaId != -1) {
#if 1
if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
pTask->sinkType = TASK_SINK__SMA;
pTask->smaSink.smaId = smaId;
pTask->smaSink.smaId = pStream->smaId;
} else {
pTask->sinkType = TASK_SINK__TABLE;
}
#endif
}
} else {
pTask->sinkType = TASK_SINK__NONE;
@ -286,35 +330,47 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
if (hasExtraSink) {
// add dispatcher
pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
if (pStream->fixedSinkVgId == 0) {
pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
ASSERT(pDb);
if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
ASSERT(pDb);
if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
sdbRelease(pSdb, pDb);
qDestroyQueryPlan(pPlan);
return -1;
}
sdbRelease(pSdb, pDb);
qDestroyQueryPlan(pPlan);
return -1;
}
sdbRelease(pSdb, pDb);
// put taskId to useDbRsp
// TODO: optimize
SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t sz = taosArrayGetSize(pVgs);
SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
int32_t sinkLvSize = taosArrayGetSize(sinkLv);
for (int32_t i = 0; i < sz; i++) {
SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
for (int32_t j = 0; j < sinkLvSize; j++) {
SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
/*printf("vgid %d node id %d\n", pVgInfo->vgId, pTask->nodeId);*/
if (pLastLevelTask->nodeId == pVgInfo->vgId) {
pVgInfo->taskId = pLastLevelTask->taskId;
/*printf("taskid %d set to %d\n", pVgInfo->taskId, pTask->taskId);*/
break;
// put taskId to useDbRsp
// TODO: optimize
SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t sz = taosArrayGetSize(pVgs);
SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
int32_t sinkLvSize = taosArrayGetSize(sinkLv);
for (int32_t i = 0; i < sz; i++) {
SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
for (int32_t j = 0; j < sinkLvSize; j++) {
SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
/*printf("vgid %d node id %d\n", pVgInfo->vgId, pTask->nodeId);*/
if (pLastLevelTask->nodeId == pVgInfo->vgId) {
pVgInfo->taskId = pLastLevelTask->taskId;
/*printf("taskid %d set to %d\n", pVgInfo->taskId, pTask->taskId);*/
break;
}
}
}
} else {
pTask->dispatchType = TASK_DISPATCH__FIXED;
pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
SArray* pArray = taosArrayGetP(pStream->tasks, 0);
// one sink only
ASSERT(taosArrayGetSize(pArray) == 1);
SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
}
}
#endif

6
source/dnode/mnode/impl/src/mndSma.c
View File

@ -415,6 +415,10 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
streamObj.dbUid = pDb->uid;
streamObj.version = 1;
streamObj.sql = pCreate->sql;
streamObj.createdBy = STREAM_CREATED_BY__SMA;
// TODO
streamObj.fixedSinkVgId = 0;
streamObj.smaId = smaObj.uid;
/*streamObj.physicalPlan = "";*/
streamObj.logicalPlan = "not implemented";
@ -428,7 +432,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER;
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, smaObj.uid) != 0) goto _OVER;
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;

10
source/dnode/mnode/impl/src/mndStream.c
View File

@ -246,7 +246,7 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
return code;
}
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId) {
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) {
SNode *pAst = NULL;
if (nodesStringToNode(ast, &pAst) < 0) {
@ -271,7 +271,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
return -1;
}
if (mndScheduleStream(pMnode, pTrans, pStream, smaId) < 0) {
if (mndScheduleStream(pMnode, pTrans, pStream) < 0) {
mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr());
return -1;
}
@ -300,6 +300,10 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
streamObj.dbUid = pDb->uid;
streamObj.version = 1;
streamObj.sql = pCreate->sql;
streamObj.createdBy = STREAM_CREATED_BY__USER;
// TODO
streamObj.fixedSinkVgId = 0;
streamObj.smaId = 0;
/*streamObj.physicalPlan = "";*/
streamObj.logicalPlan = "not implemented";
@ -310,7 +314,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
}
mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, -1) != 0) {
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) {
mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;

296
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -62,8 +62,6 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub);
int32_t mndInitSubscribe(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
.keyType = SDB_KEY_BINARY,
@ -98,14 +96,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
return NULL;
}
#if 0
if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tDeleteSMqSubscribeObj(pSub);
taosMemoryFree(pSub);
return NULL;
}
#endif
// TODO: disable alter subscribed table
return pSub;
}
@ -210,45 +200,6 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
return 0;
}
#if 0
static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode;
uint8_t *str = pMsg->rpcMsg.pCont;
SMqCMResetOffsetReq req;
SCoder decoder;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, str, pMsg->rpcMsg.contLen, TD_DECODER);
tDecodeSMqCMResetOffsetReq(&decoder, &req);
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (pHash == NULL) {
return -1;
}
for (int32_t i = 0; i < req.num; i++) {
SMqOffset *pOffset = &req.offsets[i];
SMqVgOffsets *pVgOffset = taosHashGet(pHash, &pOffset->vgId, sizeof(int32_t));
if (pVgOffset == NULL) {
pVgOffset = taosMemoryMalloc(sizeof(SMqVgOffsets));
if (pVgOffset == NULL) {
return -1;
}
pVgOffset->offsets = taosArrayInit(0, sizeof(void *));
taosArrayPush(pVgOffset->offsets, &pOffset);
}
taosHashPut(pHash, &pOffset->vgId, sizeof(int32_t), &pVgOffset, sizeof(void *));
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) {
mError("mq-reset-offset: failed since %s", terrstr());
return -1;
}
return 0;
}
#endif
static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode;
SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
@ -574,251 +525,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
return 0;
}
#if 0
for (int32_t j = 0; j < consumerNum; j++) {
bool changed = false;
bool unfished = false;
bool canUseLeft = imbalanceSolved < imbalanceVg;
bool mustUseLeft = canUseLeft && (imbalanceVg - imbalanceSolved >= consumerNum - j);
ASSERT(imbalanceVg - imbalanceSolved <= consumerNum - j);
int32_t maxVg = vgEachConsumer + canUseLeft;
int32_t minVg = vgEachConsumer + mustUseLeft;
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
int32_t vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
int32_t vgThisConsumerAfterRb;
if (vgThisConsumerBeforeRb > maxVg) {
vgThisConsumerAfterRb = maxVg;
imbalanceSolved++;
changed = true;
} else if (vgThisConsumerBeforeRb < minVg) {
vgThisConsumerAfterRb = minVg;
if (mustUseLeft) imbalanceSolved++;
changed = true;
} else {
vgThisConsumerAfterRb = vgThisConsumerBeforeRb;
}
if (vgThisConsumerBeforeRb > vgThisConsumerAfterRb) {
while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) {
// put into unassigned
SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo);
ASSERT(pConsumerEp != NULL);
ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId);
taosArrayPush(unassignedVgStash, pConsumerEp);
}
} else if (vgThisConsumerBeforeRb < vgThisConsumerAfterRb) {
// assign from unassigned
while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) {
// if no unassgined, save j
if (taosArrayGetSize(unassignedVgStash) == 0) {
taosArrayPush(unassignedConsumerIdx, &j);
unfished = true;
break;
}
// assign vg to consumer
SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
ASSERT(pConsumerEp != NULL);
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = pSubConsumer->consumerId;
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
// build msg and persist into trans
}
}
if (changed && !unfished) {
SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
pRebConsumer->epoch++;
if (vgThisConsumerAfterRb != 0) {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
} else {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE);
}
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
mndReleaseConsumer(pMnode, pRebConsumer);
// TODO: save history
}
}
for (int32_t j = 0; j < taosArrayGetSize(unassignedConsumerIdx); j++) {
bool canUseLeft = imbalanceSolved < imbalanceVg;
int32_t consumerIdx = *(int32_t *)taosArrayGet(unassignedConsumerIdx, j);
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, consumerIdx);
if (canUseLeft) imbalanceSolved++;
// must use
int32_t vgThisConsumerAfterRb = taosArrayGetSize(pSubConsumer->vgInfo) + canUseLeft;
while (taosArrayGetSize(pSubConsumer->vgInfo) < vgEachConsumer + canUseLeft) {
// assign vg to consumer
SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
ASSERT(pConsumerEp != NULL);
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = pSubConsumer->consumerId;
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
// build msg and persist into trans
}
SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
pRebConsumer->epoch++;
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
mndReleaseConsumer(pMnode, pRebConsumer);
// TODO: save history
}
#endif
#if 0
//update consumer status for the subscribption
for (int32_t i = 0; i < taosArrayGetSize(pSub->assigned); i++) {
SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i);
int64_t consumerId = pCEp->consumerId;
if (pCEp->status != -1) {
int32_t consumerHbStatus = atomic_fetch_add_32(&pCEp->consumerHbStatus, 1);
if (consumerHbStatus < MND_SUBSCRIBE_REBALANCE_CNT) {
continue;
}
// put consumer into lostConsumer
SMqConsumerEp* lostConsumer = taosArrayPush(pSub->lostConsumer, pCEp);
lostConsumer->qmsg = NULL;
// put vg into unassigned
taosArrayPush(pSub->unassignedVg, pCEp);
// remove from assigned
// TODO: swap with last one, reduce size and reset i
taosArrayRemove(pSub->assigned, i);
// remove from available consumer
for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) {
if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) {
taosArrayRemove(pSub->availConsumer, j);
break;
}
// TODO: acquire consumer, set status to unavail
}
#if 0
SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId);
pConsumer->epoch++;
printf("current epoch %ld size %ld", pConsumer->epoch, pConsumer->topics->size);
SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
sdbWriteNotFree(pMnode->pSdb, pRaw);
mndReleaseConsumer(pMnode, pConsumer);
#endif
}
}
// no available consumer, skip rebalance
if (taosArrayGetSize(pSub->availConsumer) == 0) {
continue;
}
taosArrayGet(pSub->availConsumer, 0);
// rebalance condition1 : have unassigned vg
// assign vg to a consumer, trying to find the least assigned one
if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) {
char *topic = NULL;
char *cgroup = NULL;
mndSplitSubscribeKey(pSub->key, &topic, &cgroup);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
for (int32_t i = 0; i < sz; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx);
pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer);
SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg);
pCEp->oldConsumerId = pCEp->consumerId;
pCEp->consumerId = consumerId;
taosArrayPush(pSub->assigned, pCEp);
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
pConsumer->epoch++;
SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
sdbWrite(pMnode->pSdb, pConsumerRaw);
mndReleaseConsumer(pMnode, pConsumer);
void* msg;
int32_t msgLen;
mndBuildRebalanceMsg(&msg, &msgLen, pTopic, pCEp, cgroup, topic);
// persist msg
STransAction action = {0};
action.epSet = pCEp->epSet;
action.pCont = msg;
action.contLen = sizeof(SMsgHead) + msgLen;
action.msgType = TDMT_VND_MQ_SET_CONN;
mndTransAppendRedoAction(pTrans, &action);
// persist data
SSdbRaw *pRaw = mndSubActionEncode(pSub);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pRaw);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
}
mndReleaseTopic(pMnode, pTopic);
mndTransDrop(pTrans);
taosMemoryFreeClear(topic);
taosMemoryFreeClear(cgroup);
}
// rebalance condition2 : imbalance assignment
}
return 0;
}
#endif
#if 0
static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
SQueryPlan *pPlan = qStringToQueryPlan(pTopic->physicalPlan);
SArray *pArray = NULL;
SNodeListNode *inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
SSubplan *plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
SArray *unassignedVg = pSub->unassignedVg;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pTopic->dbUid) {
sdbRelease(pSdb, pVgroup);
continue;
}
pSub->vgNum++;
plan->execNode.nodeId = pVgroup->vgId;
plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
if (schedulerConvertDagToTaskList(pPlan, &pArray) < 0) {
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql);
return -1;
}
SMqConsumerEp consumerEp = {0};
consumerEp.status = 0;
consumerEp.consumerId = -1;
STaskInfo *pTaskInfo = taosArrayGet(pArray, 0);
consumerEp.epSet = pTaskInfo->addr.epset;
consumerEp.vgId = pTaskInfo->addr.nodeId;
ASSERT(consumerEp.vgId == pVgroup->vgId);
consumerEp.qmsg = strdup(pTaskInfo->msg->msg);
taosArrayPush(unassignedVg, &consumerEp);
// TODO: free taskInfo
taosArrayDestroy(pArray);
}
/*qDestroyQueryDag(pDag);*/
return 0;
}
#endif
static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
const SMqConsumerEp *pConsumerEp) {
ASSERT(pConsumerEp->oldConsumerId == -1);
@ -1140,7 +846,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
}
}
if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void*))taosMemoryFree);
if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);
// persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);

11
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -65,9 +65,11 @@ void mndCleanupTopic(SMnode *pMnode) {}
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
int32_t size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE;
int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
int32_t size =
sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + swLen + MND_TOPIC_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
@ -86,8 +88,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
void *swBuf = taosMemoryMalloc(swLen);
void *swBuf = taosMemoryMalloc(swLen);
if (swBuf == NULL) {
goto TOPIC_ENCODE_OVER;
}

8
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -54,7 +54,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
static void mndTransExecute(SMnode *pMnode, STrans *pTrans);
static void mndTransSendRpcRsp(STrans *pTrans);
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransReq(SNodeMsg *pReq);
static int32_t mndProcessKillTransReq(SNodeMsg *pReq);
@ -737,7 +737,7 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
return 0;
}
static void mndTransSendRpcRsp(STrans *pTrans) {
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
bool sendRsp = false;
if (pTrans->stage == TRN_STAGE_FINISHED) {
@ -771,7 +771,7 @@ static void mndTransSendRpcRsp(STrans *pTrans) {
.ahandle = pTrans->rpcAHandle,
.pCont = rpcCont,
.contLen = pTrans->rpcRspLen};
rpcSendResponse(&rspMsg);
tmsgSendRsp(&pMnode->msgCb, &rspMsg);
pTrans->rpcHandle = NULL;
pTrans->rpcRsp = NULL;
pTrans->rpcRspLen = 0;
@ -1158,7 +1158,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
}
}
mndTransSendRpcRsp(pTrans);
mndTransSendRpcRsp(pMnode, pTrans);
}
static int32_t mndProcessTransReq(SNodeMsg *pReq) {

2
source/dnode/vnode/src/tq/tq.c
View File

@ -275,6 +275,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pMsg->pCont = NULL;
pMsg->contLen = 0;
pMsg->code = -1;
ASSERT(0);
rpcSendResponse(pMsg);
return 0;
}
@ -356,6 +357,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
ASSERT(0);
return -1;
}
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;

61
source/dnode/vnode/src/tq/tqRead.c
View File

@ -114,10 +114,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
int32_t numOfRows = pHandle->pBlock->numOfRows;
int32_t numOfCols = pHandle->pSchema->numOfCols;
/*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
// TODO: stable case
if (colNumNeed > pSchemaWrapper->nCols) {
colNumNeed = pSchemaWrapper->nCols;
}
@ -138,56 +137,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
colNeed++;
} else {
SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes;
/*int sz = numOfRows * pColSchema->bytes;*/
colInfo.info.bytes = pColSchema->bytes;
colInfo.info.colId = pColSchema->colId;
colInfo.info.type = pColSchema->type;
colInfo.pData = taosMemoryCalloc(1, sz);
if (colInfo.pData == NULL) {
// TODO free
taosArrayDestroy(pArray);
if (blockDataEnsureColumnCapacity(&colInfo, numOfRows) < 0) {
taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
return NULL;
}
blockDataEnsureColumnCapacity(&colInfo, numOfRows);
taosArrayPush(pArray, &colInfo);
colMeta++;
colNeed++;
}
}
int j = 0;
for (int32_t i = 0; i < colNumNeed; i++) {
col_id_t colId = *(col_id_t*)taosArrayGet(pHandle->pColIdList, i);
while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
j++;
}
if (j >= pSchemaWrapper->nCols) {
continue;
}
SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes;
colInfo.info.bytes = pColSchema->bytes;
colInfo.info.colId = colId;
colInfo.info.type = pColSchema->type;
colInfo.pData = taosMemoryCalloc(1, sz);
if (colInfo.pData == NULL) {
// TODO free
taosArrayDestroy(pArray);
return NULL;
}
blockDataEnsureColumnCapacity(&colInfo, numOfRows);
taosArrayPush(pArray, &colInfo);
}
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
// int32_t kvIdx = 0;
int32_t curRow = 0;
tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter);
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
@ -200,25 +167,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break;
}
memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes);
// TODO handle null
colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL);
}
#if 0
for (int32_t i = 0; i < colNumNeed; i++) {
SColumnInfoData* pColData = taosArrayGet(pArray, i);
STColumn* pCol = schemaColAt(pTschema, i);
// TODO
if(pCol->colId != pColData->info.colId) {
continue;
}
// void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
SCellVal sVal = {0};
if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) {
// TODO: reach end
break;
}
memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes);
}
#endif
curRow++;
}
return pArray;

13
source/libs/executor/src/executor.c
View File

@ -14,12 +14,12 @@
*/
#include "executor.h"
#include "tdatablock.h"
#include "executorimpl.h"
#include "planner.h"
#include "tdatablock.h"
#include "vnode.h"
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void** input, size_t numOfBlocks, int32_t type, char* id) {
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
ASSERT(pOperator != NULL);
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
if (pOperator->numOfDownstream == 0) {
@ -44,17 +44,18 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void** input, size_t n
}
if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) {
if (tqReadHandleSetMsg(pInfo->readerHandle, input[0], 0) < 0) {
if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) {
qError("submit msg messed up when initing stream block, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
} else {
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pDataBlock = input[i];
SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
SSDataBlock* p = createOneDataBlock(pDataBlock);
p->info = pDataBlock->info;
taosArrayClear(p->pDataBlock);
taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
taosArrayPush(pInfo->pBlockLists, &p);
}
@ -65,10 +66,10 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void** input, size_t n
}
int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type) {
qSetMultiStreamInput(tinfo, (void**) &input, 1, type);
return qSetMultiStreamInput(tinfo, input, 1, type);
}
int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, void** pBlocks, size_t numOfBlocks, int32_t type) {
int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) {
if (tinfo == NULL) {
return TSDB_CODE_QRY_APP_ERROR;
}

4088
source/libs/executor/src/executorimpl.c
View File

File diff suppressed because it is too large Load Diff

14
source/libs/index/inc/index_fst.h
View File

@ -21,9 +21,9 @@ extern "C" {
#endif
#include "indexInt.h"
#include "index_fst_node.h"
#include "index_fst_automation.h"
#include "index_fst_counting_writer.h"
#include "index_fst_node.h"
#include "index_fst_registry.h"
#include "index_fst_util.h"
@ -257,9 +257,9 @@ typedef struct FstMeta {
} FstMeta;
typedef struct Fst {
FstMeta* meta;
FstSlice* data; //
FstNode* root; //
FstMeta* meta;
FstSlice* data; //
FstNode* root; //
TdThreadMutex mtx;
} Fst;
@ -325,10 +325,10 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut);
void fstStreamBuilderDestroy(FstStreamBuilder* b);
// set up bound range
// refator, simple code by marco
FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
// set up bound range
// refator later: to simple code by marco
void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type);
#ifdef __cplusplus
}

20
source/libs/index/src/index_fst.c
View File

@ -1184,7 +1184,7 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
sws->aut = automation;
sws->inp = (SArray*)taosArrayInit(256, sizeof(uint8_t));
sws->emptyOutput.null = false;
sws->emptyOutput.null = true;
sws->emptyOutput.out = 0;
sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
@ -1239,8 +1239,8 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
for (uint32_t i = 0; i < len; i++) {
uint8_t b = data[i];
uint64_t res = 0;
bool null = fstNodeFindInput(node, b, &res);
if (null == false) {
bool find = fstNodeFindInput(node, b, &res);
if (find == true) {
FstTransition trn;
fstNodeGetTransitionAt(node, res, &trn);
void* preState = autState;
@ -1293,7 +1293,6 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
return false;
}
StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallback callback) {
AutomationCtx* aut = sws->aut;
FstOutput output = sws->emptyOutput;
@ -1356,6 +1355,7 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
taosMemoryFreeClear(buf);
fstSliceDestroy(&slice);
taosArrayDestroy(nodes);
return NULL;
}
if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
@ -1364,15 +1364,12 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
taosMemoryFreeClear(buf);
fstSliceDestroy(&slice);
taosArrayDestroy(nodes);
nodes = NULL;
return result;
}
taosMemoryFreeClear(buf);
fstSliceDestroy(&slice);
}
for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
FstNode** node = (FstNode**)taosArrayGet(nodes, i);
fstNodeDestroy(*node);
}
};
taosArrayDestroy(nodes);
return NULL;
}
@ -1425,9 +1422,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) {
taosMemoryFreeClear(b->max);
taosMemoryFree(b);
}
FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
void fstStreamBuilderSetRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
if (b == NULL) {
return NULL;
return;
}
if (type == GE) {
b->min->type = Included;
@ -1446,5 +1443,4 @@ FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, Rang
fstSliceDestroy(&(b->max->data));
b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1);
}
return b;
}

22
source/libs/index/src/index_fst_automation.c
View File

@ -85,10 +85,20 @@ static void* prefixStart(AutomationCtx* ctx) {
};
static bool prefixIsMatch(AutomationCtx* ctx, void* sv) {
StartWithStateValue* ssv = (StartWithStateValue*)sv;
return ssv->val == strlen(ctx->data);
if (ssv == NULL) {
return false;
}
if (ssv->type == FST_INT) {
return ssv->val == strlen(ctx->data);
} else {
return false;
}
}
static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
StartWithStateValue* ssv = (StartWithStateValue*)sv;
if (ssv == NULL) {
return false;
}
return ssv->val >= 0;
}
static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
@ -154,15 +164,7 @@ AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
// add more search type
}
char* dst = NULL;
if (data != NULL) {
char* src = (char*)data;
size_t len = strlen(src);
dst = (char*)taosMemoryCalloc(1, len * sizeof(char) + 1);
memcpy(dst, src, len);
}
ctx->data = dst;
ctx->data = (data != NULL ? strdup((char*)data) : NULL);
ctx->type = atype;
ctx->stdata = (void*)sv;
return ctx;

261
source/libs/index/test/fstTest.cc
View File

@ -96,11 +96,41 @@ class FstReadMemory {
char* ch = (char*)fstSliceData(s, &sz);
std::string key(ch, sz);
printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
result.push_back(rt->out.out);
swsResultDestroy(rt);
}
for (size_t i = 0; i < result.size(); i++) {
streamWithStateDestroy(st);
fstStreamBuilderDestroy(sb);
return true;
}
bool SearchRange(AutomationCtx* ctx, const std::string& low, RangeType lowType, const std::string& high,
RangeType highType, std::vector<uint64_t>& result) {
FstStreamBuilder* sb = fstSearch(_fst, ctx);
FstSlice l = fstSliceCreate((uint8_t*)low.c_str(), low.size());
FstSlice h = fstSliceCreate((uint8_t*)high.c_str(), high.size());
// range [low, high);
fstStreamBuilderSetRange(sb, &l, lowType);
fstStreamBuilderSetRange(sb, &h, highType);
fstSliceDestroy(&l);
fstSliceDestroy(&h);
StreamWithState* st = streamBuilderIntoStream(sb);
StreamWithStateResult* rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
// result.push_back((uint64_t)(rt->out.out));
FstSlice* s = &rt->data;
int32_t sz = 0;
char* ch = (char*)fstSliceData(s, &sz);
std::string key(ch, sz);
printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
result.push_back(rt->out.out);
swsResultDestroy(rt);
}
std::cout << std::endl;
streamWithStateDestroy(st);
fstStreamBuilderDestroy(sb);
return true;
}
bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
@ -233,7 +263,7 @@ void checkFstLongTerm() {
// taosMemoryFree(ctx);
// delete m;
}
void checkFstCheckIterator() {
void checkFstCheckIterator1() {
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
@ -243,8 +273,7 @@ void checkFstCheckIterator() {
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
fw->Put("Hello world", 1);
fw->Put("hello world", 2);
fw->Put("hello worle", 3);
fw->Put("Hello worle", 2);
fw->Put("hello worlf", 4);
delete fw;
@ -258,15 +287,227 @@ void checkFstCheckIterator() {
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"H", AUTOMATION_PREFIX);
AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
m->Search(ctx, result);
std::cout << "size: " << result.size() << std::endl;
// assert(result.size() == count);
for (int i = 0; i < result.size(); i++) {
// assert(result[i] == i); // check result
}
automCtxDestroy(ctx);
taosMemoryFree(ctx);
delete m;
}
void checkFstCheckIterator2() {
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
// Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
fw->Put("a", 1);
fw->Put("b", 2);
fw->Put("c", 4);
delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"He", AUTOMATION_ALWAYS);
m->Search(ctx, result);
std::cout << "size: " << result.size() << std::endl;
// assert(result.size() == count);
for (int i = 0; i < result.size(); i++) {
// assert(result[i] == i); // check result
}
automCtxDestroy(ctx);
delete m;
}
void checkFstCheckIteratorPrefix() {
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
// Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
fw->Put("Hello world", 1);
fw->Put("Hello worle", 2);
fw->Put("hello worlf", 4);
fw->Put("ja", 4);
fw->Put("jb", 4);
fw->Put("jc", 4);
fw->Put("jddddddddd", 4);
fw->Put("jefffffff", 4);
delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
{
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == 1);
automCtxDestroy(ctx);
}
{
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"Hello", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == 2);
automCtxDestroy(ctx);
}
{
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"jddd", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == 1);
automCtxDestroy(ctx);
}
delete m;
}
void checkFstCheckIteratorRange1() {
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
// Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
fw->Put("a", 1);
fw->Put("b", 2);
fw->Put("c", 3);
fw->Put("d", 4);
fw->Put("e", 5);
fw->Put("f", 5);
fw->Put("G", 5);
delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
{
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GE, "e", LT, result);
assert(result.size() == 3);
automCtxDestroy(ctx);
}
{
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GT, "e", LT, result);
assert(result.size() == 2);
automCtxDestroy(ctx);
}
{
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GT, "e", LE, result);
assert(result.size() == 3);
automCtxDestroy(ctx);
}
{
// prefix search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GE, "e", LE, result);
assert(result.size() == 4);
automCtxDestroy(ctx);
}
delete m;
}
void checkFstCheckIteratorRange2() {
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
// Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
fw->Put("ab", 1);
fw->Put("b", 2);
fw->Put("cdd", 3);
fw->Put("cde", 3);
fw->Put("ddd", 4);
fw->Put("ed", 5);
delete fw;
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
{
// range search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GE, "ed", LT, result);
assert(result.size() == 4);
automCtxDestroy(ctx);
}
{
// range search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GE, "ed", LE, result);
assert(result.size() == 5);
automCtxDestroy(ctx);
// taosMemoryFree(ctx);
}
{
// range search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GT, "ed", LE, result);
assert(result.size() == 4);
automCtxDestroy(ctx);
}
{
// range search
std::vector<uint64_t> result;
AutomationCtx* ctx = automCtxCreate((void*)"he", AUTOMATION_ALWAYS);
// [b, e)
m->SearchRange(ctx, "b", GT, "ed", LT, result);
assert(result.size() == 3);
automCtxDestroy(ctx);
}
delete m;
}
@ -332,7 +573,11 @@ int main(int argc, char* argv[]) {
// path suid colName ver
// iterTFileReader(argv[1], argv[2], argv[3], argv[4]);
//}
checkFstCheckIterator();
checkFstCheckIterator1();
checkFstCheckIterator2();
checkFstCheckIteratorPrefix();
checkFstCheckIteratorRange1();
checkFstCheckIteratorRange2();
// checkFstLongTerm();
// checkFstPrefixSearch();

2
source/libs/index/test/fstUT.cc
View File

@ -13,9 +13,9 @@
#include "index_fst_util.h"
#include "index_tfile.h"
#include "tglobal.h"
#include "tlog.h"
#include "tskiplist.h"
#include "tutil.h"
#include "tlog.h"
static std::string dir = "/tmp/index";

6
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -260,9 +260,9 @@ static SNode* logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
}
static SNode* logicProjectCopy(const SProjectLogicNode* pSrc, SProjectLogicNode* pDst) {
COPY_ALL_SCALAR_FIELDS;
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pProjections);
COPY_CHAR_ARRAY_FIELD(stmtName);
return (SNode*)pDst;
}
@ -307,10 +307,8 @@ static SNode* logicSubplanCopy(const SLogicSubplan* pSrc, SLogicSubplan* pDst) {
}
static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) {
COPY_SCALAR_FIELD(dataBlockId);
COPY_ALL_SCALAR_FIELDS;
CLONE_NODE_LIST_FIELD(pSlots);
COPY_SCALAR_FIELD(resultRowSize);
COPY_SCALAR_FIELD(precision);
return (SNode*)pDst;
}

78
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -475,6 +475,10 @@ static int32_t jsonToLogicScanNode(const SJson* pJson, void* pObj) {
}
static const char* jkProjectLogicPlanProjections = "Projections";
static const char* jkProjectLogicPlanLimit = "Limit";
static const char* jkProjectLogicPlanOffset = "Offset";
static const char* jkProjectLogicPlanSlimit = "SLimit";
static const char* jkProjectLogicPlanSoffset = "SOffset";
static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
@ -483,6 +487,18 @@ static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkProjectLogicPlanProjections, pNode->pProjections);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanLimit, pNode->limit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanOffset, pNode->offset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectLogicPlanSlimit, pNode->slimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkScanLogicPlanTableMetaSize, pNode->soffset);
}
return code;
}
@ -494,6 +510,18 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkProjectLogicPlanProjections, &pNode->pProjections);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanLimit, &pNode->limit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanOffset, &pNode->offset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkProjectLogicPlanSlimit, &pNode->slimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkScanLogicPlanTableMetaSize, &pNode->soffset);
}
return code;
}
@ -802,6 +830,10 @@ static int32_t jsonToPhysiSysTableScanNode(const SJson* pJson, void* pObj) {
}
static const char* jkProjectPhysiPlanProjections = "Projections";
static const char* jkProjectPhysiPlanLimit = "Limit";
static const char* jkProjectPhysiPlanOffset = "Offset";
static const char* jkProjectPhysiPlanSlimit = "SLimit";
static const char* jkProjectPhysiPlanSoffset = "SOffset";
static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj;
@ -810,6 +842,18 @@ static int32_t physiProjectNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkProjectPhysiPlanProjections, pNode->pProjections);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanLimit, pNode->limit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanOffset, pNode->offset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanSlimit, pNode->slimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkProjectPhysiPlanSoffset, pNode->soffset);
}
return code;
}
@ -821,6 +865,18 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkProjectPhysiPlanProjections, &pNode->pProjections);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanLimit, &pNode->limit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanOffset, &pNode->offset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanSlimit, &pNode->slimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkProjectPhysiPlanSoffset, &pNode->soffset);
}
return code;
}
@ -2057,19 +2113,26 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
static const char* jkDataBlockDescDataBlockId = "DataBlockId";
static const char* jkDataBlockDescSlots = "Slots";
static const char* jkDataBlockResultRowSize = "ResultRowSize";
static const char* jkDataBlockTotalRowSize = "TotalRowSize";
static const char* jkDataBlockOutputRowSize = "OutputRowSize";
static const char* jkDataBlockPrecision = "Precision";
static int32_t dataBlockDescNodeToJson(const void* pObj, SJson* pJson) {
const SDataBlockDescNode* pNode = (const SDataBlockDescNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkDataBlockDescDataBlockId, pNode->dataBlockId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDataBlockResultRowSize, pNode->resultRowSize);
code = tjsonAddIntegerToObject(pJson, jkDataBlockTotalRowSize, pNode->totalRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDataBlockOutputRowSize, pNode->outputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkDataBlockDescSlots, pNode->pSlots);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDataBlockPrecision, pNode->precision);
}
return code;
}
@ -2079,12 +2142,17 @@ static int32_t jsonToDataBlockDescNode(const SJson* pJson, void* pObj) {
int32_t code = tjsonGetSmallIntValue(pJson, jkDataBlockDescDataBlockId, &pNode->dataBlockId);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkDataBlockResultRowSize, &pNode->resultRowSize);
code = tjsonGetIntValue(pJson, jkDataBlockTotalRowSize, &pNode->totalRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkDataBlockOutputRowSize, &pNode->outputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkDataBlockDescSlots, &pNode->pSlots);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetSmallIntValue(pJson, jkDataBlockPrecision, &pNode->precision);
}
return code;
}

4
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -582,11 +582,11 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
nodesWalkSelectStmt(pSelect, clause, collectColumns, &cxt);
taosHashCleanup(cxt.pColIdHash);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pCols);
nodesClearList(cxt.pCols);
return cxt.errCode;
}
if (0 == LIST_LENGTH(cxt.pCols)) {
nodesDestroyList(cxt.pCols);
nodesClearList(cxt.pCols);
cxt.pCols = NULL;
}
*pCols = cxt.pCols;

2
source/libs/parser/inc/sql.y
View File

@ -349,6 +349,7 @@ literal(A) ::= NK_STRING(B).
literal(A) ::= NK_BOOL(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B)); }
literal(A) ::= TIMESTAMP(B) NK_STRING(C). { A = createRawExprNodeExt(pCxt, &B, &C, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &C)); }
literal(A) ::= duration_literal(B). { A = B; }
literal(A) ::= NULL(B). { A = createRawExprNode(pCxt, &B, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL)); }
duration_literal(A) ::= NK_VARIABLE(B). { A = createRawExprNode(pCxt, &B, createDurationValueNode(pCxt, &B)); }
@ -372,6 +373,7 @@ signed_literal(A) ::= NK_STRING(B).
signed_literal(A) ::= NK_BOOL(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); }
signed_literal(A) ::= TIMESTAMP NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); }
signed_literal(A) ::= duration_literal(B). { A = releaseRawExprNode(pCxt, B); }
signed_literal(A) ::= NULL. { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); }
%type literal_list { SNodeList* }
%destructor literal_list { nodesDestroyList($$); }

16
source/libs/parser/src/parAstCreater.c
View File

@ -577,11 +577,13 @@ SNode* createColumnNode(SAstCreateContext* pCxt, SToken* pTableAlias, SToken* pC
SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) {
SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
CHECK_OUT_OF_MEM(val);
val->literal = strndup(pLiteral->z, pLiteral->n);
if (TK_NK_ID != pLiteral->type && (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) {
trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n);
if (NULL != pLiteral) {
val->literal = strndup(pLiteral->z, pLiteral->n);
if (TK_NK_ID != pLiteral->type && (IS_VAR_DATA_TYPE(dataType) || TSDB_DATA_TYPE_TIMESTAMP == dataType)) {
trimString(pLiteral->z, pLiteral->n, val->literal, pLiteral->n);
}
CHECK_OUT_OF_MEM(val->literal);
}
CHECK_OUT_OF_MEM(val->literal);
val->node.resType.type = dataType;
val->node.resType.bytes = IS_VAR_DATA_TYPE(dataType) ? strlen(val->literal) : tDataTypes[dataType].bytes;
if (TSDB_DATA_TYPE_TIMESTAMP == dataType) {
@ -717,8 +719,10 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
CHECK_OUT_OF_MEM(limitNode);
// limitNode->limit = limit;
// limitNode->offset = offset;
limitNode->limit = strtol(pLimit->z, NULL, 10);
if (NULL != pOffset) {
limitNode->offset = strtol(pOffset->z, NULL, 10);
}
return (SNode*)limitNode;
}

1
source/libs/parser/src/parTokenizer.c
View File

@ -99,6 +99,7 @@ static SKeyword keywordTable[] = {
{"ON", TK_ON},
{"OR", TK_OR},
{"ORDER", TK_ORDER},
{"PARTITION", TK_PARTITION},
{"PASS", TK_PASS},
{"PORT", TK_PORT},
{"PRECISION", TK_PRECISION},

14
source/libs/parser/src/parTranslater.c
View File

@ -235,6 +235,8 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol
nodesListAppend(pExpr->pAssociationList, (SNode*)pCol);
if (NULL != pTable) {
strcpy(pCol->tableAlias, pTable->tableAlias);
} else if (QUERY_NODE_COLUMN == nodeType(pExpr)) {
strcpy(pCol->tableAlias, ((SColumnNode*)pExpr)->tableAlias);
}
strcpy(pCol->colName, pExpr->aliasName);
pCol->node.resType = pExpr->resType;
@ -617,7 +619,6 @@ static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) {
}
static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) {
// todo release
if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) {
return TSDB_CODE_SUCCESS;
}
@ -631,11 +632,6 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea
}
if (TSDB_CODE_SUCCESS == code) {
// todo remove
//if (NULL != vgroupList && taosArrayGetSize(vgroupList) > 0 && 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) {
// taosArrayPopTailBatch(vgroupList, taosArrayGetSize(vgroupList) - 1);
//}
code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList);
}
taosArrayDestroy(vgroupList);
@ -1488,12 +1484,6 @@ static int32_t nodeTypeToShowType(ENodeType nt) {
static int32_t translateShow(STranslateContext* pCxt, SShowStmt* pStmt) {
SShowReq showReq = { .type = nodeTypeToShowType(nodeType(pStmt)) };
// if ('\0' != pStmt->dbName[0]) {
// SName name = {0};
// tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
// char dbFname[TSDB_DB_FNAME_LEN] = {0};
// tNameGetFullDbName(&name, showReq.db);
// }
pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo));
if (NULL == pCxt->pCmdMsg) {

1699
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

5
source/libs/parser/test/parserAstTest.cpp
View File

@ -487,7 +487,10 @@ TEST_F(ParserTest, createTable) {
bind("create table "
"if not exists test.t1 using test.st1 (tag1, tag2) tags(1, 'abc') "
"if not exists test.t2 using test.st1 (tag1, tag2) tags(2, 'abc') "
"if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc')"
"if not exists test.t3 using test.st1 (tag1, tag2) tags(3, 'abc') "
"if not exists test.t4 using test.st1 (tag1, tag2) tags(3, null) "
"if not exists test.t5 using test.st1 (tag1, tag2) tags(null, 'abc') "
"if not exists test.t6 using test.st1 (tag1, tag2) tags(null, null)"
);
ASSERT_TRUE(run());

93
source/libs/planner/src/planLogicCreater.c
View File

@ -142,6 +142,46 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanCols, STab
return SCAN_TYPE_TAG;
}
static SNodeptr createPrimaryKeyCol(uint64_t tableId) {
SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL;
}
pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
pCol->tableId = tableId;
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
pCol->colType = COLUMN_TYPE_COLUMN;
strcpy(pCol->colName, "#primarykey");
return pCol;
}
static int32_t addPrimaryKeyCol(uint64_t tableId, SNodeList** pCols) {
if (NULL == *pCols) {
*pCols = nodesMakeList();
if (NULL == *pCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
bool found = false;
SNode* pCol = NULL;
FOREACH(pCol, *pCols) {
if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pCol)->colId) {
found = true;
break;
}
}
if (!found) {
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(*pCols, createPrimaryKeyCol(tableId))) {
nodesDestroyList(*pCols);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return TSDB_CODE_SUCCESS;
}
static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) {
SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN);
if (NULL == pScan) {
@ -161,7 +201,11 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
// set columns to scan
SNodeList* pCols = NULL;
int32_t code = nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols);
if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
if (TSDB_CODE_SUCCESS == code) {
code = addPrimaryKeyCol(pScan->pMeta->uid, &pCols);
}
if (TSDB_CODE_SUCCESS == code) {
pScan->pScanCols = nodesCloneList(pCols);
if (NULL == pScan) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -171,13 +215,15 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect
pScan->scanType = getScanType(pCxt, pCols, pScan->pMeta);
// set output
if (TSDB_CODE_SUCCESS == code && NULL != pCols) {
if (TSDB_CODE_SUCCESS == code) {
pScan->node.pTargets = nodesCloneList(pCols);
if (NULL == pScan) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
nodesClearList(pCols);
if (TSDB_CODE_SUCCESS == code) {
*pLogicNode = (SLogicNode*)pScan;
} else {
@ -548,6 +594,22 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
return TSDB_CODE_OUT_OF_MEMORY;
}
if (NULL != pSelect->pLimit) {
pProject->limit = ((SLimitNode*)pSelect->pLimit)->limit;
pProject->offset = ((SLimitNode*)pSelect->pLimit)->offset;
} else {
pProject->limit = -1;
pProject->offset = -1;
}
if (NULL != pSelect->pSlimit) {
pProject->slimit = ((SLimitNode*)pSelect->pSlimit)->limit;
pProject->soffset = ((SLimitNode*)pSelect->pSlimit)->offset;
} else {
pProject->slimit = -1;
pProject->soffset = -1;
}
int32_t code = TSDB_CODE_SUCCESS;
pProject->pProjections = nodesCloneList(pSelect->pProjectionList);
@ -569,15 +631,42 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel
return code;
}
static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (NULL == pSelect->pPartitionByList) {
return TSDB_CODE_SUCCESS;
}
SPartitionLogicNode* pPartition = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PARTITION);
if (NULL == pPartition) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
}
static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
if (!pSelect->isDistinct) {
return TSDB_CODE_SUCCESS;
}
return TSDB_CODE_SUCCESS;
}
static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) {
SLogicNode* pRoot = NULL;
int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot);
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createWindowLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createPartitionLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createAggLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createDistinctLogicNode, &pRoot);
}
if (TSDB_CODE_SUCCESS == code) {
code = createChildLogicNode(pCxt, pSelect, createSortLogicNode, &pRoot);
}

58
source/libs/planner/src/planPhysiCreater.c
View File

@ -131,7 +131,8 @@ static int32_t buildDataBlockSlots(SPhysiPlanContext* pCxt, SNodeList* pList, SD
code = putSlotToHash(pDataBlockDesc->dataBlockId, slotId, pNode, pHash);
}
if (TSDB_CODE_SUCCESS == code) {
pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes;
pDataBlockDesc->totalRowSize += ((SExprNode*)pNode)->resType.bytes;
pDataBlockDesc->outputRowSize += ((SExprNode*)pNode)->resType.bytes;
++slotId;
} else {
break;
@ -189,7 +190,10 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
if (TSDB_CODE_SUCCESS == code) {
code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
}
pDataBlockDesc->resultRowSize += ((SExprNode*)pExpr)->resType.bytes;
pDataBlockDesc->totalRowSize += ((SExprNode*)pExpr)->resType.bytes;
if (output) {
pDataBlockDesc->outputRowSize += ((SExprNode*)pExpr)->resType.bytes;
}
slotId = nextSlotId;
++nextSlotId;
} else {
@ -311,20 +315,6 @@ static int32_t setConditionsSlotId(SPhysiPlanContext* pCxt, const SLogicNode* pL
return TSDB_CODE_SUCCESS;
}
static SNodeptr createPrimaryKeyCol(SPhysiPlanContext* pCxt, uint64_t tableId) {
SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) {
return NULL;
}
pCol->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP;
pCol->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
pCol->tableId = tableId;
pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
pCol->colType = COLUMN_TYPE_COLUMN;
strcpy(pCol->colName, "#primarykey");
return pCol;
}
static int32_t colIdCompare(const void* pLeft, const void* pRight) {
SColumnNode* pLeftCol = *(SColumnNode**)pLeft;
SColumnNode* pRightCol = *(SColumnNode**)pRight;
@ -353,35 +343,10 @@ static int32_t sortScanCols(SNodeList* pScanCols) {
}
static int32_t createScanCols(SPhysiPlanContext* pCxt, SScanPhysiNode* pScanPhysiNode, SNodeList* pScanCols) {
if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanPhysiNode)
|| QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN == nodeType(pScanPhysiNode)) {
pScanPhysiNode->pScanCols = nodesMakeList();
if (NULL == pScanPhysiNode->pScanCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, createPrimaryKeyCol(pCxt, pScanPhysiNode->uid))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SNode* pNode;
FOREACH(pNode, pScanCols) {
if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) {
SColumnNode* pCol = nodesListGetNode(pScanPhysiNode->pScanCols, 0);
strcpy(pCol->tableAlias, ((SColumnNode*)pNode)->tableAlias);
strcpy(pCol->colName, ((SColumnNode*)pNode)->colName);
continue;
}
if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pScanPhysiNode->pScanCols, nodesCloneNode(pNode))) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}
} else {
pScanPhysiNode->pScanCols = nodesCloneList(pScanCols);
if (NULL == pScanPhysiNode->pScanCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pScanPhysiNode->pScanCols = nodesCloneList(pScanCols);
if (NULL == pScanPhysiNode->pScanCols) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return sortScanCols(pScanPhysiNode->pScanCols);
}
@ -708,6 +673,11 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild
return TSDB_CODE_OUT_OF_MEMORY;
}
pProject->limit = pProjectLogicNode->limit;
pProject->offset = pProjectLogicNode->offset;
pProject->slimit = pProjectLogicNode->slimit;
pProject->soffset = pProjectLogicNode->soffset;
int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, pProjectLogicNode->pProjections, &pProject->pProjections);
if (TSDB_CODE_SUCCESS == code) {
code = addDataBlockSlotsForProject(pCxt, pProjectLogicNode->stmtName, pProject->pProjections, pProject->node.pOutputDataBlockDesc);

28
source/libs/planner/test/plannerTest.cpp
View File

@ -207,7 +207,7 @@ TEST_F(PlannerTest, sessionWindow) {
TEST_F(PlannerTest, orderBy) {
setDatabase("root", "test");
bind("SELECT * FROM t1 order by c1");
bind("SELECT c1 FROM t1 order by c1");
ASSERT_TRUE(run());
bind("SELECT c1 FROM t1 order by c2");
@ -217,6 +217,32 @@ TEST_F(PlannerTest, orderBy) {
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, limit) {
setDatabase("root", "test");
bind("SELECT * FROM t1 limit 2");
ASSERT_TRUE(run());
bind("SELECT * FROM t1 limit 5 offset 2");
ASSERT_TRUE(run());
bind("SELECT * FROM t1 limit 2, 5");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, slimit) {
setDatabase("root", "test");
bind("SELECT * FROM t1 partition by c1 slimit 2");
ASSERT_TRUE(run());
bind("SELECT * FROM t1 partition by c1 slimit 5 soffset 2");
ASSERT_TRUE(run());
bind("SELECT * FROM t1 partition by c1 slimit 2, 5");
ASSERT_TRUE(run());
}
TEST_F(PlannerTest, showTables) {
setDatabase("root", "test");

19
source/libs/qworker/inc/qworkerMsg.h
View File

@ -30,21 +30,20 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg);
int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req);
int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code);
int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code);
int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code);
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete);
int32_t qwBuildAndSendDropRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
int32_t qwBuildAndSendCancelRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
int32_t qwBuildAndSendFetchRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength,
int32_t code);
void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete);
int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn);
int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code);
int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code);
void qwFreeFetchRsp(void *msg);
int32_t qwBuildAndSendReadyRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
int32_t qwBuildAndSendQueryRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code);
void qwFreeFetchRsp(void *msg);
int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp);
int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp);
int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code);
int32_t qwBuildAndSendHbRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SSchedulerHbRsp *rsp, int32_t code);
int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn);
#ifdef __cplusplus
}
#endif

47
source/libs/qworker/src/qworker.c
View File

@ -763,8 +763,8 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
dropConnection = &ctx->connInfo;
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
dropConnection = NULL;
qwBuildAndSendDropRsp(&ctx->connInfo, code);
qwBuildAndSendDropRsp(&mgmt->msgCb, &ctx->connInfo, code);
QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
@ -802,9 +802,9 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
dropConnection = NULL;
qwBuildAndSendDropRsp(&ctx->connInfo, code);
qwBuildAndSendDropRsp(&mgmt->msgCb, &ctx->connInfo, code);
QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
}
@ -830,12 +830,12 @@ _return:
}
if (dropConnection) {
qwBuildAndSendDropRsp(dropConnection, code);
qwBuildAndSendDropRsp(&mgmt->msgCb, dropConnection, code);
QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", dropConnection->handle, code, tstrerror(code));
}
if (cancelConnection) {
qwBuildAndSendCancelRsp(cancelConnection, code);
qwBuildAndSendCancelRsp(&mgmt->msgCb, cancelConnection, code);
QW_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", cancelConnection->handle, code, tstrerror(code));
}
@ -886,9 +886,9 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
qwBuildAndSendDropRsp(&ctx->connInfo, code);
qwBuildAndSendDropRsp(&mgmt->msgCb, &ctx->connInfo, code);
QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->connInfo.handle, code, tstrerror(code));
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED);
}
@ -918,7 +918,7 @@ _return:
}
if (readyConnection) {
qwBuildAndSendReadyRsp(readyConnection, code);
qwBuildAndSendReadyRsp(&mgmt->msgCb, readyConnection, code);
QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code));
}
@ -970,7 +970,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) {
QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR);
}
QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code));
QW_ERR_JRET(qwBuildAndSendQueryRsp(&mgmt->msgCb, &qwMsg->connInfo, code));
QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
queryRsped = true;
@ -986,9 +986,9 @@ _return:
input.code = code;
code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL);
if (!queryRsped) {
qwBuildAndSendQueryRsp(&qwMsg->connInfo, code);
qwBuildAndSendQueryRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
}
@ -1051,7 +1051,7 @@ _return:
}
if (needRsp) {
qwBuildAndSendReadyRsp(&qwMsg->connInfo, code);
qwBuildAndSendReadyRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
}
@ -1095,7 +1095,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
qwMsg->connInfo = ctx->connInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code);
qwBuildAndSendFetchRsp(&mgmt->msgCb, &qwMsg->connInfo, rsp, dataLen, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
} else {
atomic_store_8((int8_t*)&ctx->queryContinue, 1);
@ -1114,7 +1114,7 @@ _return:
rsp = NULL;
qwMsg->connInfo = ctx->connInfo;
qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code);
qwBuildAndSendFetchRsp(&mgmt->msgCb, &qwMsg->connInfo, rsp, 0, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0);
}
@ -1195,7 +1195,7 @@ _return:
}
if (code || rsp) {
qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code);
qwBuildAndSendFetchRsp(&mgmt->msgCb, &qwMsg->connInfo, rsp, dataLen, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen);
}
@ -1226,9 +1226,9 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx));
qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROPPING);
} else if (ctx->phase > 0) {
qwBuildAndSendDropRsp(&qwMsg->connInfo, code);
qwBuildAndSendDropRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
rsped = true;
} else {
@ -1241,7 +1241,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
}
_return:
if (code) {
@ -1261,7 +1261,7 @@ _return:
}
if (TSDB_CODE_SUCCESS != code) {
qwBuildAndSendDropRsp(&qwMsg->connInfo, code);
qwBuildAndSendDropRsp(&mgmt->msgCb, &qwMsg->connInfo, code);
QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
}
@ -1297,7 +1297,7 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
_return:
qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code);
qwBuildAndSendHbRsp(&mgmt->msgCb, &qwMsg->connInfo, &rsp, code);
QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code));
QW_RET(TSDB_CODE_SUCCESS);
@ -1351,8 +1351,9 @@ _return:
QW_UNLOCK(QW_READ, &mgmt->schLock);
for (int32_t j = 0; j < i; ++j) {
qwBuildAndSendHbRsp(&rspList[j].connInfo, &rspList[j].rsp, code);
QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code), (rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));
qwBuildAndSendHbRsp(&mgmt->msgCb, &rspList[j].connInfo, &rspList[j].rsp, code);
QW_DLOG("hb rsp send, handle:%p, code:%x - %s, taskNum:%d", rspList[j].connInfo.handle, code, tstrerror(code),
(rspList[j].rsp.taskStatus ? (int32_t)taosArrayGetSize(rspList[j].rsp.taskStatus) : 0));
tFreeSSchedulerHbRsp(&rspList[j].rsp);
}

47
source/libs/qworker/src/qworkerMsg.c
View File

@ -46,7 +46,7 @@ void qwFreeFetchRsp(void *msg) {
}
}
int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) {
int32_t qwBuildAndSendQueryRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
SQueryTableRsp rsp = {.code = code};
int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp);
@ -62,12 +62,12 @@ int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) {
.code = code,
};
rpcSendResponse(&rpcRsp);
tmsgSendRsp(pMsgCb, &rpcRsp);
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) {
int32_t qwBuildAndSendReadyRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
SResReadyRsp *pRsp = (SResReadyRsp *)rpcMallocCont(sizeof(SResReadyRsp));
pRsp->code = code;
@ -80,12 +80,12 @@ int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) {
.code = code,
};
rpcSendResponse(&rpcRsp);
tmsgSendRsp(pMsgCb, &rpcRsp);
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
int32_t qwBuildAndSendHbRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
void *pRsp = rpcMallocCont(contLen);
tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
@ -99,12 +99,12 @@ int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_
.code = code,
};
rpcSendResponse(&rpcRsp);
tmsgSendRsp(pMsgCb, &rpcRsp);
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) {
int32_t qwBuildAndSendFetchRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code) {
if (NULL == pRsp) {
pRsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp));
memset(pRsp, 0, sizeof(SRetrieveTableRsp));
@ -120,12 +120,12 @@ int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int3
.code = code,
};
rpcSendResponse(&rpcRsp);
tmsgSendRsp(pMsgCb, &rpcRsp);
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) {
int32_t qwBuildAndSendCancelRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
STaskCancelRsp *pRsp = (STaskCancelRsp *)rpcMallocCont(sizeof(STaskCancelRsp));
pRsp->code = code;
@ -138,11 +138,11 @@ int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) {
.code = code,
};
rpcSendResponse(&rpcRsp);
tmsgSendRsp(pMsgCb, &rpcRsp);
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) {
int32_t qwBuildAndSendDropRsp(const SMsgCb *pMsgCb, SQWConnInfo *pConn, int32_t code) {
STaskDropRsp *pRsp = (STaskDropRsp *)rpcMallocCont(sizeof(STaskDropRsp));
pRsp->code = code;
@ -155,11 +155,11 @@ int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) {
.code = code,
};
rpcSendResponse(&rpcRsp);
tmsgSendRsp(pMsgCb, &rpcRsp);
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
int32_t qwBuildAndSendShowRsp(const SMsgCb *pMsgCb, SRpcMsg *pMsg, int32_t code) {
int32_t numOfCols = 6;
SVShowTablesRsp showRsp = {0};
@ -210,11 +210,11 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
.code = code,
};
rpcSendResponse(&rpcMsg);
tmsgSendRsp(pMsgCb, &rpcMsg);
return TSDB_CODE_SUCCESS;
}
int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) {
int32_t qwBuildAndSendShowFetchRsp(const SMsgCb *pMsgCb, SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) {
SVShowTablesFetchRsp *pRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp));
int32_t handle = htonl(pFetchReq->id);
@ -227,7 +227,7 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe
.code = 0,
};
rpcSendResponse(&rpcMsg);
tmsgSendRsp(pMsgCb, &rpcMsg);
return TSDB_CODE_SUCCESS;
}
@ -287,7 +287,7 @@ int32_t qwRegisterBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) {
.code = TSDB_CODE_RPC_NETWORK_UNAVAIL,
};
rpcRegisterBrokenLinkArg(&pMsg);
tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg);
return TSDB_CODE_SUCCESS;
}
@ -498,7 +498,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
_return:
QW_ERR_RET(qwBuildAndSendCancelRsp(&qwMsg.connInfo, code));
QW_ERR_RET(qwBuildAndSendCancelRsp(&mgmt->msgCb, &qwMsg.connInfo, code));
QW_SCH_TASK_DLOG("cancel rsp send, handle:%p, code:%x - %s", qwMsg.connInfo.handle, code, tstrerror(code));
return TSDB_CODE_SUCCESS;
@ -579,15 +579,16 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_SUCCESS;
}
int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
SQWorkerMgmt *pMgmt = qWorkerMgmt;
int32_t code = 0;
SVShowTablesReq *pReq = pMsg->pCont;
QW_RET(qwBuildAndSendShowRsp(pMsg, code));
QW_RET(qwBuildAndSendShowRsp(&pMgmt->msgCb, pMsg, code));
}
int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
@ -595,8 +596,8 @@ int32_t qWorkerProcessShowFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg)
return TSDB_CODE_QRY_INVALID_INPUT;
}
SQWorkerMgmt *pMgmt = qWorkerMgmt;
SVShowTablesFetchReq *pFetchReq = pMsg->pCont;
QW_RET(qwBuildAndSendShowFetchRsp(pMsg, pFetchReq));
QW_RET(qwBuildAndSendShowFetchRsp(&pMgmt->msgCb, pMsg, pFetchReq));
}

31
source/libs/stream/src/tstream.c
View File

@ -74,7 +74,7 @@ static int32_t streamBuildDispatchMsg(SStreamTask* pTask, SArray* data, SRpcMsg*
pMsg->contLen = tlen;
pMsg->code = 0;
pMsg->msgType = pTask->dispatchMsgType;
/*pMsg->noResp = 1;*/
pMsg->noResp = 1;
return 0;
}
@ -124,22 +124,23 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
}
} else if (inputType == STREAM_DATA_TYPE_SSDATA_BLOCK) {
const SArray* blocks = (const SArray*)input;
int32_t sz = taosArrayGetSize(blocks);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pBlock = taosArrayGet(blocks, i);
qSetStreamInput(exec, pBlock, inputType);
while (1) {
SSDataBlock* output;
uint64_t ts;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
if (output == NULL) {
break;
}
taosArrayPush(pRes, output);
/*int32_t sz = taosArrayGetSize(blocks);*/
/*for (int32_t i = 0; i < sz; i++) {*/
/*SSDataBlock* pBlock = taosArrayGet(blocks, i);*/
/*qSetStreamInput(exec, pBlock, inputType);*/
qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK);
while (1) {
SSDataBlock* output;
uint64_t ts;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(false);
}
if (output == NULL) {
break;
}
taosArrayPush(pRes, output);
}
/*}*/
} else {
ASSERT(0);
}

8
source/libs/transport/src/transSrv.c
View File

@ -214,9 +214,11 @@ static void uvHandleReq(SSrvConn* pConn) {
// no ref here
}
if (pHead->noResp == 0) {
transMsg.handle = pConn;
}
// if pHead->noResp = 1,
// 1. server application should not send resp on handle
// 2. once send out data, cli conn released to conn pool immediately
// 3. not mixed with persist
transMsg.handle = pConn;
STrans* pTransInst = pConn->pTransInst;
(*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL);

55
source/util/src/tprocess.c
View File

@ -24,7 +24,6 @@
#include <sys/wait.h>
#define SHM_DEFAULT_SIZE (20 * 1024 * 1024)
#define CEIL8(n) (ceil((float)(n) / 8) * 8)
typedef void *(*ProcThreadFp)(void *param);
typedef struct SProcQueue {
@ -58,6 +57,11 @@ typedef struct SProcObj {
bool stopFlag;
} SProcObj;
static inline int32_t CEIL8(int32_t v) {
const int32_t c = ceil((float)(v) / 8) * 8;
return c < 8 ? 8 : c;
}
static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) {
TdThreadMutex *pMutex = NULL;
TdThreadMutexAttr mattr = {0};
@ -203,7 +207,8 @@ static void taosProcCleanupQueue(SProcQueue *pQueue) {
}
}
static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHeadLen, char *pBody, int32_t rawBodyLen) {
static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, const char *pBody,
int32_t rawBodyLen, ProcFuncType funcType) {
const int32_t headLen = CEIL8(rawHeadLen);
const int32_t bodyLen = CEIL8(rawBodyLen);
const int32_t fullLen = headLen + bodyLen + 8;
@ -221,10 +226,12 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea
}
if (pQueue->tail < pQueue->total) {
*(int32_t *)(pQueue->pBuffer + pQueue->head) = headLen;
*(int16_t *)(pQueue->pBuffer + pQueue->head) = headLen;
*(int8_t *)(pQueue->pBuffer + pQueue->head + 2) = (int8_t)funcType;
*(int32_t *)(pQueue->pBuffer + pQueue->head + 4) = bodyLen;
} else {
*(int32_t *)(pQueue->pBuffer) = headLen;
*(int16_t *)(pQueue->pBuffer) = headLen;
*(int8_t *)(pQueue->pBuffer + pQueue->head + 2) = (int8_t)funcType;
*(int32_t *)(pQueue->pBuffer + 4) = bodyLen;
}
@ -264,13 +271,13 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, char *pHead, int32_t rawHea
taosThreadMutexUnlock(pQueue->mutex);
tsem_post(&pQueue->sem);
uTrace("proc:%s, push msg to queue:%p remains:%d, head:%d:%p body:%d:%p", pQueue->name, pQueue, pQueue->items,
headLen, pHead, bodyLen, pBody);
uTrace("proc:%s, push msg to queue:%p remains:%d, head:%d:%p body:%d:%p ftype:%d", pQueue->name, pQueue, pQueue->items,
headLen, pHead, bodyLen, pBody, funcType);
return 0;
}
static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHeadLen, void **ppBody,
int32_t *pBodyLen) {
static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHeadLen, void **ppBody,
int32_t *pBodyLen, ProcFuncType *pFuncType) {
tsem_wait(&pQueue->sem);
taosThreadMutexLock(pQueue->mutex);
@ -281,13 +288,16 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea
return 0;
}
int32_t headLen = 0;
int16_t headLen = 0;
int8_t ftype = 0;
int32_t bodyLen = 0;
if (pQueue->head < pQueue->total) {
headLen = *(int32_t *)(pQueue->pBuffer + pQueue->head);
headLen = *(int16_t *)(pQueue->pBuffer + pQueue->head);
ftype = *(int8_t *)(pQueue->pBuffer + pQueue->head + 2);
bodyLen = *(int32_t *)(pQueue->pBuffer + pQueue->head + 4);
} else {
headLen = *(int32_t *)(pQueue->pBuffer);
headLen = *(int16_t *)(pQueue->pBuffer);
ftype = *(int8_t *)(pQueue->pBuffer + 2);
bodyLen = *(int32_t *)(pQueue->pBuffer + 4);
}
@ -341,9 +351,10 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int32_t *pHea
*ppBody = pBody;
*pHeadLen = headLen;
*pBodyLen = bodyLen;
*pFuncType = (ProcFuncType)ftype;
uTrace("proc:%s, pop msg from queue:%p remains:%d, head:%d:%p body:%d:%p", pQueue->name, pQueue, pQueue->items,
headLen, pHead, bodyLen, pBody);
uTrace("proc:%s, pop msg from queue:%p remains:%d, head:%d:%p body:%d:%p ftype:%d", pQueue->name, pQueue, pQueue->items,
headLen, pHead, bodyLen, pBody, ftype);
return 1;
}
@ -396,12 +407,14 @@ static void taosProcThreadLoop(SProcQueue *pQueue) {
ProcConsumeFp consumeFp = pQueue->consumeFp;
void *pParent = pQueue->pParent;
void *pHead, *pBody;
int32_t headLen, bodyLen;
int16_t headLen;
ProcFuncType ftype;
int32_t bodyLen;
uDebug("proc:%s, start to get msg from queue:%p", pQueue->name, pQueue);
while (1) {
int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen);
int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype);
if (numOfMsgs == 0) {
uDebug("proc:%s, get no msg from queue:%p and exit the proc thread", pQueue->name, pQueue);
break;
@ -410,7 +423,7 @@ static void taosProcThreadLoop(SProcQueue *pQueue) {
taosMsleep(1);
continue;
} else {
(*consumeFp)(pParent, pHead, headLen, pBody, bodyLen);
(*consumeFp)(pParent, pHead, headLen, pBody, bodyLen, ftype);
}
}
}
@ -458,10 +471,12 @@ void taosProcCleanup(SProcObj *pProc) {
}
}
int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) {
return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen);
int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
ProcFuncType funcType) {
return taosProcQueuePush(pProc->pChildQueue, pHead, headLen, pBody, bodyLen, funcType);
}
int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen) {
return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen);
int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
ProcFuncType funcType) {
return taosProcQueuePush(pProc->pParentQueue, pHead, headLen, pBody, bodyLen, funcType);
}

5
tests/script/jenkins/basic.txt
View File

@ -21,6 +21,10 @@
./test.sh -f tsim/insert/backquote.sim
./test.sh -f tsim/insert/null.sim
# ---- parser
#./test.sh -f tsim/parser/groupby-basic.sim
#./test.sh -f tsim/parser/fourArithmetic-basic.sim
# ---- query
./test.sh -f tsim/query/interval.sim
./test.sh -f tsim/query/interval-offset.sim
@ -33,5 +37,6 @@
# ---- tmq
./test.sh -f tsim/tmq/basic.sim
./test.sh -f tsim/tmq/basic1.sim
#======================b1-end===============

2
tests/script/tsim/insert/basic1.sim
View File

@ -41,7 +41,7 @@ print =============== insert data, mode1: one row one table in sql
print =============== insert data, mode1: mulit rows one table in sql
print =============== insert data, mode1: one rows mulit table in sql
print =============== insert data, mode1: mulit rows mulit table in sql
sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40)
sql insert into c1 values(now-1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40)
sql insert into c1 values(now+0s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+1s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40) (now+2s, true, -1, -2, -3, -4, -6.0, -7.0, 'child tbl 1', 'child tbl 1', '2022-02-25 18:00:00.000', 10, 20, 30, 40)
print =============== query data

110
tests/script/tsim/parser/fourArithmetic-basic.sim Normal file
View File

@ -0,0 +1,110 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
$dbNamme = d0
print =============== create database
sql create database $dbNamme vgroups 1
sql show databases
print $data00 $data01 $data02
if $rows != 2 then
return -1
endi
sql use $dbNamme
print =============== create super table
sql create table if not exists stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double) tags (t1 int)
sql show stables
if $rows != 1 then
return -1
endi
print =============== create child table
sql create table ct0 using stb tags(1000)
#sql create table ct1 using stb tags(2000)
#sql create table ct3 using stb tags(3000)
sql show tables
if $rows != 1 then
return -1
endi
print =============== insert data
$tbPrefix = ct
$tbNum = 1
$rowNum = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
$x = 0
while $x < $rowNum
$c2 = $x + 10
$c3 = $x * 10
$c4 = $x - 10
sql insert into $tb values ($tstart , $x , $c2 , $c3 , $c4 )
$tstart = $tstart + 1
$x = $x + 1
endw
$i = $i + 1
$tstart = 1640966400000
endw
sql select ts, c2-c1, c3/c1, c4+c1, c1*9, c1%3 from ct0
print ===> rows: $rows
print ===> $data00 $data01 $data02 $data03 $data04 $data05
print ===> $data10 $data11 $data12 $data13 $data14 $data15
print ===> $data20 $data21 $data22 $data23 $data24 $data25
print ===> $data30 $data31 $data32 $data33 $data34 $data35
if $rows != 10 then
return -1
endi
if $data01 != 10.000000000 then
return -1
endi
if $data02 != -nan then
return -1
endi
if $data03 != -10.000000000 then
return -1
endi
if $data91 != 10.000000000 then
return -1
endi
if $data92 != 10.000000000 then
return -1
endi
if $data93 != 8.000000000 then
return -1
endi
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

786
tests/script/tsim/parser/groupby-basic.sim Normal file
View File

@ -0,0 +1,786 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c walLevel -v 1
system sh/cfg.sh -n dnode1 -c maxtablespervnode -v 4
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
$dbPrefix = group_db
$tbPrefix = group_tb
$mtPrefix = group_mt
$tbNum = 8
$rowNum = 100
$totalNum = $tbNum * $rowNum
print =============== groupby.sim
$i = 0
$db = $dbPrefix . $i
$mt = $mtPrefix . $i
$i = 0
$db = $dbPrefix . $i
$mt = $mtPrefix . $i
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
print ==== create db, stable, ctables, insert data
sql drop database if exists $db -x step1
step1:
sql create database if not exists $db keep 3650
sql use $db
sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
$tg2 = ' . abc
$tg2 = $tg2 . '
sql create table $tb using $mt tags( $i , $tg2 )
$x = 0
while $x < $rowNum
$c = $x / 10
$c = $c * 10
$c = $x - $c
$binary = ' . binary
$binary = $binary . $c
$binary = $binary . '
$nchar = ' . nchar
$nchar = $nchar . $c
$nchar = $nchar . '
sql insert into $tb values ($tstart , $c , $c , $x , $x , $c , $c , $c , $binary , $nchar )
#print ==== insert into $tb values ($tstart , $c , $c , $x , $x , $c , $c , $c , $binary , $nchar )
$tstart = $tstart + 1
$x = $x + 1
endw
$i = $i + 1
$tstart = 1640966400000
endw
sleep 100
$i1 = 1
$i2 = 0
$db = $dbPrefix . $i
$mt = $mtPrefix . $i
$dbPrefix = group_db
$tbPrefix = group_tb
$mtPrefix = group_mt
$tb1 = $tbPrefix . $i1
$tb2 = $tbPrefix . $i2
$ts1 = $tb1 . .ts
$ts2 = $tb2 . .ts
print ===============================groupby_operation
print
print ==== select count(*), c1 from group_tb0 group by c1
sql select count(*), c1 from group_tb0 group by c1
print rows: $rows
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data80 $data81 $data82 $data83
print $data90 $data91 $data92 $data93
if $rows != 10 then
return -1
endi
#if $data00 != 10 then
# return -1
#endi
if $data01 != 0 then
return -1
endi
#if $data10 != 10 then
# return -1
#endi
if $data11 != 1 then
return -1
endi
#if $data90 != 10 then
# return -1
#endi
if $data91 != 9 then
return -1
endi
sql select first(ts),c1 from group_tb0 group by c1;
print rows: $rows
print $data00 $data01 $data02 $data03
print $data10 $data11 $data12 $data13
print $data20 $data21 $data22 $data23
print $data80 $data81 $data82 $data83
print $data90 $data91 $data92 $data93
if $row != 10 then
return -1
endi
if $data00 != @2022-01-01 00:00:00.000@ then
return -1
endi
if $data01 != 0 then
return -1
endi
if $data90 != @2022-01-01 00:00:00.009@ then
return -1
endi
if $data91 != 9 then
return -1
endi
sql select sum(c1), c1, avg(c1), min(c1), max(c2) from group_tb0 where c1 < 20 group by c1;
if $row != 20 then
return -1
endi
if $data00 != 0 then
return -1
endi
if $data01 != 0 then
return -1
endi
print $data02
if $data02 != 0.000000000 then
return -1
endi
if $data03 != 0 then
return -1
endi
print $data04
if $data04 != 0.00000 then
return -1
endi
if $data10 != 100 then
return -1
endi
if $data11 != 1 then
return -1
endi
print $data12
if $data12 != 1.000000000 then
return -1
endi
if $data13 != 1 then
return -1
endi
if $data14 != 1.00000 then
print expect 1.00000, actual:$data14
return -1
endi
sql_error select sum(c1), ts, c1 from group_tb0 where c1<20 group by c1;
sql_error select first(ts), ts, c2 from group_tb0 where c1 < 20 group by c1;
sql_error select sum(c3), ts, c2 from group_tb0 where c1 < 20 group by c1;
sql_error select sum(c3), first(ts), c2 from group_tb0 where c1 < 20 group by c1;
sql_error select first(c3), ts, c1, c2 from group_tb0 where c1 < 20 group by c1;
sql_error select first(c3), last(c3), ts, c1 from group_tb0 where c1 < 20 group by c1;
sql_error select ts from group_tb0 group by c1;
#===========================interval=====not support======================
sql_error select count(*), c1 from group_tb0 where c1<20 interval(1y) group by c1;
#=====tbname must be the first in the group by clause=====================
sql_error select count(*) from group_tb0 where c1 < 20 group by c1, tbname;
#super table group by normal columns
sql select count(*), c1 from group_mt0 where c1< 20 group by c1;
if $row != 20 then
return -1
endi
if $data00 != 800 then
return -1
endi
if $data01 != 0 then
return -1
endi
if $data10 != 800 then
return -1
endi
if $data11 != 1 then
return -1
endi
if $data90 != 800 then
return -1
endi
if $data91 != 9 then
return -1
endi
sql select first(c1), c1, ts from group_mt0 where c1<20 group by c1;
if $row != 20 then
return -1
endi
if $data00 != $data01 then
return -1
endi
if $data02 != @70-01-01 08:01:40.000@ then
return -1
endi
if $data10 != $data11 then
return -1
endi
if $data12 != @70-01-01 08:01:40.001@ then
return -1
endi
if $data20 != $data21 then
return -1
endi
if $data22 != @70-01-01 08:01:40.002@ then
return -1
endi
if $data90 != $data91 then
return -1
endi
if $data92 != @70-01-01 08:01:40.009@ then
return -1
endi
sql select first(c1), last(ts), first(ts), last(c1),c1,sum(c1),avg(c1),count(*) from group_mt0 where c1<20 group by c1;
if $row != 20 then
return -1
endi
if $data00 != $data03 then
return -1
endi
if $data01 != @70-01-01 08:01:49.900@ then
return -1
endi
if $data02 != @70-01-01 08:01:40.000@ then
return -1
endi
if $data07 != 800 then
return -1
endi
if $data10 != $data13 then
return -1
endi
if $data11 != @70-01-01 08:01:49.901@ then
return -1
endi
if $data12 != @70-01-01 08:01:40.001@ then
return -1
endi
if $data17 != 800 then
return -1
endi
if $data90 != $data93 then
return -1
endi
if $data91 != @70-01-01 08:01:49.909@ then
return -1
endi
if $data92 != @70-01-01 08:01:40.009@ then
return -1
endi
if $data97 != 800 then
return -1
endi
if $data95 != 7200 then
return -1
endi
if $data94 != 9 then
return -1
endi
sql select c1,sum(c1),avg(c1),count(*) from group_mt0 where c1<5 group by c1;
if $row != 5 then
return -1
endi
if $data00 != 0 then
return -1
endi
if $data11 != 800 then
return -1
endi
sql select first(c1), last(ts), first(ts), last(c1),sum(c1),avg(c1),count(*) from group_mt0 where c1<20 group by tbname,c1;
if $row != 160 then
return -1
endi
print $data00
if $data00 != 0 then
return -1
endi
if $data01 != @70-01-01 08:01:49.900@ then
return -1
endi
print $data01
if $data02 != @70-01-01 08:01:40.000@ then
return -1
endi
if $data03 != 0 then
return -1
endi
if $data04 != 0 then
return -1
endi
if $data06 != 100 then
return -1
endi
if $data07 != @group_tb0@ then
return -1
endi
if $data90 != 9 then
return -1
endi
if $data91 != @70-01-01 08:01:49.909@ then
return -1
endi
if $data92 != @70-01-01 08:01:40.009@ then
return -1
endi
if $data93 != 9 then
return -1
endi
if $data94 != 900 then
return -1
endi
if $data96 != 100 then
return -1
endi
if $data97 != @group_tb0@ then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4;
if $rows != 10000 then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data01 != @70-01-01 08:01:40.000@ then
return -1
endi
if $data02 != @70-01-01 08:01:40.000@ then
return -1
endi
if $data03 != 0 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 1;
if $rows != 1 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3) from group_tb1 group by c4 limit 20 offset 9990;
if $rows != 10 then
return -1
endi
sql select count(*),first(ts),last(ts),min(c3),max(c3),sum(c3),avg(c3),sum(c4)/count(c4) from group_tb1 group by c4;
if $rows != 10000 then
return -1
endi
print ---------------------------------> group by binary|nchar data add cases
sql select count(*) from group_tb1 group by c8;
if $rows != 100 then
return -1
endi
sql select count(*),sum(c4), count(c4), sum(c4)/count(c4) from group_tb1 group by c8
if $rows != 100 then
return -1
endi
if $data00 != 100 then
return -1
endi
if $data01 != 495000 then
return -1
endi
if $data02 != 100 then
return -1
endi
if $data03 != 4950.000000000 then
print expect 4950.000000000 , acutal $data03
return -1
endi
if $data10 != 100 then
return -1
endi
if $data11 != 495100 then
return -1
endi
if $data13 != 4951.000000000 then
return -1
endi
print ====================> group by normal column + slimit + soffset
sql select count(*), c8 from group_mt0 group by c8 limit 1 offset 0;
if $rows != 100 then
return -1
endi
sql select sum(c2),c8,avg(c2), sum(c2)/count(*) from group_mt0 group by c8 slimit 2 soffset 99
if $rows != 1 then
return -1
endi
if $data00 != 79200.000000000 then
return -1
endi
if $data01 != @binary99@ then
return -1
endi
if $data02 != 99.000000000 then
return -1
endi
if $data03 != 99.000000000 then
return -1
endi
print ============>td-1765
sql select percentile(c4, 49),min(c4),max(c4),avg(c4),stddev(c4) from group_tb0 group by c8;
if $rows != 100 then
return -1
endi
if $data00 != 4851.000000000 then
return -1
endi
if $data01 != 0 then
return -1
endi
if $data02 != 9900 then
return -1
endi
if $data03 != 4950.000000000 then
return -1
endi
if $data04 != 2886.607004772 then
return -1
endi
if $data10 != 4852.000000000 then
return -1
endi
if $data11 != 1 then
return -1
endi
if $data12 != 9901 then
return -1
endi
if $data13 != 4951.000000000 then
return -1
endi
if $data14 != 2886.607004772 then
return -1
endi
print ================>td-2090
sql select leastsquares(c2, 1, 1) from group_tb1 group by c8;
if $rows != 100 then
return -1
endi
if $data00 != @{slop:0.000000, intercept:0.000000}@ then
return -1
endi
if $data10 != @{slop:0.000000, intercept:1.000000}@ then
return -1
endi
if $data90 != @{slop:0.000000, intercept:9.000000}@ then
return -1
endi
#=========================== group by multi tags ======================
sql create table st (ts timestamp, c int) tags (t1 int, t2 int, t3 int, t4 int);
sql create table t1 using st tags(1, 1, 1, 1);
sql create table t2 using st tags(1, 2, 2, 2);
sql insert into t1 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ;
sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ;
sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ;
sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ;
print =================>TD-2665
sql_error create table txx as select avg(c) as t from st;
sql_error create table txx1 as select avg(c) as t from t1;
sql select stddev(c),stddev(c) from st group by c;
if $rows != 4 then
return -1
endi
print =================>TD-2236
sql select first(ts),last(ts) from t1 group by c;
if $rows != 4 then
return -1
endi
if $data00 != @20-03-27 04:11:16.000@ then
return -1
endi
if $data01 != @20-03-27 04:21:16.000@ then
return -1
endi
if $data10 != @20-03-27 04:11:17.000@ then
return -1
endi
if $data11 != @20-03-27 04:31:17.000@ then
return -1
endi
if $data20 != @20-03-27 04:11:18.000@ then
return -1
endi
if $data21 != @20-03-27 04:51:18.000@ then
return -1
endi
if $data30 != @20-03-27 04:11:19.000@ then
return -1
endi
if $data31 != @20-03-27 05:10:19.000@ then
return -1
endi
print ===============>
sql select stddev(c),c from st where t2=1 or t2=2 group by c;
if $rows != 4 then
return -1
endi
if $data00 != 0.000000000 then
return -1
endi
if $data01 != 1 then
return -1
endi
if $data10 != 0.000000000 then
return -1
endi
if $data11 != 2 then
return -1
endi
if $data20 != 0.000000000 then
return -1
endi
if $data21 != 3 then
return -1
endi
if $data30 != 0.000000000 then
return -1
endi
if $data31 != 4 then
return -1
endi
sql_error select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,c;
sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2;
if $rows != 40 then
return -1
endi
if $data01 != 1.000000000 then
return -1
endi
if $data02 != t1 then
return -1
endi
if $data03 != 1 then
return -1
endi
if $data04 != 1 then
return -1
endi
if $data11 != 1.000000000 then
return -1
endi
if $data12 != t1 then
return -1
endi
if $data13 != 1 then
return -1
endi
if $data14 != 1 then
return -1
endi
sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1;
if $rows != 2 then
return -1
endi
if $data11 != 1.000000000 then
return -1
endi
if $data12 != t2 then
return -1
endi
if $data13 != 1 then
return -1
endi
if $data14 != 2 then
return -1
endi
sql create table m1 (ts timestamp, k int, f1 int) tags(a int);
sql create table tm0 using m1 tags(0);
sql create table tm1 using m1 tags(1);
sql insert into tm0 values('2020-1-1 1:1:1', 1, 10);
sql insert into tm0 values('2020-1-1 1:1:2', 1, 20);
sql insert into tm1 values('2020-2-1 1:1:1', 2, 10);
sql insert into tm1 values('2020-2-1 1:1:2', 2, 20);
system sh/exec.sh -n dnode1 -s stop -x SIGINT
sleep 100
system sh/exec.sh -n dnode1 -s start
sleep 100
sql connect
sleep 100
sql use group_db0;
print =========================>TD-4894
sql select count(*),k from m1 group by k;
if $rows != 2 then
return -1
endi
if $data00 != 2 then
return -1
endi
if $data01 != 1 then
return -1
endi
if $data10 != 2 then
return -1
endi
if $data11 != 2 then
return -1
endi
sql_error select count(*) from m1 group by tbname,k,f1;
sql_error select count(*) from m1 group by tbname,k,a;
sql_error select count(*) from m1 group by k, tbname;
sql_error select count(*) from m1 group by k,f1;
sql_error select count(*) from tm0 group by tbname;
sql_error select count(*) from tm0 group by a;
sql_error select count(*) from tm0 group by k,f1;
sql_error select count(*),f1 from m1 group by tbname,k;
system sh/exec.sh -n dnode1 -s stop -x SIGINT

126
tests/script/tsim/query/interval-offset.sim
View File

@ -41,6 +41,38 @@ sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6 )
sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7 )
sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8 )
print =============== insert data into child table ct2 (d)
sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 )
sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 )
sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 )
sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 )
sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 )
sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 )
print =============== insert data into child table ct3 (n)
sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL )
sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 )
sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 )
sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 )
sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 )
sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 )
sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 )
sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 )
sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 )
print =============== insert data into child table ct4 (y)
sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 )
sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 )
sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 )
sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 )
sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 )
sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 )
sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 )
sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
print ================ start query ======================
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s)
print ===> rows: $rows
@ -102,15 +134,6 @@ if $data80 != 1 then
return -1
endi
print =============== insert data into child table ct2 (d)
sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1 )
sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2 )
sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3 )
sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4 )
sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5 )
sql insert into ct2 values ( '2022-01-03 10:00:01.000', 6 )
sql insert into ct2 values ( '2022-01-03 20:00:01.000', 7 )
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h)
print ===> rows: $rows
@ -140,45 +163,19 @@ print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45
print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55
print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65
print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75
#if $rows != 8 then
# return -1
#endi
#if $data00 != 1 then
# return -1
#endi
#if $data10 != 2 then
# return -1
#endi
#if $data20 != 2 then
# return -1
#endi
#if $data30 != 2 then
# return -1
#endi
#if $data40 != 2 then
# return -1
#endi
#if $data50 != 2 then
# return -1
#endi
#if $data60 != 2 then
# return -1
#endi
#if $data70 != 1 then
# return -1
#endi
print =============== insert data into child table ct3 (n)
sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL );
sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1 );
sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2 );
sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3 );
sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4 );
sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5 );
sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6 );
sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7 );
sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8 );
if $rows != 8 then
return -1
endi
if $data00 != 1 then
return -1
endi
if $data10 != 2 then
return -1
endi
if $data70 != 1 then
return -1
endi
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w)
print ===> rows: $rows
@ -236,18 +233,6 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74
# return -1
#endi
print =============== insert data into child table ct4 (y)
sql insert into ct4 values ( '2020-10-21 01:01:01.000', 1 )
sql insert into ct4 values ( '2020-12-31 01:01:01.000', 2 )
sql insert into ct4 values ( '2021-01-01 01:01:06.000', 3 )
sql insert into ct4 values ( '2021-05-07 01:01:10.000', 4 )
sql insert into ct4 values ( '2021-09-30 01:01:16.000', 5 )
sql insert into ct4 values ( '2022-02-01 01:01:20.000', 6 )
sql insert into ct4 values ( '2022-10-28 01:01:26.000', 7 )
sql insert into ct4 values ( '2022-12-01 01:01:30.000', 8 )
sql insert into ct4 values ( '2022-12-31 01:01:36.000', 9 )
sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n)
print ===> rows: $rows
@ -305,6 +290,27 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74
# return -1
#endi
#=================================================
print =============== stop and restart taosd
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi

9
tests/script/tsim/testCaseSuite.sim
View File

@ -2,6 +2,8 @@
run tsim/user/basic1.sim
run tsim/db/basic1.sim
run tsim/db/basic2.sim
run tsim/db/basic3.sim
run tsim/db/basic6.sim
run tsim/db/basic7.sim
run tsim/db/error1.sim
@ -13,12 +15,15 @@ run tsim/insert/basic1.sim
run tsim/insert/backquote.sim
run tsim/insert/null.sim
#run tsim/parser/groupby-basic.sim
#run tsim/parser/fourArithmetic-basic.sim
run tsim/query/interval.sim
#run tsim/query/interval-offset.sim # TD-14266
run tsim/query/interval-offset.sim
run tsim/show/basic.sim
run tsim/table/basic1.sim
run tsim/tmq/basic.sim
run tsim/tmq/basic1.sim

183
tests/script/tsim/tmq/basic1.sim Normal file
View File

@ -0,0 +1,183 @@
#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406
# vgroups=1, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# vgroups=1, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# vgroups=4, one topic for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# vgroups=4, multi topics for one consumer, include: columns from stb/ctb/ntb, * from stb/ctb/ntb, Scalar function from stb/ctb/ntb
# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN
# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5;
#
# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval).
#
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1
system sh/exec.sh -n dnode1 -s start
$loop_cnt = 0
check_dnode_ready:
$loop_cnt = $loop_cnt + 1
sleep 200
if $loop_cnt == 10 then
print ====> dnode not ready!
return -1
endi
sql show dnodes
print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05
if $data00 != 1 then
return -1
endi
if $data04 != ready then
goto check_dnode_ready
endi
sql connect
$dbNamme = d0
print =============== create database , vgroup 1
sql create database $dbNamme vgroups 1
sql show databases
print $data00 $data01 $data02
if $rows != 2 then
return -1
endi
sql use $dbNamme
print =============== create super table
sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int)
sql show stables
if $rows != 1 then
return -1
endi
print =============== create child table
sql create table ct0 using stb tags(1000)
sql create table ct1 using stb tags(2000)
#sql create table ct3 using stb tags(3000)
print =============== create normal table
sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10))
print =============== create multi topics. notes: now only support:
print =============== 1. columns from stb; 2. * from ctb; 3. columns from ctb
print =============== will support: * from stb; function from stb/ctb
sql create topic topic_stb_column as select ts, c1, c3 from stb
#sql create topic topic_stb_all as select * from stb
#sql create topic topic_stb_function as select ts, abs(c1), sina(c2) from stb
sql create topic topic_ctb_column as select ts, c1, c3 from ct0
sql create topic topic_ctb_all as select * from ct0
#sql create topic topic_ctb_function as select ts, abs(c1), sina(c2) from ct0
sql create topic topic_ntb_column as select ts, c1, c3 from ntb
sql create topic topic_ntb_all as select * from ntb
#sql create topic topic_ntb_function as select ts, abs(c1), sina(c2) from ntb
sql show tables
if $rows != 3 then
return -1
endi
print =============== insert data
$tbPrefix = ct
$tbNum = 2
$rowNum = 10
$tstart = 1640966400000 # 2022-01-01 00:00:00.000
$i = 0
while $i < $tbNum
$tb = $tbPrefix . $i
$x = 0
while $x < $rowNum
$c = $x / 10
$c = $c * 10
$c = $x - $c
$binary = ' . binary
$binary = $binary . $c
$binary = $binary . '
sql insert into $tb values ($tstart , $c , $x , $binary )
sql insert into ntb values ($tstart , $c , $x , $binary )
$tstart = $tstart + 1
$x = $x + 1
endw
$i = $i + 1
$tstart = 1640966400000
endw
#root@trd02 /home $ tmq_sim --help
# -c Configuration directory, default is
# -d The name of the database for cosumer, no default
# -t The topic string for cosumer, no default
# -k The key-value string for cosumer, no default
# -g showMsgFlag, default is 0
#
$totalMsgCnt = $rowNum * $tbNum
print inserted totalMsgCnt: $totalMsgCnt
# supported key:
# group.id:<xxx>
# enable.auto.commit:<true | false>
# auto.offset.reset:<earliest | latest | none>
# td.connect.ip:<fqdn | ipaddress>
# td.connect.user:root
# td.connect.pass:taosdata
# td.connect.port:6030
# td.connect.db:db
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
return -1
endi
#print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
#system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2"
#print cmd result----> $system_content
#if $system_content != @{consume success: 20, 0}@ then
# return -1
#endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 10, 0}@ then
return -1
endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 10, 0}@ then
return -1
endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
return -1
endi
print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2"
print cmd result----> $system_content
if $system_content != @{consume success: 20, 0}@ then
return -1
endi
print =============== create database , vgroup 4
$dbNamme = d1
sql create database $dbNamme vgroups 4
#system sh/exec.sh -n dnode1 -s stop -x SIGINT

4
tests/test/c/tmqDemo.c
View File

@ -67,7 +67,7 @@ static SConfInfo g_stConfInfo = {
"tmqdb",
"stb",
"./tmqResult.txt", // output_file
"/data2/dnode/data/vnode/vnode2/wal",
"", // /data2/dnode/data/vnode/vnode2/wal",
1, // threads
1, // tables
1, // vgroups
@ -662,7 +662,7 @@ int main(int32_t argc, char *argv[]) {
float msgsSpeed = totalMsgs / seconds;
if (0 == g_stConfInfo.simCase) {
if ((0 == g_stConfInfo.simCase) && (strlen(g_stConfInfo.vnodeWalPath))) {
walLogSize = getDirectorySize(g_stConfInfo.vnodeWalPath);
if (walLogSize <= 0) {
printf("%s size incorrect!", g_stConfInfo.vnodeWalPath);

86
tests/test/c/tmqSim.c
View File

@ -98,7 +98,7 @@ void parseArgument(int32_t argc, char *argv[]) {
}
}
#if 1
#if 0
pPrint("%s configDir:%s %s", GREEN, configDir, NC);
pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC);
pPrint("%s topicString:%s %s", GREEN, g_stConfInfo.topicString, NC);
@ -151,8 +151,6 @@ void parseInputString() {
token = strtok(NULL, delim);
}
printf("\n\n");
token = strtok(g_stConfInfo.keyString, delim);
while(token != NULL) {
//printf("%s\n", token );
@ -198,6 +196,8 @@ tmq_t* build_consumer() {
TAOS_RES* pRes = taos_query(pConn, sqlStr);
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes);
exit(-1);
}
taos_free_result(pRes);
@ -219,61 +219,33 @@ tmq_list_t* build_topic_list() {
return topic_list;
}
void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
static const int MIN_COMMIT_COUNT = 1000;
int msg_count = 0;
void loop_consume(tmq_t* tmq) {
tmq_resp_err_t err;
if ((err = tmq_subscribe(tmq, topics))) {
fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err));
return;
}
while (running) {
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
if (tmqmessage) {
msg_process(tmqmessage);
tmq_message_destroy(tmqmessage);
if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0);
}
}
err = tmq_consumer_close(tmq);
if (err)
fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err));
else
fprintf(stderr, "%% Consumer closed\n");
}
void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
tmq_resp_err_t err;
if ((err = tmq_subscribe(tmq, topics))) {
printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
return;
}
int32_t totalMsgs = 0;
int32_t totalRows = 0;
int32_t skipLogNum = 0;
//int64_t startTime = taosGetTimestampUs();
while (running) {
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
if (tmqmessage) {
totalMsgs++;
skipLogNum += tmqGetSkipLogNum(tmqmessage);
if (0 != g_stConfInfo.showMsgFlag) {
msg_process(tmqmessage);
tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 1);
if (tmqMsg) {
totalMsgs++;
#if 0
TAOS_ROW row;
while (NULL != (row = tmq_get_row(tmqMsg))) {
totalRows++;
}
tmq_message_destroy(tmqmessage);
#endif
skipLogNum += tmqGetSkipLogNum(tmqMsg);
if (0 != g_stConfInfo.showMsgFlag) {
msg_process(tmqMsg);
}
tmq_message_destroy(tmqMsg);
} else {
break;
}
}
//int64_t endTime = taosGetTimestampUs();
//double consumeTime = (double)(endTime - startTime) / 1000000;
err = tmq_consumer_close(tmq);
if (err) {
@ -281,7 +253,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
exit(-1);
}
printf("{consume success: %d}", totalMsgs);
printf("{consume success: %d, %d}", totalMsgs, totalRows);
}
int main(int32_t argc, char *argv[]) {
@ -294,7 +266,21 @@ int main(int32_t argc, char *argv[]) {
return -1;
}
perf_loop(tmq, topic_list);
tmq_resp_err_t err = tmq_subscribe(tmq, topic_list);
if (err) {
printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
exit(-1);
}
loop_consume(tmq);
#if 0
err = tmq_unsubscribe(tmq);
if (err) {
printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
exit(-1);
}
#endif
return 0;
}