xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/meta

This commit is contained in:
Hongze Cheng 2022-03-30 01:48:57 +00:00
parent f0e6b6ede6 869ce1e060
commit f8947c1790
164 changed files with 6266 additions and 4839 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.clang-format
View File

@ -5,7 +5,6 @@ AccessModifierOffset: -1
AlignAfterOpenBracket: Align
AlignConsecutiveAssignments: false
AlignConsecutiveDeclarations: true
AlignConsecutiveMacros: true
AlignEscapedNewlinesLeft: true
AlignOperands: true
AlignTrailingComments: true

6
2.0/src/query/inc/qExecutor.h
View File

@ -574,11 +574,11 @@ typedef struct SMultiwayMergeInfo {
} SMultiwayMergeInfo;
// todo support the disk-based sort
typedef struct SOrderOperatorInfo {
typedef struct SSortOperatorInfo {
int32_t colIndex;
int32_t order;
SSDataBlock *pDataBlock;
} SOrderOperatorInfo;
} SSortOperatorInfo;
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
@ -609,7 +609,7 @@ SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperator
int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput);
SOperatorInfo* createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal);
SOperatorInfo* createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal);
SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);

10
2.0/src/query/src/qExecutor.c
View File

@ -2301,7 +2301,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
}
case OP_Order: {
pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order);
pRuntimeEnv->proot = createSortOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order);
break;
}
@ -5516,7 +5516,7 @@ static SSDataBlock* doSort(void* param, bool* newgroup) {
return NULL;
}
SOrderOperatorInfo* pInfo = pOperator->info;
SSortOperatorInfo* pInfo = pOperator->info;
SSDataBlock* pBlock = NULL;
while(1) {
@ -5556,8 +5556,8 @@ static SSDataBlock* doSort(void* param, bool* newgroup) {
return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL;
}
SOperatorInfo *createOrderOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal) {
SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo));
SOperatorInfo *createSortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrderVal* pOrderVal) {
SSortOperatorInfo* pInfo = calloc(1, sizeof(SSortOperatorInfo));
{
SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock));
@ -6611,7 +6611,7 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
}
static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param;
SSortOperatorInfo* pInfo = (SSortOperatorInfo*) param;
pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock);
}

5
example/src/tmq.c
View File

@ -80,8 +80,9 @@ int32_t create_topic() {
}
taos_free_result(pRes);
const char* sql = "select * from tu1";
pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
/*const char* sql = "select * from tu1";*/
/*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
pRes = taos_query(pConn, "create topic test_stb_topic_1 as select * from tu1");
if (taos_errno(pRes) != 0) {
printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));
return -1;

4
example/src/tstream.c
View File

@ -20,7 +20,7 @@
#include "taos.h"
int32_t init_env() {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
if (pConn == NULL) {
return -1;
}
@ -65,7 +65,7 @@ int32_t init_env() {
int32_t create_stream() {
printf("create stream\n");
TAOS_RES* pRes;
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 7010);
if (pConn == NULL) {
return -1;
}

78
include/client/consumer/consumer.h
View File

@ -1,78 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_CONSUMER_H_
#define _TD_CONSUMER_H_
#include "tlist.h"
#include "tarray.h"
#include "hash.h"
#ifdef __cplusplus
extern "C" {
#endif
//consumer handle
struct tmq_consumer_t;
typedef struct tmq_consumer_t tmq_consumer_t;
//consumer config
struct tmq_consumer_config_t;
typedef struct tmq_consumer_config_t tmq_consumer_config_t;
//response err
struct tmq_resp_err_t;
typedef struct tmq_resp_err_t tmq_resp_err_t;
struct tmq_message_t;
typedef struct tmq_message_t tmq_message_t;
struct tmq_col_batch_t;
typedef struct tmq_col_batch_t tmq_col_batch_t;
//get content of message
tmq_col_batch_t* tmq_get_msg_col_by_idx(tmq_message_t*, int32_t col_id);
tmq_col_batch_t* tmq_get_msg_col_by_name(tmq_message_t*, const char*);
//consumer config
int32_t tmq_conf_set(tmq_consumer_config_t* , const char* config_key, const char* config_value, char* errstr, int32_t errstr_cap);
//consumer initialization
//resouces are supposed to be free by users by calling tmq_consumer_destroy
tmq_consumer_t* tmq_consumer_new(tmq_consumer_config_t* , char* errstr, int32_t errstr_cap);
//subscribe
tmq_resp_err_t tmq_subscribe(tmq_consumer_t*, const SList*);
tmq_resp_err_t tmq_unsubscribe(tmq_consumer_t*);
//consume
//resouces are supposed to be free by users by calling tmq_message_destroy
tmq_message_t* tmq_consume_poll(tmq_consumer_t*, int64_t blocking_time);
//destroy message and free memory
void tmq_message_destroy(tmq_message_t*);
//close consumer
int32_t tmq_consumer_close(tmq_consumer_t*);
//destroy consumer
void tmq_consumer_destroy(tmq_message_t*);
#ifdef __cplusplus
}
#endif
#endif /*_TD_CONSUMER_H_*/

27
include/client/stream/stream.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_STREAM_H_
#define _TD_STREAM_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_STREAM_H_*/

40
include/client/taos.h
View File

@ -26,9 +26,11 @@ extern "C" {
typedef void TAOS;
typedef void TAOS_STMT;
typedef void TAOS_RES;
typedef void TAOS_STREAM;
typedef void TAOS_SUB;
typedef void **TAOS_ROW;
#if 0
typedef void TAOS_STREAM;
#endif
typedef void TAOS_SUB;
// Data type definition
#define TSDB_DATA_TYPE_NULL 0 // 1 bytes
@ -149,6 +151,7 @@ DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt);
DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
@ -179,15 +182,20 @@ DLL_EXPORT int taos_errno(TAOS_RES *tres);
DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
// Shuduo: temporary enable for app build
#if 1
typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp,
void *param, int interval);
DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
#endif
#if 0
DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
int64_t stime, void *param, void (*callback)(void *));
DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
#endif
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
@ -215,8 +223,12 @@ DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
// will be removed in 3.0
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
// will replace last one
DLL_EXPORT tmq_t *tmq_consumer_new1(tmq_conf_t *conf, char *errstr, int32_t errstrLen);
DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
/* ------------------------TMQ CONSUMER INTERFACE------------------------ */
@ -255,20 +267,26 @@ int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message);
DLL_EXPORT void *tmq_get_topic_schema(tmq_t *tmq, const char *topic);
DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message);
DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
DLL_EXPORT char *tmq_get_topic_name(tmq_message_t *message);
DLL_EXPORT int32_t tmq_get_vgroup_id(tmq_message_t *message);
DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message);
DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message);
DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
DLL_EXPORT int32_t tmq_field_count(tmq_t *tmq, const char *topic);
DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message);
/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
#if 0
DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen);
#endif
DLL_EXPORT TAOS_RES *tmq_create_stream(TAOS *taos, const char *streamName, const char *tbName, const char *sql);
/* -------------------------------- OTHER -------------------------------- */
/* ------------------------------ TMQ END -------------------------------- */
#if 1 // Shuduo: temporary enable for app build
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code);
DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt);
#endif
#ifdef __cplusplus
}

5
include/common/tcommon.h
View File

@ -197,6 +197,11 @@ typedef struct SGroupbyExpr {
bool groupbyTag; // group by tag or column
} SGroupbyExpr;
enum {
FUNC_PARAM_TYPE_VALUE = 0,
FUNC_PARAM_TYPE_COLUMN,
};
typedef struct SFunctParam {
int32_t type;
SColumn* pCol;

5
include/common/tdatablock.h
View File

@ -29,8 +29,9 @@ typedef struct SCorEpSet {
} SCorEpSet;
typedef struct SBlockOrderInfo {
bool nullFirst;
int32_t order;
int32_t colIndex;
int32_t slotId;
SColumnInfoData* pColData;
} SBlockOrderInfo;
@ -176,7 +177,7 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock);
SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols);
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst);
int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRows);

56
include/common/tmsg.h
View File

@ -2215,23 +2215,6 @@ static FORCE_INLINE void* tDecodeTSmaWrapper(void* buf, STSmaWrapper* pSW) {
return buf;
}
typedef struct {
int64_t uid;
int32_t numOfRows;
char* colData;
} SMqTbData;
typedef struct {
char topicName[TSDB_TOPIC_FNAME_LEN];
int64_t committedOffset;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t bodyLen;
int32_t numOfTb;
SMqTbData* tbData;
} SMqTopicData;
typedef struct {
int8_t mqMsgType;
int32_t code;
@ -2259,8 +2242,11 @@ typedef struct {
} SMqSubVgEp;
typedef struct {
char topic[TSDB_TOPIC_FNAME_LEN];
SArray* vgs; // SArray<SMqSubVgEp>
char topic[TSDB_TOPIC_FNAME_LEN];
int8_t isSchemaAdaptive;
SArray* vgs; // SArray<SMqSubVgEp>
int32_t numOfFields;
TAOS_FIELD* fields;
} SMqSubTopicEp;
typedef struct {
@ -2281,32 +2267,6 @@ typedef struct {
SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp;
typedef struct {
int32_t curBlock;
int32_t curRow;
void** uData;
} SMqRowIter;
struct tmq_message_t {
SMqPollRsp msg;
void* vg;
SMqRowIter iter;
};
#if 0
struct tmq_message_t {
SMqRspHead head;
union {
SMqPollRsp consumeRsp;
SMqCMGetSubEpRsp getEpRsp;
};
void* extra;
int32_t curBlock;
int32_t curRow;
void** uData;
};
#endif
static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); }
static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) {
@ -2331,17 +2291,21 @@ static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) {
static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pTopicEp->topic);
tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive);
int32_t sz = taosArrayGetSize(pTopicEp->vgs);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i);
tlen += tEncodeSMqSubVgEp(buf, pVgEp);
}
tlen += taosEncodeFixedI32(buf, pTopicEp->numOfFields);
// tlen += taosEncodeBinary(buf, pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
return tlen;
}
static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) {
buf = taosDecodeStringTo(buf, pTopicEp->topic);
buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp));
@ -2353,6 +2317,8 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
buf = tDecodeSMqSubVgEp(buf, &vgEp);
taosArrayPush(pTopicEp->vgs, &vgEp);
}
buf = taosDecodeFixedI32(buf, &pTopicEp->numOfFields);
// buf = taosDecodeBinary(buf, (void**)&pTopicEp->fields, pTopicEp->numOfFields * sizeof(TAOS_FIELD));
return buf;
}

28
include/common/tmsgcb.h
View File

@ -25,6 +25,7 @@ extern "C" {
typedef struct SRpcMsg SRpcMsg;
typedef struct SEpSet SEpSet;
typedef struct SMgmtWrapper SMgmtWrapper;
typedef enum {
QUERY_QUEUE,
FETCH_QUEUE,
@ -38,24 +39,31 @@ typedef enum {
typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
typedef int32_t (*GetQueueSizeFp)(SMgmtWrapper* pWrapper, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, SEpSet* epSet, SRpcMsg* pReq);
typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, const SEpSet* epSet, SRpcMsg* pReq);
typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq);
typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, SRpcMsg* pRsp);
typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, const SRpcMsg* pRsp);
typedef void (*RegisterBrokenLinkArgFp)(SMgmtWrapper* pWrapper, SRpcMsg* pMsg);
typedef void (*ReleaseHandleFp)(SMgmtWrapper* pWrapper, void* handle, int8_t type);
typedef struct {
SMgmtWrapper* pWrapper;
PutToQueueFp queueFps[QUEUE_MAX];
GetQueueSizeFp qsizeFp;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
SMgmtWrapper* pWrapper;
PutToQueueFp queueFps[QUEUE_MAX];
GetQueueSizeFp qsizeFp;
SendReqFp sendReqFp;
SendMnodeReqFp sendMnodeReqFp;
SendRspFp sendRspFp;
RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
ReleaseHandleFp releaseHandleFp;
} SMsgCb;
void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb);
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq);
int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype);
int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq);
int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq);
int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq);
void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp);
void tmsgSendRsp(const SRpcMsg* pRsp);
void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg);
void tmsgReleaseHandle(void* handle, int8_t type);
#ifdef __cplusplus
}

24
include/common/ttokendef.h
View File

@ -139,18 +139,18 @@
#define TK_TOPIC 121
#define TK_AS 122
#define TK_NK_BOOL 123
#define TK_NK_VARIABLE 124
#define TK_NK_UNDERLINE 125
#define TK_ROWTS 126
#define TK_TBNAME 127
#define TK_QSTARTTS 128
#define TK_QENDTS 129
#define TK_WSTARTTS 130
#define TK_WENDTS 131
#define TK_WDURATION 132
#define TK_BETWEEN 133
#define TK_IS 134
#define TK_NULL 135
#define TK_NULL 124
#define TK_NK_VARIABLE 125
#define TK_NK_UNDERLINE 126
#define TK_ROWTS 127
#define TK_TBNAME 128
#define TK_QSTARTTS 129
#define TK_QENDTS 130
#define TK_WSTARTTS 131
#define TK_WENDTS 132
#define TK_WDURATION 133
#define TK_BETWEEN 134
#define TK_IS 135
#define TK_NK_LT 136
#define TK_NK_GT 137
#define TK_NK_LE 138

76
include/libs/executor/executor.h
View File

@ -20,8 +20,8 @@
extern "C" {
#endif
#include "tcommon.h"
#include "query.h"
#include "tcommon.h"
typedef void* qTaskInfo_t;
typedef void* DataSinkHandle;
@ -36,7 +36,7 @@ typedef struct SReadHandle {
#define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1
#define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2
/**
* Create the exec task for streaming mode
* @param pMsg
@ -54,6 +54,16 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle);
*/
int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type);
/**
* Set multiple input data blocks for the stream scan.
* @param tinfo
* @param pBlocks
* @param numOfInputBlock
* @param type
* @return
*/
int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type);
/**
* Update the table id list, add or remove.
*
@ -86,16 +96,6 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
*/
int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t* useconds);
/**
* Retrieve the produced results information, if current query is not paused or completed,
* this function will be blocked to wait for the query execution completed or paused,
* in which case enough results have been produced already.
*
* @param tinfo
* @return
*/
int32_t qRetrieveQueryResultInfo(qTaskInfo_t tinfo, bool* buildRes, void* pRspContext);
/**
* kill the ongoing query and free the query handle and corresponding resources automatically
* @param tinfo qhandle
@ -158,50 +158,6 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t
*/
int32_t qUpdateQueriedTableIdList(qTaskInfo_t tinfo, int64_t uid, int32_t type);
//================================================================================================
// query handle management
/**
* Query handle mgmt object
* @param vgId
* @return
*/
void* qOpenTaskMgmt(int32_t vgId);
/**
* broadcast the close information and wait for all query stop.
* @param pExecutor
*/
void qTaskMgmtNotifyClosing(void* pExecutor);
/**
* Re-open the query handle management module when opening the vnode again.
* @param pExecutor
*/
void qQueryMgmtReOpen(void* pExecutor);
/**
* Close query mgmt and clean up resources.
* @param pExecutor
*/
void qCleanupTaskMgmt(void* pExecutor);
/**
* Add the query into the query mgmt object
* @param pMgmt
* @param qId
* @param qInfo
* @return
*/
void** qRegisterTask(void* pMgmt, uint64_t qId, void* qInfo);
/**
* acquire the query handle according to the key from query mgmt object.
* @param pMgmt
* @param key
* @return
*/
void** qAcquireTask(void* pMgmt, uint64_t key);
/**
* release the query handle and decrease the reference count in cache
* @param pMgmt
@ -211,14 +167,6 @@ void** qAcquireTask(void* pMgmt, uint64_t key);
*/
void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle);
/**
* De-register the query handle from the management module and free it immediately.
* @param pMgmt
* @param pQInfo
* @return
*/
void** qDeregisterQInfo(void* pMgmt, void* pQInfo);
void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet);
#ifdef __cplusplus

1
include/libs/nodes/nodes.h
View File

@ -121,6 +121,7 @@ typedef enum ENodeType {
QUERY_NODE_LOGIC_PLAN_EXCHANGE,
QUERY_NODE_LOGIC_PLAN_WINDOW,
QUERY_NODE_LOGIC_PLAN_SORT,
QUERY_NODE_LOGIC_PLAN_PARTITION,
QUERY_NODE_LOGIC_SUBPLAN,
QUERY_NODE_LOGIC_PLAN,

25
include/libs/nodes/plannodes.h
View File

@ -66,7 +66,11 @@ typedef struct SAggLogicNode {
typedef struct SProjectLogicNode {
SLogicNode node;
SNodeList* pProjections;
char stmtName[TSDB_TABLE_NAME_LEN];
char stmtName[TSDB_TABLE_NAME_LEN];
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
} SProjectLogicNode;
typedef struct SVnodeModifLogicNode {
@ -106,6 +110,11 @@ typedef struct SSortLogicNode {
SNodeList* pSortKeys;
} SSortLogicNode;
typedef struct SPartitionLogicNode {
SLogicNode node;
SNodeList* pPartitionKeys;
} SPartitionLogicNode;
typedef enum ESubplanType {
SUBPLAN_TYPE_MERGE = 1,
SUBPLAN_TYPE_PARTIAL,
@ -150,7 +159,8 @@ typedef struct SDataBlockDescNode {
ENodeType type;
int16_t dataBlockId;
SNodeList* pSlots;
int32_t resultRowSize;
int32_t totalRowSize;
int32_t outputRowSize;
int16_t precision;
} SDataBlockDescNode;
@ -195,6 +205,10 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode;
typedef struct SProjectPhysiNode {
SPhysiNode node;
SNodeList* pProjections;
int64_t limit;
int64_t offset;
int64_t slimit;
int64_t soffset;
} SProjectPhysiNode;
typedef struct SJoinPhysiNode {
@ -283,10 +297,17 @@ typedef struct SSubplan {
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
} SSubplan;
typedef enum EQueryMode {
QUERY_MODE_NORMAL = 1,
QUERY_MODE_EXPLAIN,
QUERY_MODE_EXPLAIN_AN
} EQueryMode;
typedef struct SQueryPlan {
ENodeType type;
uint64_t queryId;
int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
} SQueryPlan;

4
include/libs/nodes/querynodes.h
View File

@ -180,8 +180,8 @@ typedef struct SOrderByExprNode {
typedef struct SLimitNode {
ENodeType type; // QUERY_NODE_LIMIT
uint64_t limit;
uint64_t offset;
int64_t limit;
int64_t offset;
} SLimitNode;
typedef struct SStateWindowNode {

5
include/libs/stream/tstream.h
View File

@ -30,6 +30,11 @@ enum {
STREAM_TASK_STATUS__STOP,
};
enum {
STREAM_CREATED_BY__USER = 1,
STREAM_CREATED_BY__SMA,
};
#if 0
// pipe -> fetch/pipe queue
// merge -> merge queue

30
include/libs/transport/trpc.h
View File

@ -49,11 +49,13 @@ typedef struct SRpcMsg {
} SRpcMsg;
typedef struct {
char user[TSDB_USER_LEN];
SRpcMsg rpcMsg;
int32_t rspLen;
void * pRsp;
void * pNode;
char user[TSDB_USER_LEN];
uint32_t clientIp;
uint16_t clientPort;
SRpcMsg rpcMsg;
int32_t rspLen;
void *pRsp;
void *pNode;
} SNodeMsg;
typedef struct SRpcInit {
@ -105,20 +107,20 @@ void rpcClose(void *);
void * rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void * rpcReallocCont(void *ptr, int contLen);
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
// Because taosd supports multi-process mode
// These functions should not be used on the server side
// Please use tmsg<xx> functions, which are defined in tmsgcb.h
void rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid);
void rpcSendResponse(const SRpcMsg *pMsg);
void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
void rpcReleaseHandle(void *handle, int8_t type); // just release client conn to rpc instance, no close sock
// These functions will not be called in the child process
void rpcSendRedirectRsp(void *pConn, const SEpSet *pEpSet);
void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx);
int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo);
void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
int rpcReportProgress(void *pConn, char *pCont, int contLen);
void rpcCancelRequest(int64_t rid);
void rpcRegisterBrokenLinkArg(SRpcMsg *msg);
// just release client conn to rpc instance, no close sock
void rpcReleaseHandle(void *handle, int8_t type); //
void rpcRefHandle(void *handle, int8_t type);
void rpcUnrefHandle(void *handle, int8_t type);
#ifdef __cplusplus
}

3
include/os/os.h
View File

@ -42,6 +42,9 @@ extern "C" {
#include <sys/types.h>
#include <termios.h>
#include <sys/statvfs.h>
#include <sys/prctl.h>
#include <sys/shm.h>
#include <sys/wait.h>
#if defined(DARWIN)
#else

2
include/util/taoserror.h
View File

@ -16,7 +16,7 @@
#ifndef _TD_UTIL_TAOS_ERROR_H_
#define _TD_UTIL_TAOS_ERROR_H_
#include "os.h"
#include <stdint.h>
#ifdef __cplusplus
extern "C" {

12
include/util/tprocess.h
View File

@ -22,11 +22,14 @@
extern "C" {
#endif
typedef enum { PROC_REQ, PROC_RSP, PROC_REG, PROC_RELEASE } ProcFuncType;
typedef struct SProcQueue SProcQueue;
typedef struct SProcObj SProcObj;
typedef void *(*ProcMallocFp)(int32_t contLen);
typedef void *(*ProcFreeFp)(void *pCont);
typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void *pBody, int32_t bodyLen,
ProcFuncType ftype);
typedef struct {
int32_t childQueueSize;
@ -52,9 +55,10 @@ int32_t taosProcRun(SProcObj *pProc);
void taosProcStop(SProcObj *pProc);
bool taosProcIsChild(SProcObj *pProc);
int32_t taosProcChildId(SProcObj *pProc);
int32_t taosProcPutToChildQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
int32_t taosProcPutToParentQueue(SProcObj *pProc, void *pHead, int32_t headLen, void *pBody, int32_t bodyLen);
int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
ProcFuncType ftype);
int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
ProcFuncType ftype);
#ifdef __cplusplus
}

236
source/client/src/clientMain.c
View File

@ -1,12 +1,27 @@
#include "os.h"
#include "tref.h"
#include "trpc.h"
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "query.h"
#include "tmsg.h"
#include "tglobal.h"
#include "catalog.h"
#include "tmsg.h"
#include "tref.h"
#include "trpc.h"
#include "version.h"
#define TSC_VAR_NOT_RELEASE 1
@ -24,7 +39,7 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) {
}
}
int ret = taos_options_imp(option, (const char*)arg);
int ret = taos_options_imp(option, (const char *)arg);
atomic_store_32(&lock, 0);
return ret;
}
@ -69,13 +84,13 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha
return taos_connect_internal(ip, user, pass, NULL, db, port);
}
void taos_close(TAOS* taos) {
void taos_close(TAOS *taos) {
if (taos == NULL) {
return;
}
STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
/*taosRemoveRef(clientConnRefPool, pTscObj->id);*/
}
@ -85,48 +100,46 @@ int taos_errno(TAOS_RES *tres) {
return terrno;
}
return ((SRequestObj*) tres)->code;
return ((SRequestObj *)tres)->code;
}
const char *taos_errstr(TAOS_RES *res) {
SRequestObj *pRequest = (SRequestObj *) res;
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest == NULL) {
return (const char*) tstrerror(terrno);
return (const char *)tstrerror(terrno);
}
if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) {
return pRequest->msgBuf;
} else {
return (const char*)tstrerror(pRequest->code);
return (const char *)tstrerror(pRequest->code);
}
}
void taos_free_result(TAOS_RES *res) {
SRequestObj* pRequest = (SRequestObj*) res;
SRequestObj *pRequest = (SRequestObj *)res;
destroyRequest(pRequest);
}
int taos_field_count(TAOS_RES *res) {
int taos_field_count(TAOS_RES *res) {
if (res == NULL) {
return 0;
}
SRequestObj* pRequest = (SRequestObj*) res;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SRequestObj *pRequest = (SRequestObj *)res;
SReqResultInfo *pResInfo = &pRequest->body.resInfo;
return pResInfo->numOfCols;
}
int taos_num_fields(TAOS_RES *res) {
return taos_field_count(res);
}
int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); }
TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
if (taos_num_fields(res) == 0) {
return NULL;
}
SReqResultInfo* pResInfo = &(((SRequestObj*) res)->body.resInfo);
SReqResultInfo *pResInfo = &(((SRequestObj *)res)->body.resInfo);
return pResInfo->fields;
}
@ -135,7 +148,7 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) {
return NULL;
}
return taos_query_l(taos, sql, (int32_t) strlen(sql));
return taos_query_l(taos, sql, (int32_t)strlen(sql));
}
TAOS_ROW taos_fetch_row(TAOS_RES *res) {
@ -143,18 +156,16 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) {
return NULL;
}
SRequestObj *pRequest = (SRequestObj *) res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS ||
taos_num_fields(res) == 0) {
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
return NULL;
}
return doFetchRow(pRequest, true);
}
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
int32_t len = 0;
for (int i = 0; i < num_fields; ++i) {
if (i > 0) {
@ -213,7 +224,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE);
int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY) {
assert(charLen <= fields[i].bytes && charLen >= 0);
} else {
@ -238,31 +249,44 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields)
return len;
}
int* taos_fetch_lengths(TAOS_RES *res) {
int *taos_fetch_lengths(TAOS_RES *res) {
if (res == NULL) {
return NULL;
}
return ((SRequestObj*) res)->body.resInfo.length;
return ((SRequestObj *)res)->body.resInfo.length;
}
// todo intergrate with tDataTypes
const char *taos_data_type(int type) {
switch (type) {
case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL";
case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL";
case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT";
case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT";
case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT";
case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT";
case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT";
case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE";
case TSDB_DATA_TYPE_VARCHAR: return "TSDB_DATA_TYPE_VARCHAR";
// case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_VARCHAR";
case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP";
case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR";
case TSDB_DATA_TYPE_JSON: return "TSDB_DATA_TYPE_JSON";
default: return "UNKNOWN";
case TSDB_DATA_TYPE_NULL:
return "TSDB_DATA_TYPE_NULL";
case TSDB_DATA_TYPE_BOOL:
return "TSDB_DATA_TYPE_BOOL";
case TSDB_DATA_TYPE_TINYINT:
return "TSDB_DATA_TYPE_TINYINT";
case TSDB_DATA_TYPE_SMALLINT:
return "TSDB_DATA_TYPE_SMALLINT";
case TSDB_DATA_TYPE_INT:
return "TSDB_DATA_TYPE_INT";
case TSDB_DATA_TYPE_BIGINT:
return "TSDB_DATA_TYPE_BIGINT";
case TSDB_DATA_TYPE_FLOAT:
return "TSDB_DATA_TYPE_FLOAT";
case TSDB_DATA_TYPE_DOUBLE:
return "TSDB_DATA_TYPE_DOUBLE";
case TSDB_DATA_TYPE_VARCHAR:
return "TSDB_DATA_TYPE_VARCHAR";
// case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_VARCHAR";
case TSDB_DATA_TYPE_TIMESTAMP:
return "TSDB_DATA_TYPE_TIMESTAMP";
case TSDB_DATA_TYPE_NCHAR:
return "TSDB_DATA_TYPE_NCHAR";
case TSDB_DATA_TYPE_JSON:
return "TSDB_DATA_TYPE_JSON";
default:
return "UNKNOWN";
}
}
@ -273,8 +297,8 @@ int taos_affected_rows(TAOS_RES *res) {
return 0;
}
SRequestObj* pRequest = (SRequestObj*) res;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SRequestObj *pRequest = (SRequestObj *)res;
SReqResultInfo *pResInfo = &pRequest->body.resInfo;
return pResInfo->numOfRows;
}
@ -295,8 +319,8 @@ int taos_select_db(TAOS *taos, const char *db) {
char sql[256] = {0};
snprintf(sql, tListLen(sql), "use %s", db);
TAOS_RES* pRequest = taos_query(taos, sql);
int32_t code = taos_errno(pRequest);
TAOS_RES *pRequest = taos_query(taos, sql);
int32_t code = taos_errno(pRequest);
taos_free_result(pRequest);
return code;
@ -307,61 +331,57 @@ void taos_stop_query(TAOS_RES *res) {
return;
}
SRequestObj* pRequest = (SRequestObj*) res;
int32_t numOfFields = taos_num_fields(pRequest);
SRequestObj *pRequest = (SRequestObj *)res;
int32_t numOfFields = taos_num_fields(pRequest);
// It is not a query, no need to stop.
if (numOfFields == 0) {
return;
}
// scheduleCancelJob(pRequest->body.pQueryJob);
// scheduleCancelJob(pRequest->body.pQueryJob);
}
bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) {
SRequestObj* pRequestObj = res;
SReqResultInfo* pResultInfo = &pRequestObj->body.resInfo;
SRequestObj *pRequestObj = res;
SReqResultInfo *pResultInfo = &pRequestObj->body.resInfo;
if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) {
return true;
}
SResultColumn* pCol = &pRequestObj->body.resInfo.pCol[col];
SResultColumn *pCol = &pRequestObj->body.resInfo.pCol[col];
return colDataIsNull_f(pCol->nullbitmap, row);
}
int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) {
if (res == NULL) {
return 0;
}
SRequestObj *pRequest = (SRequestObj *) res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS ||
taos_num_fields(res) == 0) {
SRequestObj *pRequest = (SRequestObj *)res;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || pRequest->type == TSDB_SQL_INSERT ||
pRequest->code != TSDB_CODE_SUCCESS || taos_num_fields(res) == 0) {
return 0;
}
doFetchRow(pRequest, false);
// TODO refactor
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
SReqResultInfo *pResultInfo = &pRequest->body.resInfo;
pResultInfo->current = pResultInfo->numOfRows;
*rows = pResultInfo->row;
return pResultInfo->numOfRows;
}
int taos_validate_sql(TAOS *taos, const char *sql) {
return true;
}
int taos_validate_sql(TAOS *taos, const char *sql) { return true; }
const char *taos_get_server_info(TAOS *taos) {
if (taos == NULL) {
return NULL;
}
STscObj* pTscObj = (STscObj*) taos;
STscObj *pTscObj = (STscObj *)taos;
return pTscObj->ver;
}
@ -387,52 +407,67 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) {
// TODO
}
TAOS_STMT* taos_stmt_init(TAOS* taos) {
// TODO
return NULL;
int taos_load_table_info(TAOS *taos, const char *tableNameList) {
// TODO
return -1;
}
int taos_stmt_close(TAOS_STMT* stmt) {
// TODO
return -1;
TAOS_STMT *taos_stmt_init(TAOS *taos) {
// TODO
return NULL;
}
int taos_stmt_execute(TAOS_STMT* stmt) {
// TODO
return -1;
int taos_stmt_close(TAOS_STMT *stmt) {
// TODO
return -1;
}
int taos_stmt_execute(TAOS_STMT *stmt) {
// TODO
return -1;
}
char *taos_stmt_errstr(TAOS_STMT *stmt) {
// TODO
return NULL;
// TODO
return NULL;
}
int taos_stmt_affected_rows(TAOS_STMT* stmt) {
int taos_stmt_affected_rows(TAOS_STMT *stmt) {
// TODO
return -1;
}
TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision) {
// TODO
return NULL;
}
int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind) {
// TODO
return -1;
}
int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
// TODO
return -1;
}
int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags) {
// TODO
return -1;
}
int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
// TODO
return -1;
}
int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert) {
// TODO
return -1;
}
TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) {
// TODO
return NULL;
}
int taos_stmt_bind_param(TAOS_STMT* stmt, TAOS_BIND* bind) {
// TODO
return -1;
}
int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
// TODO
return -1;
}
int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags) {
// TODO
return -1;
}
int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name) {
int taos_stmt_num_params(TAOS_STMT *stmt, int *nums) {
// TODO
return -1;
}
@ -442,6 +477,11 @@ int taos_stmt_add_batch(TAOS_STMT* stmt) {
return -1;
}
TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
// TODO
return NULL;
}
int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind) {
// TODO
return -1;

40
source/client/src/tmq.c
View File

@ -26,6 +26,18 @@
#include "tqueue.h"
#include "tref.h"
typedef struct {
int32_t curBlock;
int32_t curRow;
void** uData;
} SMqRowIter;
struct tmq_message_t {
SMqPollRsp msg;
void* vg;
SMqRowIter iter;
};
struct tmq_list_t {
SArray container;
};
@ -99,13 +111,14 @@ typedef struct {
typedef struct {
// subscribe info
int32_t sqlLen;
char* sql;
char* topicName;
int64_t topicId;
int32_t nextVgIdx;
SArray* vgs; // SArray<SMqClientVg>
SSchemaWrapper schema;
int32_t sqlLen;
char* sql;
char* topicName;
int64_t topicId;
SArray* vgs; // SArray<SMqClientVg>
int8_t isSchemaAdaptive;
int32_t numOfFields;
TAOS_FIELD* fields;
} SMqClientTopic;
typedef struct {
@ -130,11 +143,11 @@ typedef struct {
} SMqPollCbParam;
typedef struct {
tmq_t* tmq;
/*SMqClientVg* pVg;*/
tmq_t* tmq;
int32_t async;
tsem_t rspSem;
tmq_resp_err_t rspErr;
/*SMqClientVg* pVg;*/
} SMqCommitCbParam;
tmq_conf_t* tmq_conf_new() {
@ -471,7 +484,12 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
tNameExtractFullName(&name, topicFname);
tscDebug("subscribe topic: %s", topicFname);
SMqClientTopic topic = {
.nextVgIdx = 0, .sql = NULL, .sqlLen = 0, .topicId = 0, .topicName = topicFname, .vgs = NULL};
.sql = NULL,
.sqlLen = 0,
.topicId = 0,
.topicName = topicFname,
.vgs = NULL,
};
topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
taosArrayPush(tmq->clientTopics, &topic);
taosArrayPush(req.topicNames, &topicFname);
@ -615,6 +633,7 @@ _return:
return pRequest;
}
#if 0
TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos;
SRequestObj* pRequest = NULL;
@ -700,6 +719,7 @@ _return:
return pRequest;
}
#endif
static char* formatTimestamp(char* buf, int64_t val, int precision) {
time_t tt;

14
source/client/test/tmqTest.cpp
View File

@ -35,7 +35,7 @@ int main(int argc, char** argv) {
TEST(testCase, driverInit_Test) {
// taosInitGlobalCfg();
// taos_init();
// taos_init();
}
TEST(testCase, create_topic_ctb_Test) {
@ -55,8 +55,9 @@ TEST(testCase, create_topic_ctb_Test) {
taos_free_result(pRes);
char* sql = "select * from tu";
pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
// char* sql = "select * from tu";
// pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql));
pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from tu");
taos_free_result(pRes);
taos_close(pConn);
}
@ -69,7 +70,7 @@ TEST(testCase, create_topic_stb_Test) {
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
}
//taos_free_result(pRes);
// taos_free_result(pRes);
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == nullptr);
@ -79,8 +80,9 @@ TEST(testCase, create_topic_stb_Test) {
taos_free_result(pRes);
char* sql = "select * from st1";
pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
// char* sql = "select * from st1";
// pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));
pRes = taos_query(pConn, "create test_ctb_topic_1 as select * from st1");
taos_free_result(pRes);
taos_close(pConn);
}

17
source/common/src/tdatablock.c
View File

@ -647,7 +647,6 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
typedef struct SSDataBlockSortHelper {
SArray* orderInfo; // SArray<SBlockOrderInfo>
SSDataBlock* pDataBlock;
bool nullFirst;
} SSDataBlockSortHelper;
int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
@ -672,11 +671,11 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
}
if (rightNull) {
return pHelper->nullFirst ? 1 : -1;
return pOrder->nullFirst ? 1 : -1;
}
if (leftNull) {
return pHelper->nullFirst ? -1 : 1;
return pOrder->nullFirst ? -1 : 1;
}
}
@ -907,7 +906,7 @@ static __compar_fn_t getComparFn(int32_t type, int32_t order) {
}
}
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) {
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
ASSERT(pDataBlock != NULL && pOrderInfo != NULL);
if (pDataBlock->info.rows <= 1) {
return TSDB_CODE_SUCCESS;
@ -922,7 +921,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) {
SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i);
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex);
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
if (pColInfoData->hasNull) {
sortColumnHasNull = true;
}
@ -961,10 +960,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs
int64_t p0 = taosGetTimestampUs();
SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) {
struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i);
pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex);
pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
}
taosqsort(index, rows, sizeof(int32_t), &helper, dataBlockCompar);
@ -1012,7 +1011,7 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock*
for (int32_t i = 0; i < numOfCols; ++i) {
SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->slotId);
pInfo->pColData = pColInfo;
sortValLengthPerRow += pColInfo->info.bytes;
}
@ -1106,7 +1105,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF
// Allocate the additional buffer.
int64_t p0 = taosGetTimestampUs();
SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
uint32_t rows = pDataBlock->info.rows;
SHelper* index = createTupleIndex_rv(rows, helper.orderInfo, pDataBlock);

16
source/common/src/tmsgcb.c
View File

@ -16,6 +16,10 @@
#define _DEFAULT_SOURCE
#include "tmsgcb.h"
static SMsgCb tsDefaultMsgCb;
void tmsgSetDefaultMsgCb(const SMsgCb* pMsgCb) { tsDefaultMsgCb = *pMsgCb; }
int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) {
return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq);
}
@ -24,7 +28,7 @@ int32_t tmsgGetQueueSize(const SMsgCb* pMsgCb, int32_t vgId, EQueueType qtype) {
return (*pMsgCb->qsizeFp)(pMsgCb->pWrapper, vgId, qtype);
}
int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq) {
int32_t tmsgSendReq(const SMsgCb* pMsgCb, const SEpSet* epSet, SRpcMsg* pReq) {
return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq);
}
@ -32,4 +36,12 @@ int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) {
return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq);
}
void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); }
void tmsgSendRsp(const SRpcMsg* pRsp) { return (*tsDefaultMsgCb.sendRspFp)(tsDefaultMsgCb.pWrapper, pRsp); }
void tmsgRegisterBrokenLinkArg(const SMsgCb* pMsgCb, SRpcMsg* pMsg) {
(*pMsgCb->registerBrokenLinkArgFp)(pMsgCb->pWrapper, pMsg);
}
void tmsgReleaseHandle(void* handle, int8_t type) {
(*tsDefaultMsgCb.releaseHandleFp)(tsDefaultMsgCb.pWrapper, handle, type);
}

4
source/common/test/commonTests.cpp
View File

@ -167,10 +167,10 @@ TEST(testCase, Datablock_test) {
printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));
SBlockOrderInfo order = {.order = TSDB_ORDER_ASC, .colIndex = 0};
SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0};
taosArrayPush(pOrderInfo, &order);
blockDataSort(b, pOrderInfo, true);
blockDataSort(b, pOrderInfo);
blockDataDestroy(b);
taosArrayDestroy(pOrderInfo);

38
source/dnode/mgmt/CMakeLists.txt
View File

@ -1,12 +1,10 @@
aux_source_directory(src DNODE_SRC)
aux_source_directory(dnode/src DNODE_SRC)
aux_source_directory(qnode/src DNODE_SRC)
aux_source_directory(bnode/src DNODE_SRC)
aux_source_directory(snode/src DNODE_SRC)
aux_source_directory(vnode/src DNODE_SRC)
aux_source_directory(mnode/src DNODE_SRC)
aux_source_directory(container/src DNODE_SRC)
aux_source_directory(dm/src DNODE_SRC)
aux_source_directory(qm/src DNODE_SRC)
aux_source_directory(bm/src DNODE_SRC)
aux_source_directory(sm/src DNODE_SRC)
aux_source_directory(vm/src DNODE_SRC)
aux_source_directory(mm/src DNODE_SRC)
aux_source_directory(main/src DNODE_SRC)
add_library(dnode STATIC ${DNODE_SRC})
target_link_libraries(
dnode cjson mnode vnode qnode snode bnode wal sync taos tfs monitor
@ -14,16 +12,22 @@ target_link_libraries(
target_include_directories(
dnode
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dnode/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qnode/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bnode/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/snode/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vnode/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mnode/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/container/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/dm/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/qm/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/bm/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/sm/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/vm/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/mm/inc"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc"
)
add_subdirectory(main)
aux_source_directory(main/exe EXEC_SRC)
add_executable(taosd ${EXEC_SRC})
target_include_directories(
taosd
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/main/inc"
)
target_link_libraries(taosd dnode)
if(${BUILD_TEST})
add_subdirectory(test)

0
source/dnode/mgmt/bnode/inc/bm.h → source/dnode/mgmt/bm/inc/bm.h
View File

0
source/dnode/mgmt/bnode/inc/bmInt.h → source/dnode/mgmt/bm/inc/bmInt.h
View File

1
source/dnode/mgmt/bnode/src/bmInt.c → source/dnode/mgmt/bm/src/bmInt.c
View File

@ -24,6 +24,7 @@ static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

0
source/dnode/mgmt/bnode/src/bmMsg.c → source/dnode/mgmt/bm/src/bmMsg.c
View File

0
source/dnode/mgmt/bnode/src/bmWorker.c → source/dnode/mgmt/bm/src/bmWorker.c
View File

2
source/dnode/mgmt/dnode/inc/dm.h → source/dnode/mgmt/dm/inc/dm.h
View File

@ -29,7 +29,7 @@ void dmInitMsgHandles(SMgmtWrapper *pWrapper);
void dmGetMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
void dmUpdateMnodeEpSet(SDnodeMgmt *pMgmt, SEpSet *pEpSet);
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg);
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pMsg);
#ifdef __cplusplus
}

0
source/dnode/mgmt/dnode/inc/dmInt.h → source/dnode/mgmt/dm/inc/dmInt.h
View File

0
source/dnode/mgmt/dnode/src/dmFile.c → source/dnode/mgmt/dm/src/dmFile.c
View File

2
source/dnode/mgmt/dnode/src/dmInt.c → source/dnode/mgmt/dm/src/dmInt.c
View File

@ -54,7 +54,7 @@ void dmGetDnodeEp(SMgmtWrapper *pWrapper, int32_t dnodeId, char *pEp, char *pFqd
taosRUnLockLatch(&pMgmt->latch);
}
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, SRpcMsg *pReq) {
void dmSendRedirectRsp(SDnodeMgmt *pMgmt, const SRpcMsg *pReq) {
SDnode *pDnode = pMgmt->pDnode;
SEpSet epSet = {0};

0
source/dnode/mgmt/dnode/src/dmMsg.c → source/dnode/mgmt/dm/src/dmMsg.c
View File

0
source/dnode/mgmt/dnode/src/dmWorker.c → source/dnode/mgmt/dm/src/dmWorker.c
View File

9
source/dnode/mgmt/main/CMakeLists.txt
View File

@ -1,9 +0,0 @@
aux_source_directory(src EXEC_SRC)
add_executable(taosd ${EXEC_SRC})
target_include_directories(
taosd
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(taosd dnode)

199
source/dnode/mgmt/main/exe/dndMain.c Normal file
View File

@ -0,0 +1,199 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dnd.h"
#include "tconfig.h"
static struct {
bool dumpConfig;
bool generateGrant;
bool printAuth;
bool printVersion;
char envFile[PATH_MAX];
char apolloUrl[PATH_MAX];
SArray *pArgs; // SConfigPair
SDnode *pDnode;
ENodeType ntype;
} global = {0};
static void dndSigintHandle(int signum, void *info, void *ctx) {
dInfo("signal:%d is received", signum);
SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
if (pDnode != NULL) {
dndHandleEvent(pDnode, DND_EVENT_STOP);
}
}
static void dndSetSignalHandle() {
taosSetSignal(SIGTERM, dndSigintHandle);
taosSetSignal(SIGHUP, dndSigintHandle);
taosSetSignal(SIGINT, dndSigintHandle);
taosSetSignal(SIGABRT, dndSigintHandle);
taosSetSignal(SIGBREAK, dndSigintHandle);
}
static int32_t dndParseArgs(int32_t argc, char const *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
} else if (strcmp(argv[i], "-a") == 0) {
tstrncpy(global.apolloUrl, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-e") == 0) {
tstrncpy(global.envFile, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-n") == 0) {
global.ntype = atoi(argv[++i]);
} else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true;
} else if (strcmp(argv[i], "-V") == 0) {
global.printVersion = true;
} else {
}
}
return 0;
}
static void dndGenerateGrant() {
// grantParseParameter();
printf("this feature is not implemented yet\n");
}
static void dndPrintVersion() {
#ifdef TD_ENTERPRISE
char *releaseName = "enterprise";
#else
char *releaseName = "community";
#endif
printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
printf("gitinfo: %s\n", gitinfo);
printf("buildInfo: %s\n", buildinfo);
}
static void dndDumpCfg() {
SConfig *pCfg = taosGetCfg();
cfgDumpCfg(pCfg, 0, 1);
}
static SDnodeOpt dndGetOpt() {
SConfig *pCfg = taosGetCfg();
SDnodeOpt option = {0};
option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir));
tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp));
tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp));
option.serverPort = tsServerPort;
tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn));
snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort);
option.pDisks = tsDiskCfg;
option.numOfDisks = tsDiskCfgNum;
return option;
}
static int32_t dndInitLog() {
char logName[12] = {0};
snprintf(logName, sizeof(logName), "%slog", dndNodeLogStr(global.ntype));
return taosCreateLog(logName, 1, configDir, global.envFile, global.apolloUrl, global.pArgs, 0);
}
static void dndSetProcName(char **argv) {
if (global.ntype != 0) {
const char *name = dndNodeProcStr(global.ntype);
prctl(PR_SET_NAME, name);
strcpy(argv[0], name);
}
}
static int32_t dndRunDnode() {
if (dndInit() != 0) {
dError("failed to initialize environment since %s", terrstr());
return -1;
}
SDnodeOpt option = dndGetOpt();
SDnode *pDnode = dndCreate(&option);
if (pDnode == NULL) {
dError("failed to to create dnode object since %s", terrstr());
return -1;
} else {
global.pDnode = pDnode;
dndSetSignalHandle();
}
dInfo("start the service");
int32_t code = dndRun(pDnode);
dInfo("start shutting down the service");
global.pDnode = NULL;
dndClose(pDnode);
dndCleanup();
taosCloseLog();
taosCleanupCfg();
return code;
}
int main(int argc, char const *argv[]) {
if (!taosCheckSystemIsSmallEnd()) {
printf("failed to start since on non-small-end machines\n");
return -1;
}
if (dndParseArgs(argc, argv) != 0) {
printf("failed to start since parse args error\n");
return -1;
}
if (global.generateGrant) {
dndGenerateGrant();
return 0;
}
if (global.printVersion) {
dndPrintVersion();
return 0;
}
if (dndInitLog() != 0) {
printf("failed to start since init log error\n");
return -1;
}
if (taosInitCfg(configDir, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
dError("failed to start since read config error");
return -1;
}
if (global.dumpConfig) {
dndDumpCfg();
taosCleanupCfg();
taosCloseLog();
return 0;
}
dndSetProcName((char **)argv);
return dndRunDnode();
}

19
source/dnode/mgmt/container/inc/dnd.h → source/dnode/mgmt/main/inc/dnd.h
View File

@ -33,6 +33,7 @@
#include "tthread.h"
#include "ttime.h"
#include "tworker.h"
#include "tmsgcb.h"
#include "dnode.h"
#include "monitor.h"
@ -130,15 +131,19 @@ typedef struct SDnode {
SMgmtWrapper wrappers[NODE_MAX];
} SDnode;
EDndStatus dndGetStatus(SDnode *pDnode);
void dndSetStatus(SDnode *pDnode, EDndStatus stat);
void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId);
void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
void dndSendMonitorReport(SDnode *pDnode);
const char *dndNodeLogStr(ENodeType ntype);
const char *dndNodeProcStr(ENodeType ntype);
EDndStatus dndGetStatus(SDnode *pDnode);
void dndSetStatus(SDnode *pDnode, EDndStatus stat);
void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId);
void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
void dndSendMonitorReport(SDnode *pDnode);
int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pMsg);
void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp);
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg);
void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper);
int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg);
int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed);

1
source/dnode/mgmt/container/inc/dndInt.h → source/dnode/mgmt/main/inc/dndInt.h
View File

@ -56,6 +56,7 @@ void dndCleanupServer(SDnode *pDnode);
int32_t dndInitClient(SDnode *pDnode);
void dndCleanupClient(SDnode *pDnode);
int32_t dndInitMsgHandle(SDnode *pDnode);
void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp);
#ifdef __cplusplus
}

48
source/dnode/mgmt/main/inc/dndMain.h
View File

@ -1,48 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_DND_MAIN_H_
#define _TD_DND_MAIN_H_
#include "dnode.h"
#include "taoserror.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tlog.h"
#include "version.h"
#ifdef __cplusplus
extern "C" {
#endif
#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }}
#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }}
#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }}
#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }}
#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }}
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }}
void dndDumpCfg();
void dndPrintVersion();
void dndGenerateGrant();
SDnodeOpt dndGetOpt();
#ifdef __cplusplus
}
#endif
#endif /*_TD_DND_MAIN_H_*/

57
source/dnode/mgmt/container/src/dndExec.c → source/dnode/mgmt/main/src/dndExec.c
View File

@ -77,6 +77,8 @@ static int32_t dndRunInSingleProcess(SDnode *pDnode) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[n];
pWrapper->required = dndRequireNode(pWrapper);
if (!pWrapper->required) continue;
SMsgCb msgCb = dndCreateMsgcb(pWrapper);
tmsgSetDefaultMsgCb(&msgCb);
if (taosMkDir(pWrapper->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
@ -116,11 +118,11 @@ static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) {
}
}
static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t msgLen, void *pCont, int32_t contLen) {
static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
ProcFuncType ftype) {
SRpcMsg *pRpc = &pMsg->rpcMsg;
pRpc->pCont = pCont;
dTrace("msg:%p, get from child process queue, type:%s handle:%p app:%p", pMsg, TMSG_INFO(pRpc->msgType),
pRpc->handle, pRpc->ahandle);
dTrace("msg:%p, get from child queue, handle:%p app:%p", pMsg, pRpc->handle, pRpc->ahandle);
NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pRpc->msgType)];
int32_t code = (*msgFp)(pWrapper, pMsg);
@ -138,13 +140,27 @@ static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t
}
}
static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, int32_t msgLen, void *pCont, int32_t contLen) {
pRpc->pCont = pCont;
dTrace("msg:%p, get from parent process queue, type:%s handle:%p app:%p", pRpc, TMSG_INFO(pRpc->msgType),
pRpc->handle, pRpc->ahandle);
static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen,
ProcFuncType ftype) {
pMsg->pCont = pCont;
dTrace("msg:%p, get from parent queue, handle:%p app:%p", pMsg, pMsg->handle, pMsg->ahandle);
dndSendRsp(pWrapper, pRpc);
taosMemoryFree(pRpc);
switch (ftype) {
case PROC_REG:
rpcRegisterBrokenLinkArg(pMsg);
break;
case PROC_RELEASE:
rpcReleaseHandle(pMsg->handle, (int8_t)pMsg->code);
rpcFreeCont(pCont);
break;
case PROC_REQ:
// todo send to dnode
dndSendReqToMnode(pWrapper, pMsg);
default:
dndSendRpcRsp(pWrapper, pMsg);
break;
}
taosMemoryFree(pMsg);
}
static int32_t dndRunInMultiProcess(SDnode *pDnode) {
@ -155,6 +171,9 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) {
pWrapper->required = dndRequireNode(pWrapper);
if (!pWrapper->required) continue;
SMsgCb msgCb = dndCreateMsgcb(pWrapper);
tmsgSetDefaultMsgCb(&msgCb);
if (taosMkDir(pWrapper->path) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to create dir:%s since %s", pWrapper->path, terrstr());
@ -202,15 +221,23 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) {
dndClearNodesExecpt(pDnode, n);
dInfo("node:%s, will be initialized in child process", pWrapper->name);
dndOpenNode(pWrapper);
if (dndOpenNode(pWrapper) != 0) {
dInfo("node:%s, failed to init in child process since %s", pWrapper->name, terrstr());
return -1;
}
if (taosProcRun(pProc) != 0) {
dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
return -1;
}
break;
} else {
dInfo("node:%s, will not start in parent process, child pid:%d", pWrapper->name, taosProcChildId(pProc));
pWrapper->procType = PROC_PARENT;
}
if (taosProcRun(pProc) != 0) {
dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
return -1;
if (taosProcRun(pProc) != 0) {
dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr());
return -1;
}
}
}

0
source/dnode/mgmt/container/src/dndFile.c → source/dnode/mgmt/main/src/dndFile.c
View File

13
source/dnode/mgmt/container/src/dndInt.c → source/dnode/mgmt/main/src/dndInt.c
View File

@ -80,19 +80,6 @@ void dndSetStatus(SDnode *pDnode, EDndStatus status) {
}
}
const char *dndStatStr(EDndStatus status) {
switch (status) {
case DND_STAT_INIT:
return "init";
case DND_STAT_RUNNING:
return "running";
case DND_STAT_STOPPED:
return "stopped";
default:
return "unknown";
}
}
void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc) {
SStartupReq *pStartup = &pDnode->startup;
tstrncpy(pStartup->name, pName, TSDB_STEP_NAME_LEN);

138
source/dnode/mgmt/main/src/dndMain.c
View File

@ -1,138 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndMain.h"
static struct {
bool dumpConfig;
bool generateGrant;
bool printAuth;
bool printVersion;
char envFile[PATH_MAX];
char apolloUrl[PATH_MAX];
SDnode *pDnode;
} global = {0};
static void dndSigintHandle(int signum, void *info, void *ctx) {
dInfo("signal:%d is received", signum);
SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode);
if (pDnode != NULL) {
dndHandleEvent(pDnode, DND_EVENT_STOP);
}
}
static void dndSetSignalHandle() {
taosSetSignal(SIGTERM, dndSigintHandle);
taosSetSignal(SIGHUP, dndSigintHandle);
taosSetSignal(SIGINT, dndSigintHandle);
taosSetSignal(SIGABRT, dndSigintHandle);
taosSetSignal(SIGBREAK, dndSigintHandle);
}
static int32_t dndParseOption(int32_t argc, char const *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
} else if (strcmp(argv[i], "-C") == 0) {
global.dumpConfig = true;
} else if (strcmp(argv[i], "-k") == 0) {
global.generateGrant = true;
} else if (strcmp(argv[i], "-V") == 0) {
global.printVersion = true;
} else {
}
}
return 0;
}
static int32_t dndRunDnode() {
if (dndInit() != 0) {
dInfo("failed to initialize dnode environment since %s", terrstr());
return -1;
}
SDnodeOpt option = dndGetOpt();
SDnode *pDnode = dndCreate(&option);
if (pDnode == NULL) {
dError("failed to to create dnode object since %s", terrstr());
return -1;
} else {
global.pDnode = pDnode;
dndSetSignalHandle();
}
dInfo("start the TDengine service");
int32_t code = dndRun(pDnode);
dInfo("start shutting down the TDengine service");
global.pDnode = NULL;
dndClose(pDnode);
dndCleanup();
taosCloseLog();
taosCleanupCfg();
return code;
}
int main(int argc, char const *argv[]) {
if (!taosCheckSystemIsSmallEnd()) {
dError("failed to start TDengine since on non-small-end machines");
return -1;
}
if (dndParseOption(argc, argv) != 0) {
return -1;
}
if (global.generateGrant) {
dndGenerateGrant();
return 0;
}
if (global.printVersion) {
dndPrintVersion();
return 0;
}
if (taosCreateLog("taosdlog", 1, configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
dError("failed to start TDengine since read log config error");
return -1;
}
if (taosInitCfg(configDir, global.envFile, global.apolloUrl, NULL, 0) != 0) {
dError("failed to start TDengine since read config error");
return -1;
}
if (global.dumpConfig) {
dndDumpCfg();
taosCleanupCfg();
taosCloseLog();
return 0;
}
return dndRunDnode();
}

0
source/dnode/mgmt/container/src/dndMonitor.c → source/dnode/mgmt/main/src/dndMonitor.c
View File

4
source/dnode/mgmt/container/src/dndMsg.c → source/dnode/mgmt/main/src/dndMsg.c
View File

@ -42,6 +42,8 @@ static inline int32_t dndBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
}
memcpy(pMsg->user, connInfo.user, TSDB_USER_LEN);
pMsg->clientIp = connInfo.clientIp;
pMsg->clientPort = connInfo.clientPort;
memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
return 0;
}
@ -66,7 +68,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) {
} else if (pWrapper->procType == PROC_PARENT) {
dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle,
pRpc->ahandle, pMsg->user);
code = taosProcPutToChildQueue(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen);
code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ);
} else {
dTrace("msg:%p, should not processed in child process, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle,
pMsg->user);

0
source/dnode/mgmt/container/src/dndObj.c → source/dnode/mgmt/main/src/dndObj.c
View File

64
source/dnode/mgmt/main/src/dndStr.c Normal file
View File

@ -0,0 +1,64 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndInt.h"
const char *dndStatStr(EDndStatus status) {
switch (status) {
case DND_STAT_INIT:
return "init";
case DND_STAT_RUNNING:
return "running";
case DND_STAT_STOPPED:
return "stopped";
default:
return "unknown";
}
}
const char *dndNodeLogStr(ENodeType ntype) {
switch (ntype) {
case VNODES:
return "vnode";
case QNODE:
return "qnode";
case SNODE:
return "snode";
case MNODE:
return "mnode";
case BNODE:
return "bnode";
default:
return "taosd";
}
}
const char *dndNodeProcStr(ENodeType ntype) {
switch (ntype) {
case VNODES:
return "taosv";
case QNODE:
return "taosq";
case SNODE:
return "taoss";
case MNODE:
return "taosm";
case BNODE:
return "taosb";
default:
return "taosd";
}
}

81
source/dnode/mgmt/container/src/dndTransport.c → source/dnode/mgmt/main/src/dndTransport.c
View File

@ -309,7 +309,7 @@ int32_t dndInitMsgHandle(SDnode *pDnode) {
return 0;
}
static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
static int32_t dndSendRpcReq(STransMgmt *pMgmt, const SEpSet *pEpSet, SRpcMsg *pReq) {
if (pMgmt->clientRpc == NULL) {
terrno = TSDB_CODE_DND_OFFLINE;
return -1;
@ -319,9 +319,8 @@ static int32_t dndSendRpcReq(STransMgmt *pMgmt, SEpSet *pEpSet, SRpcMsg *pReq) {
return 0;
}
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq) {
if (pWrapper->procType == PROC_CHILD) {
} else {
int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pReq) {
if (pWrapper->procType != PROC_CHILD) {
SDnode *pDnode = pWrapper->pDnode;
if (dndGetStatus(pDnode) != DND_STAT_RUNNING) {
terrno = TSDB_CODE_DND_OFFLINE;
@ -329,26 +328,27 @@ int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, SEpSet *pEpSet, SRpcMsg *pReq)
return -1;
}
return dndSendRpcReq(&pDnode->trans, pEpSet, pReq);
} else {
while (taosProcPutToParentQ(pWrapper->pProc, pReq, sizeof(SRpcMsg), pReq->pCont, pReq->contLen, PROC_REQ) != 0) {
taosMsleep(1);
}
}
}
int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pReq) {
if (pWrapper->procType == PROC_CHILD) {
} else {
SDnode *pDnode = pWrapper->pDnode;
STransMgmt *pTrans = &pDnode->trans;
SEpSet epSet = {0};
SDnode *pDnode = pWrapper->pDnode;
STransMgmt *pTrans = &pDnode->trans;
SEpSet epSet = {0};
SMgmtWrapper *pWrapper = dndAcquireWrapper(pDnode, DNODE);
if (pWrapper != NULL) {
dmGetMnodeEpSet(pWrapper->pMgmt, &epSet);
dndReleaseWrapper(pWrapper);
}
return dndSendRpcReq(pTrans, &epSet, pReq);
SMgmtWrapper *pWrapper2 = dndAcquireWrapper(pDnode, DNODE);
if (pWrapper2 != NULL) {
dmGetMnodeEpSet(pWrapper2->pMgmt, &epSet);
dndReleaseWrapper(pWrapper2);
}
return dndSendRpcReq(pTrans, &epSet, pReq);
}
static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
if (pRsp->code == TSDB_CODE_APP_NOT_READY) {
SMgmtWrapper *pDnodeWrapper = dndAcquireWrapper(pWrapper->pDnode, DNODE);
if (pDnodeWrapper != NULL) {
@ -362,16 +362,45 @@ static void dndSendRpcRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
}
}
void dndSendRsp(SMgmtWrapper *pWrapper, SRpcMsg *pRsp) {
if (pWrapper->procType == PROC_CHILD) {
int32_t code = -1;
do {
code = taosProcPutToParentQueue(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen);
if (code != 0) {
taosMsleep(10);
}
} while (code != 0);
} else {
void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp) {
if (pWrapper->procType != PROC_CHILD) {
dndSendRpcRsp(pWrapper, pRsp);
} else {
while (taosProcPutToParentQ(pWrapper->pProc, pRsp, sizeof(SRpcMsg), pRsp->pCont, pRsp->contLen, PROC_RSP) != 0) {
taosMsleep(1);
}
}
}
void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) {
if (pWrapper->procType != PROC_CHILD) {
rpcRegisterBrokenLinkArg(pMsg);
} else {
while (taosProcPutToParentQ(pWrapper->pProc, pMsg, sizeof(SRpcMsg), pMsg->pCont, pMsg->contLen, PROC_REG) != 0) {
taosMsleep(1);
}
}
}
static void dndReleaseHandle(SMgmtWrapper *pWrapper, void *handle, int8_t type) {
if (pWrapper->procType != PROC_CHILD) {
rpcReleaseHandle(handle, type);
} else {
SRpcMsg msg = {.handle = handle, .code = type};
while (taosProcPutToParentQ(pWrapper->pProc, &msg, sizeof(SRpcMsg), NULL, 0, PROC_RELEASE) != 0) {
taosMsleep(1);
}
}
}
SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper) {
SMsgCb msgCb = {
.pWrapper = pWrapper,
.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg,
.releaseHandleFp = dndReleaseHandle,
.sendMnodeReqFp = dndSendReqToMnode,
.sendReqFp = dndSendReqToDnode,
.sendRspFp = dndSendRsp,
};
return msgCb;
}

55
source/dnode/mgmt/main/src/dndUtil.c
View File

@ -1,55 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndMain.h"
void dndGenerateGrant() {
#if 0
grantParseParameter();
#endif
}
void dndPrintVersion() {
#ifdef TD_ENTERPRISE
char *releaseName = "enterprise";
#else
char *releaseName = "community";
#endif
printf("%s version: %s compatible_version: %s\n", releaseName, version, compatible_version);
printf("gitinfo: %s\n", gitinfo);
printf("builuInfo: %s\n", buildinfo);
}
void dndDumpCfg() {
SConfig *pCfg = taosGetCfg();
cfgDumpCfg(pCfg, 0, 1);
}
SDnodeOpt dndGetOpt() {
SConfig *pCfg = taosGetCfg();
SDnodeOpt option = {0};
option.numOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32;
tstrncpy(option.dataDir, tsDataDir, sizeof(option.dataDir));
tstrncpy(option.firstEp, tsFirst, sizeof(option.firstEp));
tstrncpy(option.secondEp, tsSecond, sizeof(option.firstEp));
option.serverPort = tsServerPort;
tstrncpy(option.localFqdn, tsLocalFqdn, sizeof(option.localFqdn));
snprintf(option.localEp, sizeof(option.localEp), "%s:%u", option.localFqdn, option.serverPort);
option.pDisks = tsDiskCfg;
option.numOfDisks = tsDiskCfgNum;
return option;
}

0
source/dnode/mgmt/mnode/inc/mm.h → source/dnode/mgmt/mm/inc/mm.h
View File

0
source/dnode/mgmt/mnode/inc/mmInt.h → source/dnode/mgmt/mm/inc/mmInt.h
View File

0
source/dnode/mgmt/mnode/src/mmFile.c → source/dnode/mgmt/mm/src/mmFile.c
View File

1
source/dnode/mgmt/mnode/src/mmInt.c → source/dnode/mgmt/mm/src/mmInt.c
View File

@ -52,6 +52,7 @@ static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

0
source/dnode/mgmt/mnode/src/mmMsg.c → source/dnode/mgmt/mm/src/mmMsg.c
View File

0
source/dnode/mgmt/mnode/src/mmWorker.c → source/dnode/mgmt/mm/src/mmWorker.c
View File

0
source/dnode/mgmt/qnode/inc/qm.h → source/dnode/mgmt/qm/inc/qm.h
View File

0
source/dnode/mgmt/qnode/inc/qmInt.h → source/dnode/mgmt/qm/inc/qmInt.h
View File

1
source/dnode/mgmt/qnode/src/qmInt.c → source/dnode/mgmt/qm/src/qmInt.c
View File

@ -27,6 +27,7 @@ static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

0
source/dnode/mgmt/qnode/src/qmMsg.c → source/dnode/mgmt/qm/src/qmMsg.c
View File

0
source/dnode/mgmt/qnode/src/qmWorker.c → source/dnode/mgmt/qm/src/qmWorker.c
View File

0
source/dnode/mgmt/snode/inc/sm.h → source/dnode/mgmt/sm/inc/sm.h
View File

0
source/dnode/mgmt/snode/inc/smInt.h → source/dnode/mgmt/sm/inc/smInt.h
View File

1
source/dnode/mgmt/snode/src/smInt.c → source/dnode/mgmt/sm/src/smInt.c
View File

@ -24,6 +24,7 @@ static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
pOption->msgCb = msgCb;
}

0
source/dnode/mgmt/snode/src/smMsg.c → source/dnode/mgmt/sm/src/smMsg.c
View File

0
source/dnode/mgmt/snode/src/smWorker.c → source/dnode/mgmt/sm/src/smWorker.c
View File

0
source/dnode/mgmt/vnode/inc/vm.h → source/dnode/mgmt/vm/inc/vm.h
View File

0
source/dnode/mgmt/vnode/inc/vmInt.h → source/dnode/mgmt/vm/inc/vmInt.h
View File

0
source/dnode/mgmt/vnode/src/vmFile.c → source/dnode/mgmt/vm/src/vmFile.c
View File

1
source/dnode/mgmt/vnode/src/vmInt.c → source/dnode/mgmt/vm/src/vmInt.c
View File

@ -137,6 +137,7 @@ static void *vmOpenVnodeFunc(void *param) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid};
SVnode *pImpl = vnodeOpen(pCfg->path, &cfg);
if (pImpl == NULL) {

1
source/dnode/mgmt/vnode/src/vmMsg.c → source/dnode/mgmt/vm/src/vmMsg.c
View File

@ -91,6 +91,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
msgCb.sendReqFp = dndSendReqToDnode;
msgCb.sendMnodeReqFp = dndSendReqToMnode;
msgCb.sendRspFp = dndSendRsp;
msgCb.registerBrokenLinkArgFp = dndRegisterBrokenLinkArg;
vnodeCfg.msgCb = msgCb;
vnodeCfg.pTfs = pMgmt->pTfs;

0
source/dnode/mgmt/vnode/src/vmWorker.c → source/dnode/mgmt/vm/src/vmWorker.c
View File

6
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -731,10 +731,10 @@ typedef struct {
int32_t vgNum;
SRWLatch lock;
int8_t status;
int8_t sourceType;
int8_t sinkType;
// int32_t sqlLen;
int32_t sinkVgId; // 0 for automatic
int8_t createdBy; // STREAM_CREATED_BY__USER or SMA
int32_t fixedSinkVgId; // 0 for shuffle
int64_t smaId; // 0 for unused
char* sql;
char* logicalPlan;
char* physicalPlan;

2
source/dnode/mnode/impl/inc/mndScheduler.h
View File

@ -27,7 +27,7 @@ void mndCleanupScheduler(SMnode* pMnode);
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub);
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId);
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream);
#ifdef __cplusplus
}

2
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -31,7 +31,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId);
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans);
#ifdef __cplusplus
}

10
source/dnode/mnode/impl/src/mndDef.c
View File

@ -26,8 +26,11 @@ int32_t tEncodeSStreamObj(SCoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeI64(pEncoder, pObj->dbUid) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->version) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
if (tEncodeI8(pEncoder, pObj->createdBy) < 0) return -1;
if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1;
if (tEncodeI64(pEncoder, pObj->smaId) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1;
if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;
/*if (tEncodeCStr(pEncoder, pObj->logicalPlan) < 0) return -1;*/
if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1;
// TODO encode tasks
if (pObj->tasks) {
@ -69,8 +72,11 @@ int32_t tDecodeSStreamObj(SCoder *pDecoder, SStreamObj *pObj) {
if (tDecodeI64(pDecoder, &pObj->dbUid) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->version) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
if (tDecodeI8(pDecoder, &pObj->createdBy) < 0) return -1;
if (tDecodeI32(pDecoder, &pObj->fixedSinkVgId) < 0) return -1;
if (tDecodeI64(pDecoder, &pObj->smaId) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->sql) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;
/*if (tDecodeCStrAlloc(pDecoder, &pObj->logicalPlan) < 0) return -1;*/
if (tDecodeCStrAlloc(pDecoder, &pObj->physicalPlan) < 0) return -1;
pObj->tasks = NULL;
int32_t sz;

312
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -20,138 +20,178 @@
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
//!!!! Note: only APPEND columns in below tables, NO insert !!!!
static const SInfosTableSchema dnodesSchema[] = {{.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema mnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema modulesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema qnodesSchema[] = {{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema userDBSchema[] = {{.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "wallevel", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
// {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update
};
static const SInfosTableSchema userFuncSchema[] = {{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userIdxSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userStbsSchema[] = {{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userStreamsSchema[] = {{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY},
{.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema dnodesSchema[] = {
{.name = "id", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "max_vnodes", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema mnodesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema modulesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "module", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema qnodesSchema[] = {
{.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "endpoint", .bytes = 134, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
};
static const SInfosTableSchema userDBSchema[] = {
{.name = "name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "vgroups", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "ntables", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "replica", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "quorum", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "days", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT},
{.name = "keep", .bytes = 24 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "cache", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "wallevel", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "fsync", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT},
{.name = "precision", .bytes = 3 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
// {.name = "update", .bytes = 1, .type =
// TSDB_DATA_TYPE_TINYINT}, // disable update
};
static const SInfosTableSchema userFuncSchema[] = {
{.name = "name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "ntables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "precision", .bytes = 2, .type = TSDB_DATA_TYPE_BINARY},
{.name = "status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userIdxSchema[] = {
{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_database", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "column_name", .bytes = 64, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userStbsSchema[] = {
{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tags", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "last_update", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "table_comment", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userStreamsSchema[] = {
{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "user_name", .bytes = 23, .type = TSDB_DATA_TYPE_BINARY},
{.name = "dest_table", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema userTblsSchema[] = {
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userTblDistSchema[] = {{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY},
{.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE},
{.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userUsersSchema[] = {{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema vgroupsSchema[] = {{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "columns", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "uid", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userTblDistSchema[] = {
{.name = "db_name", .bytes = 32, .type = TSDB_DATA_TYPE_BINARY},
{.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "distributed_histogram", .bytes = 500, .type = TSDB_DATA_TYPE_BINARY},
{.name = "min_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "max_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "avg_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "stddev_of_rows", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "rows", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "blocks", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "storage_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "compression_ratio", .bytes = 8, .type = TSDB_DATA_TYPE_DOUBLE},
{.name = "rows_in_mem", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "seek_header_time", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema userUsersSchema[] = {
{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableSchema vgroupsSchema[] = {
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "onlines", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v1_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v2_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v2_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "v3_status", .bytes = 10, .type = TSDB_DATA_TYPE_BINARY},
{.name = "compacting", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableMeta infosMeta[] = {{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
{TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
{TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
{TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
{TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
{TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
{TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
{TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
{TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
};
// TODO put into perf schema
static const SInfosTableSchema topicSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024, .type = TSDB_DATA_TYPE_BINARY},
{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
//connection/application/
static const SInfosTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// ep
// up time
// topics
};
static const SInfosTableSchema subscribeSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
};
static const SInfosTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema)},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema)},
{TSDB_INS_TABLE_MODULES, modulesSchema, tListLen(modulesSchema)},
{TSDB_INS_TABLE_QNODES, qnodesSchema, tListLen(qnodesSchema)},
{TSDB_INS_TABLE_USER_DATABASES, userDBSchema, tListLen(userDBSchema)},
{TSDB_INS_TABLE_USER_FUNCTIONS, userFuncSchema, tListLen(userFuncSchema)},
{TSDB_INS_TABLE_USER_INDEXES, userIdxSchema, tListLen(userIdxSchema)},
{TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)},
{TSDB_INS_TABLE_USER_STREAMS, userStreamsSchema, tListLen(userStreamsSchema)},
{TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)},
{TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
{TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
};
// connection/application/
int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, SSchema **pDst) {
SSchema *schema = taosMemoryCalloc(colNum, sizeof(SSchema));
if (NULL == schema) {
@ -161,7 +201,7 @@ int32_t mndInitInfosTableSchema(const SInfosTableSchema *pSrc, int32_t colNum, S
for (int32_t i = 0; i < colNum; ++i) {
strcpy(schema[i].name, pSrc[i].name);
schema[i].type = pSrc[i].type;
schema[i].colId = i + 1;
schema[i].bytes = pSrc[i].bytes;
@ -182,11 +222,11 @@ int32_t mndInsInitMeta(SHashObj *hash) {
for (int32_t i = 0; i < tListLen(infosMeta); ++i) {
strcpy(meta.tbName, infosMeta[i].name);
meta.numOfColumns = infosMeta[i].colNum;
if (mndInitInfosTableSchema(infosMeta[i].schema, infosMeta[i].colNum, &meta.pSchemas)) {
return -1;
}
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -210,7 +250,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
}
*pRsp = *meta;
pRsp->pSchemas = taosMemoryCalloc(meta->numOfColumns, sizeof(SSchema));
if (pRsp->pSchemas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -237,13 +277,13 @@ void mndCleanupInfos(SMnode *pMnode) {
if (NULL == pMnode->infosMeta) {
return;
}
void *pIter = taosHashIterate(pMnode->infosMeta, NULL);
while (pIter) {
STableMetaRsp *meta = (STableMetaRsp *)pIter;
taosMemoryFreeClear(meta->pSchemas);
pIter = taosHashIterate(pMnode->infosMeta, pIter);
}
@ -251,5 +291,3 @@ void mndCleanupInfos(SMnode *pMnode) {
pMnode->infosMeta = NULL;
}

31
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -44,7 +44,8 @@ typedef struct {
SQueryDesc *pQueries;
} SConnObj;
static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime);
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid,
const char *app, int64_t startTime);
static void mndFreeConn(SConnObj *pConn);
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
@ -94,7 +95,8 @@ void mndCleanupProfile(SMnode *pMnode) {
}
}
static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid, const char *app, int64_t startTime) {
static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, uint32_t ip, uint16_t port, int32_t pid,
const char *app, int64_t startTime) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1);
@ -104,8 +106,8 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
SConnObj connObj = {.id = connId,
.appStartTimeMs = startTime,
.pid = pid,
.ip = pInfo->clientIp,
.port = pInfo->clientPort,
.ip = ip,
.port = port,
.killed = 0,
.loginTimeMs = taosGetTimestampMs(),
.lastAccessTimeMs = 0,
@ -114,17 +116,17 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
.pQueries = NULL};
connObj.lastAccessTimeMs = connObj.loginTimeMs;
tstrncpy(connObj.user, pInfo->user, TSDB_USER_LEN);
tstrncpy(connObj.user, user, TSDB_USER_LEN);
tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN);
int32_t keepTime = tsShellActivityTimer * 3;
SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000);
if (pConn == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("conn:%d, failed to put into cache since %s, user:%s", connId, pInfo->user, terrstr());
mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr());
return NULL;
} else {
mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, pInfo->user);
mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, user);
return pConn;
}
}
@ -184,20 +186,14 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
SConnObj *pConn = NULL;
int32_t code = -1;
SConnectReq connReq = {0};
char ip[30] = {0};
if (tDeserializeSConnectReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &connReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto CONN_OVER;
}
SRpcConnInfo info = {0};
if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) {
mError("user:%s, failed to login while get connection info since %s", pReq->user, terrstr());
goto CONN_OVER;
}
char ip[30];
taosIp2String(info.clientIp, ip);
taosIp2String(pReq->clientIp, ip);
pUser = mndAcquireUser(pMnode, pReq->user);
if (pUser == NULL) {
@ -216,7 +212,8 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
}
}
pConn = mndCreateConn(pMnode, &info, connReq.pid, connReq.app, connReq.startTime);
pConn =
mndCreateConn(pMnode, pReq->user, pReq->clientIp, pReq->clientPort, connReq.pid, connReq.app, connReq.startTime);
if (pConn == NULL) {
mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr());
goto CONN_OVER;
@ -241,7 +238,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
pReq->rspLen = contLen;
pReq->pRsp = pRsp;
mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app);
mDebug("user:%s, login from %s, conn:%d, app:%s", pReq->user, ip, pConn->id, connReq.app);
code = 0;

17
source/dnode/mnode/impl/src/mndQuery.c
View File

@ -14,15 +14,15 @@
*/
#include "mndQuery.h"
#include "mndMnode.h"
#include "executor.h"
#include "mndMnode.h"
#include "qworker.h"
int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
mTrace("message in query queue is processing");
SMnode *pMnode = pReq->pNode;
SMnode *pMnode = pReq->pNode;
SReadHandle handle = {0};
mTrace("msg:%p, in query queue is processing", pReq);
switch (pReq->rpcMsg.msgType) {
case TDMT_VND_QUERY:
return qWorkerProcessQueryMsg(&handle, pMnode->pQuery, &pReq->rpcMsg);
@ -35,9 +35,9 @@ int32_t mndProcessQueryMsg(SNodeMsg *pReq) {
}
int32_t mndProcessFetchMsg(SNodeMsg *pReq) {
mTrace("message in fetch queue is processing");
SMnode *pMnode = pReq->pNode;
mTrace("msg:%p, in fetch queue is processing", pReq);
switch (pReq->rpcMsg.msgType) {
case TDMT_VND_FETCH:
return qWorkerProcessFetchMsg(pMnode, pMnode->pQuery, &pReq->rpcMsg);
@ -52,9 +52,9 @@ int32_t mndProcessFetchMsg(SNodeMsg *pReq) {
}
int32_t mndInitQuery(SMnode *pMnode) {
int32_t code = qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb);
if (code) {
return code;
if (qWorkerInit(NODE_TYPE_MNODE, MND_VGID, NULL, (void **)&pMnode->pQuery, &pMnode->msgCb) != 0) {
mError("failed to init qworker in mnode since %s", terrstr());
return -1;
}
mndSetMsgHandle(pMnode, TDMT_VND_QUERY, mndProcessQueryMsg);
@ -67,4 +67,3 @@ int32_t mndInitQuery(SMnode *pMnode) {
}
void mndCleanupQuery(SMnode *pMnode) { qWorkerDestroy((void **)&pMnode->pQuery); }

118
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -119,7 +119,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) {
return pVgroup;
}
int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId) {
int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SSdb* pSdb = pMnode->pSdb;
void* pIter = NULL;
SArray* tasks = taosArrayGetP(pStream->tasks, 0);
@ -151,9 +151,9 @@ int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream,
pTask->execType = TASK_EXEC__NONE;
// sink
if (smaId != -1) {
if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
pTask->sinkType = TASK_SINK__SMA;
pTask->smaSink.smaId = smaId;
pTask->smaSink.smaId = pStream->smaId;
} else {
pTask->sinkType = TASK_SINK__TABLE;
}
@ -166,7 +166,45 @@ int32_t mndAddSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream,
return 0;
}
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, int64_t smaId) {
int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
ASSERT(pStream->fixedSinkVgId != 0);
SArray* tasks = taosArrayGetP(pStream->tasks, 0);
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
taosArrayPush(tasks, &pTask);
pTask->nodeId = pStream->fixedSinkVgId;
SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId);
if (pVgroup == NULL) {
return -1;
}
pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup);
// source
pTask->sourceType = TASK_SOURCE__MERGE;
// exec
pTask->execType = TASK_EXEC__NONE;
// sink
if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
pTask->sinkType = TASK_SINK__SMA;
pTask->smaSink.smaId = pStream->smaId;
} else {
pTask->sinkType = TASK_SINK__TABLE;
}
//
// dispatch
pTask->dispatchType = TASK_DISPATCH__NONE;
mndPersistTaskDeployReq(pTrans, pTask, &pTask->epSet, TDMT_VND_TASK_DEPLOY, pVgroup->vgId);
return 0;
}
int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
SSdb* pSdb = pMnode->pSdb;
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
if (pPlan == NULL) {
@ -185,7 +223,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
taosArrayPush(pStream->tasks, &taskOneLevel);
// add extra sink
hasExtraSink = true;
mndAddSinkToStream(pMnode, pTrans, pStream, smaId);
if (pStream->fixedSinkVgId == 0) {
mndAddShuffledSinkToStream(pMnode, pTrans, pStream);
} else {
mndAddFixedSinkToStream(pMnode, pTrans, pStream);
}
}
for (int32_t level = 0; level < totLevel; level++) {
@ -221,12 +263,14 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
pTask->sinkType = TASK_SINK__SHOW;
pTask->showSink.reserved = 0;
if (!hasExtraSink) {
if (smaId != -1) {
#if 1
if (pStream->createdBy == STREAM_CREATED_BY__SMA) {
pTask->sinkType = TASK_SINK__SMA;
pTask->smaSink.smaId = smaId;
pTask->smaSink.smaId = pStream->smaId;
} else {
pTask->sinkType = TASK_SINK__TABLE;
}
#endif
}
} else {
pTask->sinkType = TASK_SINK__NONE;
@ -286,35 +330,47 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream, i
if (hasExtraSink) {
// add dispatcher
pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
if (pStream->fixedSinkVgId == 0) {
pTask->dispatchType = TASK_DISPATCH__SHUFFLE;
pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
ASSERT(pDb);
if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
SDbObj* pDb = mndAcquireDb(pMnode, pStream->db);
ASSERT(pDb);
if (mndExtractDbInfo(pMnode, pDb, &pTask->shuffleDispatcher.dbInfo, NULL) < 0) {
sdbRelease(pSdb, pDb);
qDestroyQueryPlan(pPlan);
return -1;
}
sdbRelease(pSdb, pDb);
qDestroyQueryPlan(pPlan);
return -1;
}
sdbRelease(pSdb, pDb);
// put taskId to useDbRsp
// TODO: optimize
SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t sz = taosArrayGetSize(pVgs);
SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
int32_t sinkLvSize = taosArrayGetSize(sinkLv);
for (int32_t i = 0; i < sz; i++) {
SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
for (int32_t j = 0; j < sinkLvSize; j++) {
SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
/*printf("vgid %d node id %d\n", pVgInfo->vgId, pTask->nodeId);*/
if (pLastLevelTask->nodeId == pVgInfo->vgId) {
pVgInfo->taskId = pLastLevelTask->taskId;
/*printf("taskid %d set to %d\n", pVgInfo->taskId, pTask->taskId);*/
break;
// put taskId to useDbRsp
// TODO: optimize
SArray* pVgs = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
int32_t sz = taosArrayGetSize(pVgs);
SArray* sinkLv = taosArrayGetP(pStream->tasks, 0);
int32_t sinkLvSize = taosArrayGetSize(sinkLv);
for (int32_t i = 0; i < sz; i++) {
SVgroupInfo* pVgInfo = taosArrayGet(pVgs, i);
for (int32_t j = 0; j < sinkLvSize; j++) {
SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j);
/*printf("vgid %d node id %d\n", pVgInfo->vgId, pTask->nodeId);*/
if (pLastLevelTask->nodeId == pVgInfo->vgId) {
pVgInfo->taskId = pLastLevelTask->taskId;
/*printf("taskid %d set to %d\n", pVgInfo->taskId, pTask->taskId);*/
break;
}
}
}
} else {
pTask->dispatchType = TASK_DISPATCH__FIXED;
pTask->dispatchMsgType = TDMT_VND_TASK_WRITE_EXEC;
SArray* pArray = taosArrayGetP(pStream->tasks, 0);
// one sink only
ASSERT(taosArrayGetSize(pArray) == 1);
SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0);
pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId;
pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId;
pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet;
}
}
#endif

6
source/dnode/mnode/impl/src/mndSma.c
View File

@ -415,6 +415,10 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
streamObj.dbUid = pDb->uid;
streamObj.version = 1;
streamObj.sql = pCreate->sql;
streamObj.createdBy = STREAM_CREATED_BY__SMA;
// TODO
streamObj.fixedSinkVgId = 0;
streamObj.smaId = smaObj.uid;
/*streamObj.physicalPlan = "";*/
streamObj.logicalPlan = "not implemented";
@ -428,7 +432,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SNodeMsg *pReq, SMCreateSmaReq *pCre
if (mndSetCreateSmaRedoLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER;
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, smaObj.uid) != 0) goto _OVER;
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;

10
source/dnode/mnode/impl/src/mndStream.c
View File

@ -246,7 +246,7 @@ static int32_t mndStreamGetPlanString(const char *ast, char **pStr) {
return code;
}
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans, int64_t smaId) {
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, STrans *pTrans) {
SNode *pAst = NULL;
if (nodesStringToNode(ast, &pAst) < 0) {
@ -271,7 +271,7 @@ int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast
return -1;
}
if (mndScheduleStream(pMnode, pTrans, pStream, smaId) < 0) {
if (mndScheduleStream(pMnode, pTrans, pStream) < 0) {
mError("stream:%ld, schedule stream since %s", pStream->uid, terrstr());
return -1;
}
@ -300,6 +300,10 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
streamObj.dbUid = pDb->uid;
streamObj.version = 1;
streamObj.sql = pCreate->sql;
streamObj.createdBy = STREAM_CREATED_BY__USER;
// TODO
streamObj.fixedSinkVgId = 0;
streamObj.smaId = 0;
/*streamObj.physicalPlan = "";*/
streamObj.logicalPlan = "not implemented";
@ -310,7 +314,7 @@ static int32_t mndCreateStream(SMnode *pMnode, SNodeMsg *pReq, SCMCreateStreamRe
}
mDebug("trans:%d, used to create stream:%s", pTrans->id, pCreate->name);
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans, -1) != 0) {
if (mndAddStreamToTrans(pMnode, &streamObj, pCreate->ast, pTrans) != 0) {
mError("trans:%d, failed to add stream since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;

296
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -62,8 +62,6 @@ static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqT
static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp);
static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub);
int32_t mndInitSubscribe(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_SUBSCRIBE,
.keyType = SDB_KEY_BINARY,
@ -98,14 +96,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
return NULL;
}
#if 0
if (mndInitUnassignedVg(pMnode, pTopic, pSub) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tDeleteSMqSubscribeObj(pSub);
taosMemoryFree(pSub);
return NULL;
}
#endif
// TODO: disable alter subscribed table
return pSub;
}
@ -210,45 +200,6 @@ static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMq
return 0;
}
#if 0
static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode;
uint8_t *str = pMsg->rpcMsg.pCont;
SMqCMResetOffsetReq req;
SCoder decoder;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, str, pMsg->rpcMsg.contLen, TD_DECODER);
tDecodeSMqCMResetOffsetReq(&decoder, &req);
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (pHash == NULL) {
return -1;
}
for (int32_t i = 0; i < req.num; i++) {
SMqOffset *pOffset = &req.offsets[i];
SMqVgOffsets *pVgOffset = taosHashGet(pHash, &pOffset->vgId, sizeof(int32_t));
if (pVgOffset == NULL) {
pVgOffset = taosMemoryMalloc(sizeof(SMqVgOffsets));
if (pVgOffset == NULL) {
return -1;
}
pVgOffset->offsets = taosArrayInit(0, sizeof(void *));
taosArrayPush(pVgOffset->offsets, &pOffset);
}
taosHashPut(pHash, &pOffset->vgId, sizeof(int32_t), &pVgOffset, sizeof(void *));
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) {
mError("mq-reset-offset: failed since %s", terrstr());
return -1;
}
return 0;
}
#endif
static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode;
SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont;
@ -574,251 +525,6 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) {
return 0;
}
#if 0
for (int32_t j = 0; j < consumerNum; j++) {
bool changed = false;
bool unfished = false;
bool canUseLeft = imbalanceSolved < imbalanceVg;
bool mustUseLeft = canUseLeft && (imbalanceVg - imbalanceSolved >= consumerNum - j);
ASSERT(imbalanceVg - imbalanceSolved <= consumerNum - j);
int32_t maxVg = vgEachConsumer + canUseLeft;
int32_t minVg = vgEachConsumer + mustUseLeft;
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j);
int32_t vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo);
int32_t vgThisConsumerAfterRb;
if (vgThisConsumerBeforeRb > maxVg) {
vgThisConsumerAfterRb = maxVg;
imbalanceSolved++;
changed = true;
} else if (vgThisConsumerBeforeRb < minVg) {
vgThisConsumerAfterRb = minVg;
if (mustUseLeft) imbalanceSolved++;
changed = true;
} else {
vgThisConsumerAfterRb = vgThisConsumerBeforeRb;
}
if (vgThisConsumerBeforeRb > vgThisConsumerAfterRb) {
while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) {
// put into unassigned
SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo);
ASSERT(pConsumerEp != NULL);
ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId);
taosArrayPush(unassignedVgStash, pConsumerEp);
}
} else if (vgThisConsumerBeforeRb < vgThisConsumerAfterRb) {
// assign from unassigned
while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) {
// if no unassgined, save j
if (taosArrayGetSize(unassignedVgStash) == 0) {
taosArrayPush(unassignedConsumerIdx, &j);
unfished = true;
break;
}
// assign vg to consumer
SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
ASSERT(pConsumerEp != NULL);
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = pSubConsumer->consumerId;
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
// build msg and persist into trans
}
}
if (changed && !unfished) {
SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
pRebConsumer->epoch++;
if (vgThisConsumerAfterRb != 0) {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
} else {
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE);
}
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
mndReleaseConsumer(pMnode, pRebConsumer);
// TODO: save history
}
}
for (int32_t j = 0; j < taosArrayGetSize(unassignedConsumerIdx); j++) {
bool canUseLeft = imbalanceSolved < imbalanceVg;
int32_t consumerIdx = *(int32_t *)taosArrayGet(unassignedConsumerIdx, j);
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, consumerIdx);
if (canUseLeft) imbalanceSolved++;
// must use
int32_t vgThisConsumerAfterRb = taosArrayGetSize(pSubConsumer->vgInfo) + canUseLeft;
while (taosArrayGetSize(pSubConsumer->vgInfo) < vgEachConsumer + canUseLeft) {
// assign vg to consumer
SMqConsumerEp *pConsumerEp = taosArrayPop(unassignedVgStash);
ASSERT(pConsumerEp != NULL);
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = pSubConsumer->consumerId;
taosArrayPush(pSubConsumer->vgInfo, pConsumerEp);
// build msg and persist into trans
}
SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId);
pRebConsumer->epoch++;
atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
mndReleaseConsumer(pMnode, pRebConsumer);
// TODO: save history
}
#endif
#if 0
//update consumer status for the subscribption
for (int32_t i = 0; i < taosArrayGetSize(pSub->assigned); i++) {
SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i);
int64_t consumerId = pCEp->consumerId;
if (pCEp->status != -1) {
int32_t consumerHbStatus = atomic_fetch_add_32(&pCEp->consumerHbStatus, 1);
if (consumerHbStatus < MND_SUBSCRIBE_REBALANCE_CNT) {
continue;
}
// put consumer into lostConsumer
SMqConsumerEp* lostConsumer = taosArrayPush(pSub->lostConsumer, pCEp);
lostConsumer->qmsg = NULL;
// put vg into unassigned
taosArrayPush(pSub->unassignedVg, pCEp);
// remove from assigned
// TODO: swap with last one, reduce size and reset i
taosArrayRemove(pSub->assigned, i);
// remove from available consumer
for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) {
if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) {
taosArrayRemove(pSub->availConsumer, j);
break;
}
// TODO: acquire consumer, set status to unavail
}
#if 0
SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId);
pConsumer->epoch++;
printf("current epoch %ld size %ld", pConsumer->epoch, pConsumer->topics->size);
SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
sdbWriteNotFree(pMnode->pSdb, pRaw);
mndReleaseConsumer(pMnode, pConsumer);
#endif
}
}
// no available consumer, skip rebalance
if (taosArrayGetSize(pSub->availConsumer) == 0) {
continue;
}
taosArrayGet(pSub->availConsumer, 0);
// rebalance condition1 : have unassigned vg
// assign vg to a consumer, trying to find the least assigned one
if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) {
char *topic = NULL;
char *cgroup = NULL;
mndSplitSubscribeKey(pSub->key, &topic, &cgroup);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
for (int32_t i = 0; i < sz; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx);
pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer);
SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg);
pCEp->oldConsumerId = pCEp->consumerId;
pCEp->consumerId = consumerId;
taosArrayPush(pSub->assigned, pCEp);
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
pConsumer->epoch++;
SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
sdbWrite(pMnode->pSdb, pConsumerRaw);
mndReleaseConsumer(pMnode, pConsumer);
void* msg;
int32_t msgLen;
mndBuildRebalanceMsg(&msg, &msgLen, pTopic, pCEp, cgroup, topic);
// persist msg
STransAction action = {0};
action.epSet = pCEp->epSet;
action.pCont = msg;
action.contLen = sizeof(SMsgHead) + msgLen;
action.msgType = TDMT_VND_MQ_SET_CONN;
mndTransAppendRedoAction(pTrans, &action);
// persist data
SSdbRaw *pRaw = mndSubActionEncode(pSub);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pRaw);
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
}
mndReleaseTopic(pMnode, pTopic);
mndTransDrop(pTrans);
taosMemoryFreeClear(topic);
taosMemoryFreeClear(cgroup);
}
// rebalance condition2 : imbalance assignment
}
return 0;
}
#endif
#if 0
static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
SQueryPlan *pPlan = qStringToQueryPlan(pTopic->physicalPlan);
SArray *pArray = NULL;
SNodeListNode *inner = (SNodeListNode*)nodesListGetNode(pPlan->pSubplans, 0);
SSubplan *plan = (SSubplan*)nodesListGetNode(inner->pNodeList, 0);
SArray *unassignedVg = pSub->unassignedVg;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pTopic->dbUid) {
sdbRelease(pSdb, pVgroup);
continue;
}
pSub->vgNum++;
plan->execNode.nodeId = pVgroup->vgId;
plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
if (schedulerConvertDagToTaskList(pPlan, &pArray) < 0) {
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql);
return -1;
}
SMqConsumerEp consumerEp = {0};
consumerEp.status = 0;
consumerEp.consumerId = -1;
STaskInfo *pTaskInfo = taosArrayGet(pArray, 0);
consumerEp.epSet = pTaskInfo->addr.epset;
consumerEp.vgId = pTaskInfo->addr.nodeId;
ASSERT(consumerEp.vgId == pVgroup->vgId);
consumerEp.qmsg = strdup(pTaskInfo->msg->msg);
taosArrayPush(unassignedVg, &consumerEp);
// TODO: free taskInfo
taosArrayDestroy(pArray);
}
/*qDestroyQueryDag(pDag);*/
return 0;
}
#endif
static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,
const SMqConsumerEp *pConsumerEp) {
ASSERT(pConsumerEp->oldConsumerId == -1);
@ -1140,7 +846,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
}
}
if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void*))taosMemoryFree);
if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);
// persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);

11
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -65,9 +65,11 @@ void mndCleanupTopic(SMnode *pMnode) {}
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
int32_t size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE;
int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1;
int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1;
int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
int32_t size =
sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + swLen + MND_TOPIC_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
if (pRaw == NULL) goto TOPIC_ENCODE_OVER;
@ -86,8 +88,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER);
int32_t swLen = taosEncodeSSchemaWrapper(NULL, &pTopic->schema);
void *swBuf = taosMemoryMalloc(swLen);
void *swBuf = taosMemoryMalloc(swLen);
if (swBuf == NULL) {
goto TOPIC_ENCODE_OVER;
}

12
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -54,7 +54,7 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans);
static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans);
static void mndTransExecute(SMnode *pMnode, STrans *pTrans);
static void mndTransSendRpcRsp(STrans *pTrans);
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans);
static int32_t mndProcessTransReq(SNodeMsg *pReq);
static int32_t mndProcessKillTransReq(SNodeMsg *pReq);
@ -737,7 +737,7 @@ static int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
return 0;
}
static void mndTransSendRpcRsp(STrans *pTrans) {
static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
bool sendRsp = false;
if (pTrans->stage == TRN_STAGE_FINISHED) {
@ -771,7 +771,7 @@ static void mndTransSendRpcRsp(STrans *pTrans) {
.ahandle = pTrans->rpcAHandle,
.pCont = rpcCont,
.contLen = pTrans->rpcRspLen};
rpcSendResponse(&rspMsg);
tmsgSendRsp(&rspMsg);
pTrans->rpcHandle = NULL;
pTrans->rpcRsp = NULL;
pTrans->rpcRspLen = 0;
@ -898,7 +898,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr
pAction->msgReceived = 0;
pAction->errCode = 0;
} else {
mDebug("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr());
mError("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr());
return -1;
}
}
@ -938,7 +938,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
return errCode;
}
} else {
mDebug("trans:%d, %d of %d actions executed, code:0x%04x", pTrans->id, numOfReceived, numOfActions, errCode & 0XFFFF);
mDebug("trans:%d, %d of %d actions executing", pTrans->id, numOfReceived, numOfActions);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
}
@ -1158,7 +1158,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
}
}
mndTransSendRpcRsp(pTrans);
mndTransSendRpcRsp(pMnode, pTrans);
}
static int32_t mndProcessTransReq(SNodeMsg *pReq) {

7
source/dnode/vnode/src/tq/tq.c
View File

@ -275,7 +275,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pMsg->pCont = NULL;
pMsg->contLen = 0;
pMsg->code = -1;
rpcSendResponse(pMsg);
tmsgSendRsp(pMsg);
return 0;
}
@ -340,7 +340,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pMsg->pCont = buf;
pMsg->contLen = tlen;
pMsg->code = 0;
rpcSendResponse(pMsg);
tmsgSendRsp(pMsg);
return 0;
}
} else {
@ -356,6 +356,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
ASSERT(0);
return -1;
}
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
@ -367,7 +368,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
pMsg->pCont = buf;
pMsg->contLen = tlen;
pMsg->code = 0;
rpcSendResponse(pMsg);
tmsgSendRsp(pMsg);
/*}*/
return 0;

61
source/dnode/vnode/src/tq/tqRead.c
View File

@ -114,10 +114,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
int32_t numOfRows = pHandle->pBlock->numOfRows;
int32_t numOfCols = pHandle->pSchema->numOfCols;
/*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
// TODO: stable case
if (colNumNeed > pSchemaWrapper->nCols) {
colNumNeed = pSchemaWrapper->nCols;
}
@ -138,56 +137,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
colNeed++;
} else {
SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes;
/*int sz = numOfRows * pColSchema->bytes;*/
colInfo.info.bytes = pColSchema->bytes;
colInfo.info.colId = pColSchema->colId;
colInfo.info.type = pColSchema->type;
colInfo.pData = taosMemoryCalloc(1, sz);
if (colInfo.pData == NULL) {
// TODO free
taosArrayDestroy(pArray);
if (blockDataEnsureColumnCapacity(&colInfo, numOfRows) < 0) {
taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock);
return NULL;
}
blockDataEnsureColumnCapacity(&colInfo, numOfRows);
taosArrayPush(pArray, &colInfo);
colMeta++;
colNeed++;
}
}
int j = 0;
for (int32_t i = 0; i < colNumNeed; i++) {
col_id_t colId = *(col_id_t*)taosArrayGet(pHandle->pColIdList, i);
while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) {
j++;
}
if (j >= pSchemaWrapper->nCols) {
continue;
}
SSchema* pColSchema = &pSchemaWrapper->pSchema[j];
SColumnInfoData colInfo = {0};
int sz = numOfRows * pColSchema->bytes;
colInfo.info.bytes = pColSchema->bytes;
colInfo.info.colId = colId;
colInfo.info.type = pColSchema->type;
colInfo.pData = taosMemoryCalloc(1, sz);
if (colInfo.pData == NULL) {
// TODO free
taosArrayDestroy(pArray);
return NULL;
}
blockDataEnsureColumnCapacity(&colInfo, numOfRows);
taosArrayPush(pArray, &colInfo);
}
STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pTschema);
STSRow* row;
// int32_t kvIdx = 0;
int32_t curRow = 0;
tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter);
while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
@ -200,25 +167,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) {
break;
}
memcpy(POINTER_SHIFT(pColData->pData, curRow * pColData->info.bytes), sVal.val, pColData->info.bytes);
// TODO handle null
colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL);
}
#if 0
for (int32_t i = 0; i < colNumNeed; i++) {
SColumnInfoData* pColData = taosArrayGet(pArray, i);
STColumn* pCol = schemaColAt(pTschema, i);
// TODO
if(pCol->colId != pColData->info.colId) {
continue;
}
// void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
SCellVal sVal = {0};
if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) {
// TODO: reach end
break;
}
memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes);
}
#endif
curRow++;
}
return pArray;

6
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -205,8 +205,7 @@ _exit:
rpcMsg.contLen = rspLen;
rpcMsg.code = code;
rpcSendResponse(&rpcMsg);
tmsgSendRsp(&rpcMsg);
return TSDB_CODE_SUCCESS;
}
@ -276,8 +275,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
.code = 0,
};
rpcSendResponse(&rpcMsg);
tmsgSendRsp(&rpcMsg);
taosArrayDestroyEx(pArray, freeItemHelper);
return 0;
}

22
source/libs/executor/inc/executorimpl.h
View File

@ -430,9 +430,10 @@ typedef struct STagScanInfo {
} STagScanInfo;
typedef struct SStreamBlockScanInfo {
SArray* pBlockLists; // multiple SSDatablock.
SSDataBlock* pRes; // result SSDataBlock
int32_t blockType; // current block type
bool blockValid; // Is current data has returned?
int32_t validBlockIndex; // Is current data has returned?
SColumnInfo* pCols; // the output column info
uint64_t numOfRows; // total scanned rows
uint64_t numOfExec; // execution times
@ -572,11 +573,13 @@ typedef struct SGroupbyOperatorInfo {
typedef struct SSessionAggOperatorInfo {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
SGroupResInfo groupResInfo;
STimeWindow curWindow; // current time window
TSKEY prevTs; // previous timestamp
int32_t numOfRows; // number of rows
int32_t start; // start row index
bool reptScan; // next round scan
int64_t gap; // session window gap
} SSessionAggOperatorInfo;
typedef struct SStateWindowOperatorInfo {
@ -593,8 +596,7 @@ typedef struct SSortedMergeOperatorInfo {
SOptrBasicInfo binfo;
bool hasVarCol;
SArray *orderInfo; // SArray<SBlockOrderInfo>
bool nullFirst;
SArray* pSortInfo;
int32_t numOfSources;
SSortHandle *pSortHandle;
@ -613,12 +615,10 @@ typedef struct SSortedMergeOperatorInfo {
SAggSupporter aggSup;
} SSortedMergeOperatorInfo;
typedef struct SOrderOperatorInfo {
typedef struct SSortOperatorInfo {
uint32_t sortBufSize; // max buffer size for in-memory sort
SSDataBlock *pDataBlock;
bool hasVarCol; // has variable length column, such as binary/varchar/nchar
SArray *orderInfo;
bool nullFirst;
SArray* pSortInfo;
SSortHandle *pSortHandle;
int32_t bufPageSize;
int32_t numOfRowsInRes;
@ -629,7 +629,7 @@ typedef struct SOrderOperatorInfo {
uint64_t totalSize; // total load bytes from remote
uint64_t totalRows; // total number of rows
uint64_t totalElapsed; // total elapsed time
} SOrderOperatorInfo;
} SSortOperatorInfo;
typedef struct SDistinctDataInfo {
int32_t index;
@ -655,15 +655,15 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SArray* pOrderVal, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo *createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataBlock* pResBlock, const SName* pName,
SNode* pCondition, SEpSet epset, SArray* colList, SExecTaskInfo* pTaskInfo, bool showRewrite, int32_t accountId);
SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, SLimit* pLimit, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval,
const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock,
SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SInterval* pInterval, SSDataBlock* pResBlock,

2
source/libs/executor/inc/tsort.h
View File

@ -63,7 +63,7 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void*
* @param type
* @return
*/
SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr);
SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr);
/**
*

2
source/libs/executor/src/executil.c
View File

@ -19,8 +19,6 @@
#include "executil.h"
#include "executorimpl.h"
//#include "queryLog.h"
#include "tbuffer.h"
#include "tcompression.h"
#include "tlosertree.h"

28
source/libs/executor/src/executor.c
View File

@ -16,9 +16,10 @@
#include "executor.h"
#include "executorimpl.h"
#include "planner.h"
#include "tdatablock.h"
#include "vnode.h"
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t type, char* id) {
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) {
ASSERT(pOperator != NULL);
if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
if (pOperator->numOfDownstream == 0) {
@ -31,7 +32,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
return TSDB_CODE_QRY_APP_ERROR;
}
pOperator->status = OP_NOT_OPENED;
return doSetStreamBlock(pOperator->pDownstream[0], input, type, id);
return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id);
} else {
SStreamBlockScanInfo* pInfo = pOperator->info;
@ -48,15 +49,16 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
return TSDB_CODE_QRY_APP_ERROR;
}
} else {
ASSERT(!pInfo->blockValid);
for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i];
SSDataBlock* pDataBlock = input;
pInfo->pRes->info = pDataBlock->info;
taosArrayClear(pInfo->pRes->pDataBlock);
taosArrayAddAll(pInfo->pRes->pDataBlock, pDataBlock->pDataBlock);
SSDataBlock* p = createOneDataBlock(pDataBlock);
p->info = pDataBlock->info;
// set current block valid.
pInfo->blockValid = true;
taosArrayClear(p->pDataBlock);
taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock);
taosArrayPush(pInfo->pBlockLists, &p);
}
}
return TSDB_CODE_SUCCESS;
@ -64,17 +66,21 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, int32_t t
}
int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type) {
return qSetMultiStreamInput(tinfo, input, 1, type);
}
int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) {
if (tinfo == NULL) {
return TSDB_CODE_QRY_APP_ERROR;
}
if (input == NULL) {
if (pBlocks == NULL || numOfBlocks == 0) {
return TSDB_CODE_SUCCESS;
}
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*)input, type, GET_TASKID(pTaskInfo));
int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo));
} else {

4439
source/libs/executor/src/executorimpl.c
View File

File diff suppressed because it is too large Load Diff

38
source/libs/executor/src/tsort.c
View File

@ -23,20 +23,19 @@
#include "tsort.h"
#include "tutil.h"
typedef struct STupleHandle {
struct STupleHandle {
SSDataBlock* pBlock;
int32_t rowIndex;
} STupleHandle;
};
typedef struct SSortHandle {
struct SSortHandle {
int32_t type;
int32_t pageSize;
int32_t numOfPages;
SDiskbasedBuf *pBuf;
SArray *pOrderInfo;
bool nullFirst;
SArray *pSortInfo;
SArray *pOrderedSource;
_sort_fetch_block_fn_t fetchfp;
@ -60,7 +59,7 @@ typedef struct SSortHandle {
bool inMemSort;
bool needAdjust;
STupleHandle tupleHandle;
} SSortHandle;
};
static int32_t msortComparFn(const void *pLeft, const void *pRight, void *param);
@ -90,18 +89,18 @@ static SSDataBlock* createDataBlock_rv(SSchema* pSchema, int32_t numOfCols) {
* @param type
* @return
*/
SSortHandle* tsortCreateSortHandle(SArray* pOrderInfo, bool nullFirst, int32_t type, int32_t pageSize, int32_t numOfPages, SSchema* pSchema, int32_t numOfCols, const char* idstr) {
SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize, int32_t numOfPages, SSDataBlock* pBlock, const char* idstr) {
SSortHandle* pSortHandle = taosMemoryCalloc(1, sizeof(SSortHandle));
pSortHandle->type = type;
pSortHandle->pageSize = pageSize;
pSortHandle->numOfPages = numOfPages;
pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES);
pSortHandle->pOrderInfo = pOrderInfo;
pSortHandle->nullFirst = nullFirst;
pSortHandle->cmpParam.orderInfo = pOrderInfo;
pSortHandle->pSortInfo = pSortInfo;
pSortHandle->pDataBlock = createOneDataBlock(pBlock);
pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES);
pSortHandle->cmpParam.orderInfo = pSortInfo;
pSortHandle->pDataBlock = createDataBlock_rv(pSchema, numOfCols);
tsortSetComparFp(pSortHandle, msortComparFn);
if (idstr != NULL) {
@ -364,14 +363,14 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
for(int32_t i = 0; i < pInfo->size; ++i) {
SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex);
SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
bool leftNull = false;
if (pLeftColInfoData->hasNull) {
leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, pLeftBlock->pBlockAgg);
}
SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex);
SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId);
bool rightNull = false;
if (pRightColInfoData->hasNull) {
rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, pRightBlock->pBlockAgg);
@ -415,6 +414,9 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) {
static int32_t doInternalMergeSort(SSortHandle* pHandle) {
size_t numOfSources = taosArrayGetSize(pHandle->pOrderedSource);
if (numOfSources == 0) {
return 0;
}
// Calculate the I/O counts to complete the data sort.
double sortPass = floorl(log2(numOfSources) / log2(pHandle->numOfPages));
@ -542,7 +544,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
if (size > sortBufSize) {
// Perform the in-memory sort and then flush data in the buffer into disk.
int64_t p = taosGetTimestampUs();
blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst);
blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
int64_t el = taosGetTimestampUs() - p;
pHandle->sortElapsed += el;
@ -555,7 +557,7 @@ static int32_t createInitialSortedMultiSources(SSortHandle* pHandle) {
size_t size = blockDataGetSize(pHandle->pDataBlock);
// Perform the in-memory sort and then flush data in the buffer into disk.
blockDataSort(pHandle->pDataBlock, pHandle->pOrderInfo, pHandle->nullFirst);
blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
// All sorted data can fit in memory, external memory sort is not needed. Return to directly
if (size <= sortBufSize) {
@ -603,6 +605,10 @@ int32_t tsortOpen(SSortHandle* pHandle) {
ASSERT(numOfSources <= getNumOfInMemBufPages(pHandle->pBuf));
}
if (numOfSources == 0) {
return 0;
}
code = sortComparInit(&pHandle->cmpParam, pHandle->pOrderedSource, 0, numOfSources - 1, pHandle);
if (code != TSDB_CODE_SUCCESS) {
return code;

4
source/libs/executor/test/executorTests.cpp
View File

@ -966,7 +966,7 @@ TEST(testCase, inMem_sort_Test) {
exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
taosArrayPush(pExprInfo, &exp1);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL);
SOperatorInfo* pOperator = createSortOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL);
bool newgroup = false;
SSDataBlock* pRes = pOperator->getNextFn(pOperator, &newgroup);
@ -1035,7 +1035,7 @@ TEST(testCase, external_sort_Test) {
exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
// taosArrayPush(pExprInfo, &exp1);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL);
SOperatorInfo* pOperator = createSortOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL);
bool newgroup = false;
SSDataBlock* pRes = NULL;

Some files were not shown because too many files have changed in this diff Show More