xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/vnode

This commit is contained in:
Hongze Cheng 2021-12-17 13:46:06 +08:00
parent 2c6dd6c3a0 540516aace
commit 1ff5a7682a
104 changed files with 4813 additions and 3363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.devcontainer/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile
# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
ARG VARIANT="bullseye"
FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
# [Optional] Uncomment this section to install additional packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here>

32
.devcontainer/devcontainer.json Normal file
View File

@ -0,0 +1,32 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp
{
"name": "C++",
"build": {
"dockerfile": "Dockerfile",
// Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04
// Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon
"args": { "VARIANT": "ubuntu-21.04" }
},
"runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
// Set *default* container specific settings.json values on container create.
"settings": {},
// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"ms-vscode.cpptools",
"ms-vscode.cmake-tools",
"austin.code-gnu-global",
"visualstudioexptteam.vscodeintel"
],
// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],
// Use 'postCreateCommand' to run commands after the container is created.
// "postCreateCommand": "gcc -v",
// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "vscode"
}

65
include/client/taos.h
View File

@ -39,7 +39,7 @@ typedef void **TAOS_ROW;
#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes
#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes
#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes
#define TSDB_DATA_TYPE_BINARY 8 // string
#define TSDB_DATA_TYPE_BINARY 8 // string, alias for varchar
#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes
#define TSDB_DATA_TYPE_NCHAR 10 // unicode string
#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte
@ -47,10 +47,10 @@ typedef void **TAOS_ROW;
#define TSDB_DATA_TYPE_UINT 13 // 4 bytes
#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes
#define TSDB_DATA_TYPE_VARCHAR 15 // string
#define TSDB_DATA_TYPE_JSON 16 // json
#define TSDB_DATA_TYPE_DECIMAL 17 // decimal
#define TSDB_DATA_TYPE_BLOB 18 // binary string
#define TSDB_DATA_TYPE_LONGBLOB 19 // long binary string
#define TSDB_DATA_TYPE_VARBINARY 16 // binary
#define TSDB_DATA_TYPE_JSON 17 // json
#define TSDB_DATA_TYPE_DECIMAL 18 // decimal
#define TSDB_DATA_TYPE_BLOB 19 // binary
typedef enum {
TSDB_OPTION_LOCALE,
@ -61,10 +61,27 @@ typedef enum {
TSDB_MAX_OPTIONS
} TSDB_OPTION;
typedef enum {
TSDB_SML_UNKNOWN_PROTOCOL = 0,
TSDB_SML_LINE_PROTOCOL = 1,
TSDB_SML_TELNET_PROTOCOL = 2,
TSDB_SML_JSON_PROTOCOL = 3,
} TSDB_SML_PROTOCOL_TYPE;
typedef enum {
TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0,
TSDB_SML_TIMESTAMP_HOURS,
TSDB_SML_TIMESTAMP_MINUTES,
TSDB_SML_TIMESTAMP_SECONDS,
TSDB_SML_TIMESTAMP_MILLI_SECONDS,
TSDB_SML_TIMESTAMP_MICRO_SECONDS,
TSDB_SML_TIMESTAMP_NANO_SECONDS,
} TSDB_SML_TIMESTAMP_TYPE;
typedef struct taosField {
char name[65];
uint8_t type;
int16_t bytes;
int8_t type;
int32_t bytes;
} TAOS_FIELD;
#ifdef _TD_GO_DLL_
@ -136,37 +153,36 @@ DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt);
DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql);
DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res);
DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result
DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result
DLL_EXPORT void taos_free_result(TAOS_RES *res);
DLL_EXPORT int taos_field_count(TAOS_RES *res);
DLL_EXPORT int taos_num_fields(TAOS_RES *res);
DLL_EXPORT int taos_affected_rows(TAOS_RES *res);
DLL_EXPORT int taos_field_count(TAOS_RES *res);
DLL_EXPORT int taos_num_fields(TAOS_RES *res);
DLL_EXPORT int taos_affected_rows(TAOS_RES *res);
DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res);
DLL_EXPORT int taos_select_db(TAOS *taos, const char *db);
DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT int taos_select_db(TAOS *taos, const char *db);
DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields);
DLL_EXPORT void taos_stop_query(TAOS_RES *res);
DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows);
DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql);
DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res);
// TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild);
// TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild);
// TODO: the return value should be `const`
DLL_EXPORT const char *taos_get_server_info(TAOS *taos);
DLL_EXPORT const char *taos_get_client_info();
DLL_EXPORT const char *taos_errstr(TAOS_RES *tres);
DLL_EXPORT int taos_errno(TAOS_RES *tres);
DLL_EXPORT const char *taos_errstr(TAOS_RES *tres);
DLL_EXPORT int taos_errno(TAOS_RES *tres);
DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param);
DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param);
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval);
typedef void (*__taos_sub_fn_t)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, __taos_sub_fn_t fp, void *param, int interval);
DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
@ -175,8 +191,7 @@ DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)
DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList);
DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines);
DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision);
#ifdef __cplusplus
}

231
include/common/taosmsg.h
View File

@ -50,10 +50,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_ACK, "mq-ack" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_RESET, "mq-reset" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" )
// message from client to mnode
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_ACCT, "alter-acct" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_ACCT, "drop-acct" )
@ -215,17 +214,15 @@ typedef enum _mgmt_table {
extern char *taosMsg[];
typedef struct SBuildTableMetaInput {
int32_t vgId;
char *tableFullName;
int32_t vgId;
char *tableFullName;
} SBuildTableMetaInput;
typedef struct SBuildUseDBInput {
char db[TSDB_TABLE_FNAME_LEN];
int32_t vgroupVersion;
int32_t dbGroupVersion;
int32_t vgVersion;
} SBuildUseDBInput;
#pragma pack(push, 1)
// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
@ -235,7 +232,7 @@ typedef struct {
} SEpAddrMsg;
typedef struct {
char* fqdn;
char *fqdn;
uint16_t port;
} SEpAddr1;
@ -262,10 +259,10 @@ typedef struct SSubmitBlk {
// Submit message for this TSDB
typedef struct SSubmitMsg {
SMsgHead header;
int32_t length;
int32_t numOfBlocks;
char blocks[];
SMsgHead header;
int32_t length;
int32_t numOfBlocks;
char blocks[];
} SSubmitMsg;
typedef struct {
@ -344,7 +341,7 @@ typedef struct {
typedef struct {
char tableFname[TSDB_TABLE_FNAME_LEN];
char db[TSDB_FULL_DB_NAME_LEN];
int16_t type; /* operation type */
int16_t type; /* operation type */
int16_t numOfCols; /* number of schema */
int32_t tagValLen;
SSchema schema[];
@ -353,17 +350,17 @@ typedef struct {
} SAlterTableMsg;
typedef struct {
SMsgHead head;
int64_t uid;
int32_t tid;
int16_t tversion;
int16_t colId;
int8_t type;
int16_t bytes;
int32_t tagValLen;
int16_t numOfTags;
int32_t schemaLen;
char data[];
SMsgHead head;
int64_t uid;
int32_t tid;
int16_t tversion;
int16_t colId;
int8_t type;
int16_t bytes;
int32_t tagValLen;
int16_t numOfTags;
int32_t schemaLen;
char data[];
} SUpdateTableTagValMsg;
typedef struct {
@ -407,8 +404,10 @@ typedef struct {
} SDropUserMsg, SDropAcctMsg;
typedef struct {
int8_t type;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
int8_t superUser; // denote if it is a super user or not
int32_t reserve[8];
} SCreateUserMsg, SAlterUserMsg;
@ -424,7 +423,7 @@ typedef struct {
int32_t contLen;
int32_t vgId;
uint64_t uid;
char tableFname[TSDB_TABLE_FNAME_LEN];
char tableFname[TSDB_TABLE_FNAME_LEN];
} SDropSTableMsg;
typedef struct SColIndex {
@ -437,7 +436,7 @@ typedef struct SColIndex {
typedef struct SColumnFilterInfo {
int16_t lowerRelOptr;
int16_t upperRelOptr;
int16_t filterstr; // denote if current column is char(binary/nchar)
int16_t filterstr; // denote if current column is char(binary/nchar)
union {
struct {
@ -456,9 +455,9 @@ typedef struct SColumnFilterInfo {
} SColumnFilterInfo;
typedef struct SColumnFilterList {
int16_t numOfFilters;
union{
int64_t placeholder;
int16_t numOfFilters;
union {
int64_t placeholder;
SColumnFilterInfo *filterInfo;
};
} SColumnFilterList;
@ -467,10 +466,10 @@ typedef struct SColumnFilterList {
* But for data in vnode side, we need all the following information.
*/
typedef struct SColumnInfo {
int16_t colId;
int16_t type;
int16_t bytes;
SColumnFilterList flist;
int16_t colId;
int16_t type;
int16_t bytes;
SColumnFilterList flist;
} SColumnInfo;
typedef struct STableIdInfo {
@ -484,14 +483,14 @@ typedef struct STimeWindow {
} STimeWindow;
typedef struct {
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
int32_t tsOrder; // ts comp block order
int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed
int32_t tsLen; // total length of ts comp block
int32_t tsNumOfBlocks; // ts comp block numbers
int32_t tsOrder; // ts comp block order
} STsBufInfo;
typedef struct SInterval {
int32_t tz; // query client timezone
int32_t tz; // query client timezone
char intervalUnit;
char slidingUnit;
char offsetUnit;
@ -501,51 +500,51 @@ typedef struct SInterval {
} SInterval;
typedef struct {
SMsgHead head;
char version[TSDB_VERSION_LEN];
SMsgHead head;
char version[TSDB_VERSION_LEN];
bool stableQuery; // super table query or not
bool topBotQuery; // TODO used bitwise flag
bool interpQuery; // interp query or not
bool groupbyColumn; // denote if this is a groupby normal column query
bool hasTagResults; // if there are tag values in final result or not
bool timeWindowInterpo;// if the time window start/end required interpolation
bool queryBlockDist; // if query data block distribution
bool stabledev; // super table stddev query
bool tsCompQuery; // is tscomp query
bool simpleAgg;
bool pointInterpQuery; // point interpolation query
bool needReverseScan; // need reverse scan
bool stateWindow; // state window flag
bool stableQuery; // super table query or not
bool topBotQuery; // TODO used bitwise flag
bool interpQuery; // interp query or not
bool groupbyColumn; // denote if this is a groupby normal column query
bool hasTagResults; // if there are tag values in final result or not
bool timeWindowInterpo; // if the time window start/end required interpolation
bool queryBlockDist; // if query data block distribution
bool stabledev; // super table stddev query
bool tsCompQuery; // is tscomp query
bool simpleAgg;
bool pointInterpQuery; // point interpolation query
bool needReverseScan; // need reverse scan
bool stateWindow; // state window flag
STimeWindow window;
int32_t numOfTables;
int16_t order;
int16_t orderColId;
int16_t numOfCols; // the number of columns will be load from vnode
int16_t numOfCols; // the number of columns will be load from vnode
SInterval interval;
// SSessionWindow sw; // session window
int16_t tagCondLen; // tag length in current query
int16_t colCondLen; // column length in current query
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx
int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query.
int16_t prjOrder; // global order in super table projection query.
int64_t limit;
int64_t offset;
int32_t queryType; // denote another query process
int16_t numOfOutput; // final output columns numbers
int16_t fillType; // interpolate type
int64_t fillVal; // default value array list
int32_t secondStageOutput;
STsBufInfo tsBuf; // tsBuf info
int32_t numOfTags; // number of tags columns involved
int32_t sqlstrLen; // sql query string
int32_t prevResultLen; // previous result length
// SSessionWindow sw; // session window
int16_t tagCondLen; // tag length in current query
int16_t colCondLen; // column length in current query
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;
int16_t orderType; // used in group by xx order by xxx
int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query.
int16_t prjOrder; // global order in super table projection query.
int64_t limit;
int64_t offset;
int32_t queryType; // denote another query process
int16_t numOfOutput; // final output columns numbers
int16_t fillType; // interpolate type
int64_t fillVal; // default value array list
int32_t secondStageOutput;
STsBufInfo tsBuf; // tsBuf info
int32_t numOfTags; // number of tags columns involved
int32_t sqlstrLen; // sql query string
int32_t prevResultLen; // previous result length
int32_t numOfOperator;
int32_t tableScanOperator;// table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function
int32_t tableScanOperator; // table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function
int32_t udfContentOffset;
int32_t udfContentLen;
SColumnInfo tableCols[];
@ -571,19 +570,19 @@ typedef struct {
} SRetrieveTableMsg;
typedef struct SRetrieveTableRsp {
int32_t numOfRows;
int64_t offset; // updated offset value for multi-vnode projection query
int64_t useconds;
int8_t completed; // all results are returned to client
int8_t precision;
int8_t compressed;
int8_t reserved;
int32_t compLen;
int32_t numOfRows;
char data[];
} SRetrieveTableRsp;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
int32_t numOfVgroups;
int32_t cacheBlockSize; // MB
int32_t totalBlocks;
int32_t daysPerFile;
@ -626,9 +625,7 @@ typedef struct {
typedef struct {
char db[TSDB_TABLE_FNAME_LEN];
int8_t ignoreNotExists;
int32_t vgroupVersion;
int32_t dbGroupVersion;
int32_t vgVersion;
int32_t reserve[8];
} SUseDbMsg;
@ -790,8 +787,8 @@ typedef struct {
} SStbInfoMsg;
typedef struct {
SMsgHead msgHead;
char tableFname[TSDB_TABLE_FNAME_LEN];
SMsgHead msgHead;
char tableFname[TSDB_TABLE_FNAME_LEN];
} STableInfoMsg;
typedef struct {
@ -808,6 +805,9 @@ typedef struct SSTableVgroupMsg {
typedef struct SVgroupInfo {
int32_t vgId;
uint32_t hashBegin;
uint32_t hashEnd;
int8_t inUse;
int8_t numOfEps;
SEpAddrMsg epAddr[TSDB_MAX_REPLICA];
} SVgroupInfo;
@ -830,19 +830,19 @@ typedef struct {
} SVgroupsMsg, SVgroupsInfo;
typedef struct {
char tbFname[TSDB_TABLE_FNAME_LEN]; // table id
char stbFname[TSDB_TABLE_FNAME_LEN];
int32_t numOfTags;
int32_t numOfColumns;
int8_t precision;
int8_t tableType;
int8_t update;
int32_t sversion;
int32_t tversion;
uint64_t tuid;
uint64_t suid;
int32_t vgId;
SSchema pSchema[];
char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name
char stbFname[TSDB_TABLE_FNAME_LEN];
int32_t numOfTags;
int32_t numOfColumns;
int8_t precision;
int8_t tableType;
int8_t update;
int32_t sversion;
int32_t tversion;
uint64_t tuid;
uint64_t suid;
int32_t vgId;
SSchema pSchema[];
} STableMetaMsg;
typedef struct SMultiTableMeta {
@ -863,18 +863,12 @@ typedef struct {
} STagData;
typedef struct {
int32_t vgroupNum;
int32_t vgroupVersion;
char db[TSDB_TABLE_FNAME_LEN];
int32_t dbVgroupVersion;
int32_t dbVgroupNum;
int32_t dbHashRange;
int32_t dbHashType;
char db[TSDB_FULL_DB_NAME_LEN];
int32_t vgVersion;
int32_t vgNum;
int8_t hashMethod;
SVgroupInfo vgroupInfo[];
//int32_t vgIdList[];
} SUseDbRspMsg;
} SUseDbRsp;
/*
* sql: show tables like '%a_%'
@ -889,7 +883,7 @@ typedef struct {
} SShowMsg;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_FULL_DB_NAME_LEN];
int32_t numOfVgroup;
int32_t vgid[];
} SCompactMsg;
@ -1010,45 +1004,37 @@ typedef struct {
} SAuthMsg, SAuthRsp;
typedef struct {
int8_t finished;
int8_t reserved1[7];
char name[TSDB_STEP_NAME_LEN];
char desc[TSDB_STEP_DESC_LEN];
int8_t finished;
int8_t reserved1[7];
char name[TSDB_STEP_NAME_LEN];
char desc[TSDB_STEP_DESC_LEN];
} SStartupMsg;
// mq related
typedef struct {
} SMqConnectReq;
typedef struct {
} SMqConnectRsp;
typedef struct {
} SMqDisconnectReq;
typedef struct {
} SMqDisconnectRsp;
typedef struct {
} SMqAckReq;
typedef struct {
} SMqAckRsp;
typedef struct {
} SMqResetReq;
typedef struct {
} SMqResetRsp;
//mq related end
// mq related end
typedef struct {
/* data */
@ -1102,7 +1088,6 @@ typedef struct {
/* data */
} SUpdateTagValRsp;
#pragma pack(pop)
#ifdef __cplusplus

2
include/common/tmsgtype.h
View File

@ -62,7 +62,7 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_DNODE, "cfg-dnode" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_MNODE, "cfg-mnode" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW, "show" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE, "retrieve" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_MNODE, "retrieve" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_QUERY, "kill-query" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_STREAM, "kill-stream" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_CONNECTION, "kill-connection" )

194
include/dnode/vnode/tq/tq.h
View File

@ -16,9 +16,9 @@
#ifndef _TD_TQ_H_
#define _TD_TQ_H_
#include "mallocator.h"
#include "os.h"
#include "tutil.h"
#include "mallocator.h"
#ifdef __cplusplus
extern "C" {
@ -62,10 +62,10 @@ typedef struct TmqDisconnectRsp {
int8_t status;
} TmqDisconnectRsp;
typedef struct TmqConsumeReq {
typedef struct STqConsumeReq {
TmqMsgHead head;
TmqAcks acks;
} TmqConsumeReq;
} STqConsumeReq;
typedef struct TmqMsgContent {
int64_t topicId;
@ -73,11 +73,11 @@ typedef struct TmqMsgContent {
char msg[];
} TmqMsgContent;
typedef struct TmqConsumeRsp {
typedef struct STqConsumeRsp {
TmqMsgHead head;
int64_t bodySize;
TmqMsgContent msgs[];
} TmqConsumeRsp;
} STqConsumeRsp;
typedef struct TmqSubscribeReq {
TmqMsgHead head;
@ -97,128 +97,125 @@ typedef struct TmqHeartbeatReq {
typedef struct TmqHeartbeatRsp {
} TmqHeartbeatRsp;
typedef struct TqTopicVhandle {
typedef struct STqTopicVhandle {
int64_t topicId;
// executor for filter
void* filterExec;
// callback for mnode
// trigger when vnode list associated topic change
void* (*mCallback)(void*, void*);
} TqTopicVhandle;
} STqTopicVhandle;
#define TQ_BUFFER_SIZE 8
typedef struct TqBufferItem {
typedef struct STqBufferItem {
int64_t offset;
// executors are identical but not concurrent
// so there must be a copy in each item
void* executor;
int64_t size;
void* content;
} TqBufferItem;
} STqBufferItem;
typedef struct TqBufferHandle {
typedef struct STqBufferHandle {
// char* topic; //c style, end with '\0'
// int64_t cgId;
// void* ahandle;
int64_t nextConsumeOffset;
int64_t topicId;
int32_t head;
int32_t tail;
TqBufferItem buffer[TQ_BUFFER_SIZE];
} TqBufferHandle;
int64_t nextConsumeOffset;
int64_t floatingCursor;
int64_t topicId;
int32_t head;
int32_t tail;
STqBufferItem buffer[TQ_BUFFER_SIZE];
} STqBufferHandle;
typedef struct TqListHandle {
TqBufferHandle bufHandle;
struct TqListHandle* next;
} TqListHandle;
typedef struct STqListHandle {
STqBufferHandle bufHandle;
struct STqListHandle* next;
} STqListHandle;
typedef struct TqGroupHandle {
int64_t cId;
int64_t cgId;
void* ahandle;
int32_t topicNum;
TqListHandle* head;
} TqGroupHandle;
typedef struct STqGroupHandle {
int64_t cId;
int64_t cgId;
void* ahandle;
int32_t topicNum;
STqListHandle* head;
} STqGroupHandle;
typedef struct TqQueryExec {
void* src;
TqBufferItem* dest;
void* executor;
} TqQueryExec;
typedef struct STqQueryExec {
void* src;
STqBufferItem* dest;
void* executor;
} STqQueryExec;
typedef struct TqQueryMsg {
TqQueryExec* exec;
struct TqQueryMsg* next;
} TqQueryMsg;
typedef struct STqQueryMsg {
STqQueryExec* exec;
struct STqQueryMsg* next;
} STqQueryMsg;
typedef struct TqLogReader {
typedef struct STqLogReader {
void* logHandle;
int32_t (*logRead)(void* logHandle, void** data, int64_t ver);
int64_t (*logGetFirstVer)(void* logHandle);
int64_t (*logGetSnapshotVer)(void* logHandle);
int64_t (*logGetLastVer)(void* logHandle);
} TqLogReader;
} STqLogReader;
typedef struct STqCfg {
// TODO
} STqCfg;
typedef struct TqMemRef {
SMemAllocatorFactory *pAlloctorFactory;
SMemAllocator *pAllocator;
} TqMemRef;
typedef struct STqMemRef {
SMemAllocatorFactory* pAlloctorFactory;
SMemAllocator* pAllocator;
} STqMemRef;
typedef struct TqSerializedHead {
typedef struct STqSerializedHead {
int16_t ver;
int16_t action;
int32_t checksum;
int64_t ssize;
char content[];
} TqSerializedHead;
} STqSerializedHead;
typedef int (*TqSerializeFun)(const void* pObj, TqSerializedHead** ppHead);
typedef const void* (*TqDeserializeFun)(const TqSerializedHead* pHead, void** ppObj);
typedef void (*TqDeleteFun)(void*);
typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead);
typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj);
typedef void (*FTqDelete)(void*);
#define TQ_BUCKET_MASK 0xFF
#define TQ_BUCKET_SIZE 256
#define TQ_PAGE_SIZE 4096
//key + offset + size
// key + offset + size
#define TQ_IDX_SIZE 24
//4096 / 24
// 4096 / 24
#define TQ_MAX_IDX_ONE_PAGE 170
//24 * 170
// 24 * 170
#define TQ_IDX_PAGE_BODY_SIZE 4080
//4096 - 4080
// 4096 - 4080
#define TQ_IDX_PAGE_HEAD_SIZE 16
#define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1
#define TQ_ACTION_CONST 0
#define TQ_ACTION_INUSE 1
#define TQ_ACTION_INUSE_CONT 2
#define TQ_ACTION_INTXN 3
#define TQ_ACTION_INTXN 3
#define TQ_SVER 0
#define TQ_SVER 0
//TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1
// TODO: inplace mode is not implemented
#define TQ_UPDATE_INPLACE 0
#define TQ_UPDATE_APPEND 1
#define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2
#define TQ_DUP_INTXN_REJECT 2
static inline bool TqUpdateAppend(int32_t tqConfigFlag) {
return tqConfigFlag & TQ_UPDATE_APPEND;
}
static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
static inline bool TqDupIntxnReject(int32_t tqConfigFlag) {
return tqConfigFlag & TQ_DUP_INTXN_REJECT;
}
static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef struct TqMetaHandle {
int64_t key;
@ -229,62 +226,63 @@ typedef struct TqMetaHandle {
} STqMetaHandle;
typedef struct TqMetaList {
STqMetaHandle handle;
STqMetaHandle handle;
struct TqMetaList* next;
//struct TqMetaList* inTxnPrev;
//struct TqMetaList* inTxnNext;
// struct TqMetaList* inTxnPrev;
// struct TqMetaList* inTxnNext;
struct TqMetaList* unpersistPrev;
struct TqMetaList* unpersistNext;
} STqMetaList;
typedef struct TqMetaStore {
STqMetaList* bucket[TQ_BUCKET_SIZE];
//a table head
STqMetaList* unpersistHead;
//TODO:temporaral use, to be replaced by unified tfile
int fileFd;
//TODO:temporaral use, to be replaced by unified tfile
int idxFd;
char* dirPath;
int32_t tqConfigFlag;
TqSerializeFun pSerializer;
TqDeserializeFun pDeserializer;
TqDeleteFun pDeleter;
STqMetaList* bucket[TQ_BUCKET_SIZE];
// a table head
STqMetaList* unpersistHead;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;
// TODO:temporaral use, to be replaced by unified tfile
int idxFd;
char* dirPath;
int32_t tqConfigFlag;
FTqSerialize pSerializer;
FTqDeserialize pDeserializer;
FTqDelete pDeleter;
} STqMetaStore;
typedef struct STQ {
// the collection of group handle
// the handle of kvstore
char* path;
STqCfg* tqConfig;
TqLogReader* tqLogReader;
TqMemRef tqMemRef;
char* path;
STqCfg* tqConfig;
STqLogReader* tqLogReader;
STqMemRef tqMemRef;
STqMetaStore* tqMeta;
} STQ;
// open in each vnode
STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac);
STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac);
void tqClose(STQ*);
// void* will be replace by a msg type
int tqPushMsg(STQ*, void* msg, int64_t version);
int tqCommit(STQ*);
int tqSetCursor(STQ*, void* msg);
int tqConsume(STQ*, TmqConsumeReq*);
int tqConsume(STQ*, STqConsumeReq*);
TqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
TqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqMoveOffsetToNext(TqGroupHandle*);
int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
int tqRegisterContext(TqGroupHandle*, void* ahandle);
int tqLaunchQuery(TqGroupHandle*);
int tqSendLaunchQuery(TqGroupHandle*);
STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqMoveOffsetToNext(STqGroupHandle*);
int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
int tqRegisterContext(STqGroupHandle*, void* ahandle);
int tqLaunchQuery(STqGroupHandle*);
int tqSendLaunchQuery(STqGroupHandle*);
int tqSerializeGroupHandle(const TqGroupHandle* gHandle, TqSerializedHead** ppHead);
int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead);
const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle** gHandle);
const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle);
#ifdef __cplusplus
}

100
include/libs/catalog/catalog.h
View File

@ -32,8 +32,7 @@ extern "C" {
struct SCatalog;
typedef struct SCatalogReq {
char dbName[TSDB_DB_NAME_LEN];
SArray *pTableName; // table full name
SArray *pTableName; // element is SNAME
SArray *pUdf; // udf name
bool qNodeRequired; // valid qnode
} SCatalogReq;
@ -54,54 +53,86 @@ typedef struct SCatalogCfg {
int32_t catalogInit(SCatalogCfg *cfg);
/**
* Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side.
* There is ONLY one SCatalog object for one process space, and this function returns a singleton.
* @param clusterId
* @return
* Get a cluster's catalog handle for all later operations.
* @param clusterId (input, end with \0)
* @param catalogHandle (output, NO need to free it)
* @return error code
*/
int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle);
int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version);
/**
* get cluster vgroup list.
* @pVgroupList - hash of vgroup list, key:vgId, value:SVgroupInfo
* @return
*/
int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash);
int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup);
int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version);
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo);
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo);
int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo);
/**
* Get a table's meta data.
* @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name)
* @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code
*/
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
/**
* Force renew a table's local cached meta data.
* @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name)
* @return error code
*/
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName);
/**
* Force renew a table's local cached meta data and get the new one.
* @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name)
* @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code
*/
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
/**
* get table's vgroup list.
* @param clusterId
* @pVgroupList - array of SVgroupInfo
* @return
* Get a table's actual vgroup, for stable it's all possible vgroup list.
* @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name)
* @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller)
* @return error code
*/
int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList);
int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList);
/**
* Get a table's vgroup from its name's hash value.
* @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name)
* @param vgInfo (output, vgroup info)
* @return error code
*/
int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo);
/**
* Get the required meta data from mnode.
* Note that this is a synchronized API and is also thread-safety.
* @param pCatalog
* @param pMgmtEps
* @param pMetaReq
* @param pMetaData
* @return
* Get all meta data required in pReq.
* @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pReq (input, reqest info)
* @param pRsp (output, response data)
* @return error code
*/
int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
@ -112,7 +143,6 @@ int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, S
/**
* Destroy catalog and relase all resources
* @param pCatalog
*/
void catalogDestroy(void);

9
include/libs/index/index.h
View File

@ -28,6 +28,15 @@ typedef struct SIndexOpts SIndexOpts;
typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
typedef struct SArray SIndexMultiTerm;
typedef enum {
ADD_VALUE, // add index colume value
DEL_VALUE, // delete index column value
UPDATE_VALUE, // update index column value
ADD_INDEX, // add index on specify column
DROP_INDEX, // drop existed index
DROP_SATBLE // drop stable
} SIndexColumnType;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType;
/*

14
include/libs/parser/parser.h
View File

@ -132,13 +132,15 @@ struct SInsertStmtInfo;
bool qIsInsertSql(const char* pStr, size_t length);
typedef struct SParseContext {
const char* pAcctId;
const char* pDbname;
void *pRpc;
const char* pClusterId;
const SEpSet* pEpSet;
int64_t id; // query id, generated by uuid generator
int8_t schemaAttached; // denote if submit block is built with table schema or not
const char* pSql; // sql string
size_t sqlLen; // length of the sql string
int64_t id; // operator id, generated by uuid generator
const char* pDbname;
const SEpSet* pEpSet;
int8_t schemaAttached; // denote if submit block is built with table schema or not
char* pMsg; // extended error message if exists to help avoid the problem in sql statement.
int32_t msgLen; // max length of the msg
} SParseContext;
@ -151,7 +153,7 @@ typedef struct SParseContext {
* @param msg extended error message if exists.
* @return error code
*/
int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen);
int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen);
typedef enum {
PAYLOAD_TYPE_KV = 0,

109
include/libs/planner/planner.h
View File

@ -20,52 +20,92 @@
extern "C" {
#endif
#include "taosmsg.h"
#define QUERY_TYPE_MERGE 1
#define QUERY_TYPE_PARTIAL 2
#define QUERY_TYPE_SCAN 3
enum OPERATOR_TYPE_E {
OP_TableScan = 1,
OP_DataBlocksOptScan = 2,
OP_TableSeqScan = 3,
OP_TagScan = 4,
OP_TableBlockInfoScan= 5,
OP_Aggregate = 6,
OP_Project = 7,
OP_Groupby = 8,
OP_Limit = 9,
OP_SLimit = 10,
OP_TimeWindow = 11,
OP_SessionWindow = 12,
OP_StateWindow = 22,
OP_Fill = 13,
OP_MultiTableAggregate = 14,
OP_MultiTableTimeInterval = 15,
// OP_DummyInput = 16, //TODO remove it after fully refactor.
// OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream.
// OP_GlobalAggregate = 18, // global merge for the multi-way data sources.
OP_Filter = 19,
OP_Distinct = 20,
OP_Join = 21,
OP_AllTimeWindow = 23,
OP_AllMultiTableTimeInterval = 24,
OP_Order = 25,
OP_Exchange = 26,
OP_Unknown,
#define INCLUDE_AS_ENUM
#include "plannerOp.h"
#undef INCLUDE_AS_ENUM
OP_TotalNum
};
struct SEpSet;
struct SQueryPlanNode;
struct SPhyNode;
struct SQueryStmtInfo;
typedef SSchema SSlotSchema;
typedef struct SDataBlockSchema {
SSlotSchema *pSchema;
int32_t numOfCols; // number of columns
} SDataBlockSchema;
typedef struct SQueryNodeBasicInfo {
int32_t type; // operator type
const char *name; // operator name
} SQueryNodeBasicInfo;
typedef struct SPhyNode {
SQueryNodeBasicInfo info;
SArray *pTargets; // target list to be computed or scanned at this node
SArray *pConditions; // implicitly-ANDed qual conditions
SDataBlockSchema targetSchema;
// children plan to generated result for current node to process
// in case of join, multiple plan nodes exist.
SArray *pChildren;
struct SPhyNode *pParent;
} SPhyNode;
typedef struct SScanPhyNode {
SPhyNode node;
uint64_t uid; // unique id of the table
int8_t tableType;
} SScanPhyNode;
typedef SScanPhyNode SSystemTableScanPhyNode;
typedef SScanPhyNode STagScanPhyNode;
typedef struct STableScanPhyNode {
SScanPhyNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not
STimeWindow window;
SArray *pTagsConditions; // implicitly-ANDed tag qual conditions
} STableScanPhyNode;
typedef STableScanPhyNode STableSeqScanPhyNode;
typedef struct SProjectPhyNode {
SPhyNode node;
} SProjectPhyNode;
typedef struct SExchangePhyNode {
SPhyNode node;
uint64_t srcTemplateId; // template id of datasource suplans
SArray *pSourceEpSet; // SEpSet, scheduler fill by calling qSetSuplanExecutionNode
} SExchangePhyNode;
typedef struct SSubplanId {
uint64_t queryId;
uint64_t templateId;
uint64_t subplanId;
} SSubplanId;
typedef struct SSubplan {
int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN
SArray *pDatasource; // the datasource subplan,from which to fetch the result
struct SPhyNode *pNode; // physical plan of current subplan
SSubplanId id; // unique id of the subplan
int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN
int32_t level; // the execution level of current subplan, starting from 0.
SEpSet execEpSet; // for the scan sub plan, the optional execution node
SArray *pChildern; // the datasource subplan,from which to fetch the result
SArray *pParents; // the data destination subplan, get data from current subplan
SPhyNode *pNode; // physical plan of current subplan
} SSubplan;
typedef struct SQueryDag {
SArray **pSubplans;
SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0.
} SQueryDag;
/**
@ -73,8 +113,11 @@ typedef struct SQueryDag {
*/
int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag);
int32_t qSetSuplanExecutionNode(SArray* subplans, SArray* nodes);
int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str);
/**
* Convert to subplan to string for the scheduler to send to the executor
*/
@ -85,7 +128,7 @@ int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str);
* @param pQueryPhyNode
* @return
*/
void* qDestroyQueryDag(struct SQueryDag* pDag);
void qDestroyQueryDag(struct SQueryDag* pDag);
#ifdef __cplusplus
}

48
include/libs/planner/plannerOp.h Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#if defined(INCLUDE_AS_ENUM) // enum define mode
#undef OP_ENUM_MACRO
#define OP_ENUM_MACRO(op) OP_##op,
#elif defined(INCLUDE_AS_NAME) // comment define mode
#undef OP_ENUM_MACRO
#define OP_ENUM_MACRO(op) #op,
#else
#error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME
#endif
OP_ENUM_MACRO(TableScan)
OP_ENUM_MACRO(DataBlocksOptScan)
OP_ENUM_MACRO(TableSeqScan)
OP_ENUM_MACRO(TagScan)
OP_ENUM_MACRO(SystemTableScan)
OP_ENUM_MACRO(Aggregate)
OP_ENUM_MACRO(Project)
OP_ENUM_MACRO(Groupby)
OP_ENUM_MACRO(Limit)
OP_ENUM_MACRO(SLimit)
OP_ENUM_MACRO(TimeWindow)
OP_ENUM_MACRO(SessionWindow)
OP_ENUM_MACRO(StateWindow)
OP_ENUM_MACRO(Fill)
OP_ENUM_MACRO(MultiTableAggregate)
OP_ENUM_MACRO(MultiTableTimeInterval)
OP_ENUM_MACRO(Filter)
OP_ENUM_MACRO(Distinct)
OP_ENUM_MACRO(Join)
OP_ENUM_MACRO(AllTimeWindow)
OP_ENUM_MACRO(AllMultiTableTimeInterval)
OP_ENUM_MACRO(Order)
OP_ENUM_MACRO(Exchange)

27
include/libs/query/query.h
View File

@ -21,6 +21,8 @@ extern "C" {
#endif
#include "tarray.h"
#include "thash.h"
#include "tlog.h"
typedef SVgroupListRspMsg SVgroupListInfo;
@ -63,16 +65,14 @@ typedef struct STableMeta {
typedef struct SDBVgroupInfo {
int32_t vgroupVersion;
SArray *vgId;
int32_t hashRange;
int32_t hashType;
int32_t vgVersion;
int8_t hashMethod;
SHashObj *vgInfo; //key:vgId, value:SVgroupInfo
} SDBVgroupInfo;
typedef struct SUseDbOutput {
SVgroupListInfo *vgroupList;
char db[TSDB_TABLE_FNAME_LEN];
SDBVgroupInfo *dbVgroup;
char db[TSDB_FULL_DB_NAME_LEN];
SDBVgroupInfo dbVgroup;
} SUseDbOutput;
typedef struct STableMetaOutput {
@ -86,6 +86,19 @@ typedef struct STableMetaOutput {
extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize);
extern void msgInit();
extern int32_t qDebugFlag;
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}

34
include/libs/scheduler/scheduler.h
View File

@ -20,6 +20,12 @@
extern "C" {
#endif
#include "planner.h"
typedef struct SSchedulerCfg {
} SSchedulerCfg;
typedef struct SQueryProfileSummary {
int64_t startTs; // Object created and added into the message queue
int64_t endTs; // the timestamp when the task is completed
@ -43,43 +49,23 @@ typedef struct SQueryProfileSummary {
uint64_t resultSize; // generated result size in Kb.
} SQueryProfileSummary;
typedef struct SQueryTask {
uint64_t queryId; // query id
uint64_t taskId; // task id
char *pSubplan; // operator tree
uint64_t status; // task status
SQueryProfileSummary summary; // task execution summary
void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage
} SQueryTask;
typedef struct SQueryJob {
SArray **pSubtasks;
// todo
} SQueryJob;
/**
* Process the query job, generated according to the query physical plan.
* This is a synchronized API, and is also thread-safety.
* @param pJob
* @return
*/
int32_t qProcessQueryJob(struct SQueryJob* pJob);
int32_t scheduleQueryJob(SQueryDag* pDag, void** pJob);
int32_t scheduleFetchRows(void *pJob, void *data);
/**
* The SSqlObj should not be here????
* @param pSql
* @param pVgroupId
* @param pRetVgroupId
* @return
*/
//SArray* qGetInvolvedVgroupIdList(struct SSqlObj* pSql, SArray* pVgroupId, SArray* pRetVgroupId);
/**
* Cancel query job
* @param pJob
* @return
*/
int32_t qKillQueryJob(struct SQueryJob* pJob);
int32_t scheduleCancelJob(void *pJob);
#ifdef __cplusplus
}

128
include/libs/wal/wal.h
View File

@ -16,49 +16,75 @@
#define _TD_WAL_H_
#include "os.h"
#include "tarray.h"
#include "tdef.h"
#include "tlog.h"
#include "tarray.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int32_t wDebugFlag;
#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }}
#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }}
#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }}
#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }}
#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }}
#define wFatal(...) \
{ \
if (wDebugFlag & DEBUG_FATAL) { \
taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \
} \
}
#define wError(...) \
{ \
if (wDebugFlag & DEBUG_ERROR) { \
taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \
} \
}
#define wWarn(...) \
{ \
if (wDebugFlag & DEBUG_WARN) { \
taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \
} \
}
#define wInfo(...) \
{ \
if (wDebugFlag & DEBUG_INFO) { \
taosPrintLog("WAL ", 255, __VA_ARGS__); \
} \
}
#define wDebug(...) \
{ \
if (wDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
}
#define wTrace(...) \
{ \
if (wDebugFlag & DEBUG_TRACE) { \
taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \
} \
}
#define WAL_HEAD_VER 0
#define WAL_HEAD_VER 0
#define WAL_NOSUFFIX_LEN 20
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1)
#define WAL_LOG_SUFFIX "log"
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
#define WAL_LOG_SUFFIX "log"
#define WAL_INDEX_SUFFIX "idx"
#define WAL_REFRESH_MS 1000
#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_REFRESH_MS 1000
#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_CUR_FAILED 1
#define WAL_CUR_FAILED 1
#pragma pack(push, 1)
typedef enum {
TAOS_WAL_NOLOG = 0,
TAOS_WAL_WRITE = 1,
TAOS_WAL_FSYNC = 2
} EWalType;
typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType;
typedef struct SWalReadHead {
int8_t headVer;
uint8_t msgType;
int8_t reserved[2];
int32_t len;
int64_t ingestTs; //not implemented
int64_t version;
char body[];
int8_t headVer;
uint8_t msgType;
int8_t reserved[2];
int32_t len;
int64_t ingestTs; // not implemented
int64_t version;
char body[];
} SWalReadHead;
typedef struct {
@ -68,12 +94,12 @@ typedef struct {
int32_t rollPeriod; // secs
int64_t retentionSize;
int64_t segSize;
EWalType level; // wal level
EWalType level; // wal level
} SWalCfg;
typedef struct {
uint32_t cksumHead;
uint32_t cksumBody;
uint32_t cksumHead;
uint32_t cksumBody;
SWalReadHead head;
} SWalHead;
@ -89,37 +115,37 @@ typedef struct SWal {
// cfg
SWalCfg cfg;
int32_t fsyncSeq;
//meta
// meta
SWalVer vers;
int64_t writeLogTfd;
int64_t writeIdxTfd;
int32_t writeCur;
SArray* fileInfoSet;
//status
SArray *fileInfoSet;
// status
int64_t totSize;
int64_t lastRollSeq;
//ctl
int64_t refId;
// ctl
int64_t refId;
pthread_mutex_t mutex;
//path
// path
char path[WAL_PATH_LEN];
//reusable write head
// reusable write head
SWalHead writeHead;
} SWal; // WAL HANDLE
typedef struct SWalReadHandle {
SWal* pWal;
int64_t readLogTfd;
int64_t readIdxTfd;
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;
int64_t status; //if cursor valid
SWalHead* pHead;
SWal *pWal;
int64_t readLogTfd;
int64_t readIdxTfd;
int64_t curFileFirstVer;
int64_t curVersion;
int64_t capacity;
int64_t status; // if cursor valid
SWalHead *pHead;
} SWalReadHandle;
#pragma pack(pop)
//typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
// typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
// module initialization
int32_t walInit();
@ -141,15 +167,15 @@ int32_t walRollback(SWal *, int64_t ver);
// notify that previous logs can be pruned safely
int32_t walBeginSnapshot(SWal *, int64_t ver);
int32_t walEndSnapshot(SWal *);
//int32_t walDataCorrupted(SWal*);
// int32_t walDataCorrupted(SWal*);
// read
SWalReadHandle* walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
SWalReadHandle *walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
int32_t walRead(SWal *, SWalHead **, int64_t ver);
//int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
// int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum);
// lifecycle check
int64_t walGetFirstVer(SWal *);

1
include/os/os.h
View File

@ -46,6 +46,7 @@ extern "C" {
#include <math.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <dirent.h>
#include <unistd.h>

12
include/os/osSysinfo.h
View File

@ -20,6 +20,8 @@
extern "C" {
#endif
#include "os.h"
#define TSDB_LOCALE_LEN 64
#define TSDB_TIMEZONE_LEN 96
@ -57,11 +59,11 @@ char * taosGetCmdlineByPID(int pid);
void taosSetCoreDump(bool enable);
typedef struct {
const char *sysname;
const char *nodename;
const char *release;
const char *version;
const char *machine;
char sysname[_UTSNAME_MACHINE_LENGTH];
char nodename[_UTSNAME_MACHINE_LENGTH];
char release[_UTSNAME_MACHINE_LENGTH];
char version[_UTSNAME_MACHINE_LENGTH];
char machine[_UTSNAME_MACHINE_LENGTH];
} SysNameInfo;
SysNameInfo taosGetSysNameInfo();

1
include/util/taoserror.h
View File

@ -324,6 +324,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica")
#define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) //"invalid time condition")
#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) //"System error")
#define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) //"invalid input")
// grant

2
include/util/tarray.h
View File

@ -41,7 +41,7 @@ typedef struct SArray {
* @param elemSize
* @return
*/
void* taosArrayInit(size_t size, size_t elemSize);
SArray* taosArrayInit(size_t size, size_t elemSize);
/**
*

7
include/util/tdef.h
View File

@ -196,6 +196,7 @@ do { \
#define TSDB_AUTH_LEN 16
#define TSDB_PASSWORD_LEN 32
#define TSDB_USET_PASSWORD_LEN 129
#define TSDB_VERSION_LEN 12
#define TSDB_LABEL_LEN 8
@ -232,8 +233,8 @@ do { \
#define TSDB_CQ_SQL_SIZE 1024
#define TSDB_MIN_VNODES 64
#define TSDB_MAX_VNODES 512
#define TSDB_MIN_VNODES_PER_DB 2
#define TSDB_MAX_VNODES_PER_DB 64
#define TSDB_MIN_VNODES_PER_DB 1
#define TSDB_MAX_VNODES_PER_DB 4096
#define TSDB_DNODE_ROLE_ANY 0
#define TSDB_DNODE_ROLE_MGMT 1
@ -246,7 +247,7 @@ do { \
#define TSDB_RES_COL_ID (-5000)
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta
#define TSDB_MIN_CACHE_BLOCK_SIZE 1
#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode
#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16

42
include/util/tmacro.h
View File

@ -1,42 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_MACRO_H_
#define _TD_UTIL_MACRO_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
// Module init/clear MACRO definitions
#define TD_MOD_UNINITIALIZED 0
#define TD_MOD_INITIALIZED 1
#define TD_MOD_UNCLEARD 0
#define TD_MOD_CLEARD 1
typedef int8_t td_mode_flag_t;
#define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED)
#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD)
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_MACRO_H_*/

2
source/client/CMakeLists.txt
View File

@ -9,7 +9,7 @@ target_link_libraries(
taos
PRIVATE common
INTERFACE api
PRIVATE os util common transport parser
PRIVATE os util common transport parser catalog function query
)
ADD_SUBDIRECTORY(test)

34
source/client/inc/clientInt.h
View File

@ -21,13 +21,14 @@ extern "C" {
#endif
#include "taos.h"
#include "common.h"
#include "taosmsg.h"
#include "tdef.h"
#include "tep.h"
#include "thash.h"
#include "tlist.h"
#include "trpc.h"
#include "tdef.h"
#include "tmsgtype.h"
#include "tep.h"
#include "trpc.h"
typedef struct SQueryExecMetric {
int64_t start; // start timestamp
@ -86,12 +87,29 @@ typedef struct STscObj {
SAppInstInfo *pAppInfo;
} STscObj;
typedef struct SClientResultInfo {
const char *pMsg;
const char *pData;
TAOS_FIELD *fields;
int32_t numOfCols;
int32_t numOfRows;
int32_t current;
int32_t *length;
TAOS_ROW row;
char **pCol;
} SClientResultInfo;
typedef struct SReqBody {
tsem_t rspSem; // not used now
void* fp;
void* param;
int32_t paramLen;
int64_t execId; // showId/queryId
SClientResultInfo* pResInfo;
} SRequestBody;
#define ERROR_MSG_BUF_DEFAULT_SIZE 512
typedef struct SRequestObj {
uint64_t requestId;
int32_t type; // request type
@ -118,7 +136,7 @@ extern int32_t tscReqRef;
extern void *tscQhandle;
extern int32_t tscConnRef;
extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg);
extern int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody);
extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen);
int taos_init();
@ -129,8 +147,6 @@ void destroyTscObj(void*pObj);
void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type);
void destroyRequest(SRequestObj* pRequest);
TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
void taos_init_imp(void);
int taos_options_imp(TSDB_OPTION option, const char *str);
@ -139,6 +155,12 @@ void* openTransporter(const char *user, const char *auth);
void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet);
void initMsgHandleFp();
TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port);
TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
void* doFetchRow(SRequestObj* pRequest);
void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
#ifdef __cplusplus
}
#endif

4
source/client/inc/tscLog.h → source/client/inc/clientLog.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCLOG_H
#define TDENGINE_TSCLOG_H
#ifndef TDENGINE_CLIENTLOG_H
#define TDENGINE_CLIENTLOG_H
#ifdef __cplusplus
extern "C" {

63
source/client/src/client.c
View File

@ -1,63 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tdef.h"
#include "tglobal.h"
#include "clientInt.h"
#include "tscLog.h"
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
int32_t p = (port != 0)? port:tsServerPort;
tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
if (user == NULL) {
user = TSDB_DEFAULT_USER;
}
if (pass == NULL) {
pass = TSDB_DEFAULT_PASS;
}
return taos_connect_internal(ip, user, pass, NULL, db, p);
}
TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) {
tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db);
if (user == NULL) {
user = TSDB_DEFAULT_USER;
}
if (auth == NULL) {
tscError("No auth info is given, failed to connect to server");
return NULL;
}
return taos_connect_internal(ip, user, NULL, auth, db, port);
}
TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) {
char ipStr[TSDB_EP_LEN] = {0};
char dbStr[TSDB_DB_NAME_LEN] = {0};
char userStr[TSDB_USER_LEN] = {0};
char passStr[TSDB_PASSWORD_LEN] = {0};
strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen));
strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen));
strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen));
strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen));
return taos_connect(ipStr, userStr, passStr, dbStr, port);
}

182
source/client/src/clientImpl.c
View File

@ -1,15 +1,18 @@
#include <tpagedfile.h>
#include "clientInt.h"
#include "clientLog.h"
#include "tdef.h"
#include "tep.h"
#include "tglobal.h"
#include "tmsgtype.h"
#include "tnote.h"
#include "tpagedfile.h"
#include "tref.h"
#include "tscLog.h"
#include "parser.h"
static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody);
static void destroyConnectMsg(SRequestMsgBody* pMsgBody);
static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody);
static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId);
@ -96,17 +99,84 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
if (pInst == NULL) {
pInst = calloc(1, sizeof(struct SAppInstInfo));
SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo));
pInst->mgmtEp = epSet;
pInst->pTransporter = openTransporter(user, secretEncrypt);
p->mgmtEp = epSet;
p->pTransporter = openTransporter(user, secretEncrypt);
taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES);
taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES);
pInst = &p;
}
return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst);
tfree(key);
return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst);
}
TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
STscObj *pTscObj = (STscObj *)taos;
if (sqlLen > (size_t) tsMaxSQLStringLen) {
tscError("sql string exceeds max length:%d", tsMaxSQLStringLen);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
return NULL;
}
nPrintTsc("%s", sql)
SRequestObj* pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT);
if (pRequest == NULL) {
tscError("failed to malloc sqlObj");
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return NULL;
}
pRequest->sqlstr = malloc(sqlLen + 1);
if (pRequest->sqlstr == NULL) {
tscError("0x%"PRIx64" failed to prepare sql string buffer", pRequest->self);
pRequest->msgBuf = strdup("failed to prepare sql string buffer");
terrno = pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return pRequest;
}
strntolower(pRequest->sqlstr, sql, (int32_t)sqlLen);
pRequest->sqlstr[sqlLen] = 0;
tscDebugL("0x%"PRIx64" SQL: %s", pRequest->requestId, pRequest->sqlstr);
int32_t code = 0;
if (qIsInsertSql(pRequest->sqlstr, sqlLen)) {
// todo add
} else {
int32_t type = 0;
void* output = NULL;
int32_t outputLen = 0;
code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE);
if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW) {
pRequest->type = type;
pRequest->body.param = output;
pRequest->body.paramLen = outputLen;
SRequestMsgBody body = {0};
buildRequestMsgFp[type](pRequest, &body);
int64_t transporterId = 0;
sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
tsem_wait(&pRequest->body.rspSem);
destroyRequestMsgBody(&body);
} else {
assert(0);
}
}
if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code;
return pRequest;
}
return pRequest;
}
int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) {
@ -166,7 +236,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con
sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
tsem_wait(&pRequest->body.rspSem);
destroyConnectMsg(&body);
destroyRequestMsgBody(&body);
if (pRequest->code != TSDB_CODE_SUCCESS) {
const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno);
@ -213,7 +283,7 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody)
return 0;
}
static void destroyConnectMsg(SRequestMsgBody* pMsgBody) {
static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody) {
assert(pMsgBody != NULL);
tfree(pMsgBody->pData);
}
@ -269,11 +339,18 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType],
tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start);
if (handleRequestRspFp[pRequest->type]) {
pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen);
char *p = malloc(pMsg->contLen);
if (p == NULL) {
pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
terrno = pRequest->code;
} else {
memcpy(p, pMsg->pCont, pMsg->contLen);
pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, p, pMsg->contLen);
}
}
} else {
tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType],
tstrerror(pMsg->code), pMsg->contLen);
tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%"PRId64" ms", pRequest->requestId, taosMsg[pMsg->msgType],
tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start);
}
taosReleaseRef(tscReqRef, requestRefId);
@ -281,3 +358,80 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
sem_post(&pRequest->body.rspSem);
}
TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) {
tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db);
if (user == NULL) {
user = TSDB_DEFAULT_USER;
}
if (auth == NULL) {
tscError("No auth info is given, failed to connect to server");
return NULL;
}
return taos_connect_internal(ip, user, NULL, auth, db, port);
}
TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) {
char ipStr[TSDB_EP_LEN] = {0};
char dbStr[TSDB_DB_NAME_LEN] = {0};
char userStr[TSDB_USER_LEN] = {0};
char passStr[TSDB_PASSWORD_LEN] = {0};
strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen));
strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen));
strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen));
strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen));
return taos_connect(ipStr, userStr, passStr, dbStr, port);
}
void* doFetchRow(SRequestObj* pRequest) {
assert(pRequest != NULL);
SClientResultInfo* pResultInfo = pRequest->body.pResInfo;
if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) {
pRequest->type = TSDB_SQL_RETRIEVE_MNODE;
SRequestMsgBody body = {0};
buildRequestMsgFp[pRequest->type](pRequest, &body);
int64_t transporterId = 0;
STscObj* pTscObj = pRequest->pTscObj;
sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
tsem_wait(&pRequest->body.rspSem);
destroyRequestMsgBody(&body);
pResultInfo->current = 0;
if (pResultInfo->numOfRows <= pResultInfo->current) {
return NULL;
}
}
for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) {
pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current;
if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
pResultInfo->length[i] = varDataLen(pResultInfo->row[i]);
pResultInfo->row[i] = varDataVal(pResultInfo->row[i]);
}
}
pResultInfo->current += 1;
return pResultInfo->row;
}
void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
if (numOfRows == 0) {
return;
}
int32_t offset = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
pResultInfo->length[i] = pResultInfo->fields[i].bytes;
pResultInfo->row[i] = pResultInfo->pData + offset * pResultInfo->numOfRows;
pResultInfo->pCol[i] = pResultInfo->row[i];
offset += pResultInfo->fields[i].bytes;
}
}

269
source/client/src/clientMain.c Normal file
View File

@ -0,0 +1,269 @@
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "taosmsg.h"
#include "tcache.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tnote.h"
#include "tref.h"
#include "trpc.h"
#include "tsched.h"
#include "ttime.h"
#include "ttimezone.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
static int32_t sentinel = TSC_VAR_NOT_RELEASE;
int taos_options(TSDB_OPTION option, const void *arg, ...) {
static int32_t lock = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
tscInfo("haven't acquire lock after spin %d times.", i);
sched_yield();
}
}
int ret = taos_options_imp(option, (const char*)arg);
atomic_store_32(&lock, 0);
return ret;
}
// this function may be called by user or system, or by both simultaneously.
void taos_cleanup(void) {
tscDebug("start to cleanup client environment");
if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
return;
}
int32_t id = tscReqRef;
tscReqRef = -1;
taosCloseRef(id);
void* p = tscQhandle;
tscQhandle = NULL;
taosCleanUpScheduler(p);
id = tscConnRef;
tscConnRef = -1;
taosCloseRef(id);
rpcCleanup();
taosCloseLog();
}
TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) {
int32_t p = (port != 0) ? port : tsServerPort;
tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db);
if (user == NULL) {
user = TSDB_DEFAULT_USER;
}
if (pass == NULL) {
pass = TSDB_DEFAULT_PASS;
}
return taos_connect_internal(ip, user, pass, NULL, db, p);
}
void taos_close(TAOS* taos) {
if (taos == NULL) {
return;
}
STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
taosRemoveRef(tscConnRef, pTscObj->id);
}
int taos_errno(TAOS_RES *tres) {
if (tres == NULL) {
return terrno;
}
return ((SRequestObj*) tres)->code;
}
const char *taos_errstr(TAOS_RES *res) {
SRequestObj *pRequest = (SRequestObj *) res;
if (pRequest == NULL) {
return (const char*) tstrerror(terrno);
}
if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) {
return pRequest->msgBuf;
} else {
return (const char*)tstrerror(pRequest->code);
}
}
void taos_free_result(TAOS_RES *res) {
SRequestObj* pRequest = (SRequestObj*) res;
destroyRequest(pRequest);
}
int taos_field_count(TAOS_RES *res) {
if (res == NULL) {
return 0;
}
SRequestObj* pRequest = (SRequestObj*) res;
SClientResultInfo* pResInfo = pRequest->body.pResInfo;
if (pResInfo == NULL) {
return 0;
}
return pResInfo->numOfCols;
}
int taos_num_fields(TAOS_RES *res) {
return taos_field_count(res);
}
TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
if (taos_num_fields(res) == 0) {
return NULL;
}
SClientResultInfo* pResInfo = ((SRequestObj*) res)->body.pResInfo;
return pResInfo->fields;
}
TAOS_RES *taos_query(TAOS *taos, const char *sql) {
if (taos == NULL || sql == NULL) {
return NULL;
}
return taos_query_l(taos, sql, strlen(sql));
}
TAOS_ROW taos_fetch_row(TAOS_RES *pRes) {
if (pRes == NULL) {
return NULL;
}
SRequestObj *pRequest = (SRequestObj *) pRes;
if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
pRequest->type == TSDB_SQL_INSERT) {
return NULL;
}
return doFetchRow(pRequest);
}
int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) {
int32_t len = 0;
for (int i = 0; i < num_fields; ++i) {
if (i > 0) {
str[len++] = ' ';
}
if (row[i] == NULL) {
len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR);
continue;
}
switch (fields[i].type) {
case TSDB_DATA_TYPE_TINYINT:
len += sprintf(str + len, "%d", *((int8_t *)row[i]));
break;
case TSDB_DATA_TYPE_UTINYINT:
len += sprintf(str + len, "%u", *((uint8_t *)row[i]));
break;
case TSDB_DATA_TYPE_SMALLINT:
len += sprintf(str + len, "%d", *((int16_t *)row[i]));
break;
case TSDB_DATA_TYPE_USMALLINT:
len += sprintf(str + len, "%u", *((uint16_t *)row[i]));
break;
case TSDB_DATA_TYPE_INT:
len += sprintf(str + len, "%d", *((int32_t *)row[i]));
break;
case TSDB_DATA_TYPE_UINT:
len += sprintf(str + len, "%u", *((uint32_t *)row[i]));
break;
case TSDB_DATA_TYPE_BIGINT:
len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_UBIGINT:
len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i]));
break;
case TSDB_DATA_TYPE_FLOAT: {
float fv = 0;
fv = GET_FLOAT_VAL(row[i]);
len += sprintf(str + len, "%f", fv);
} break;
case TSDB_DATA_TYPE_DOUBLE: {
double dv = 0;
dv = GET_DOUBLE_VAL(row[i]);
len += sprintf(str + len, "%lf", dv);
} break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: {
int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE);
if (fields[i].type == TSDB_DATA_TYPE_BINARY) {
assert(charLen <= fields[i].bytes && charLen >= 0);
} else {
assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0);
}
memcpy(str + len, row[i], charLen);
len += charLen;
} break;
case TSDB_DATA_TYPE_TIMESTAMP:
len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i]));
break;
case TSDB_DATA_TYPE_BOOL:
len += sprintf(str + len, "%d", *((int8_t *)row[i]));
default:
break;
}
}
return len;
}
int* taos_fetch_lengths(TAOS_RES *res) {
if (res == NULL) {
return NULL;
}
return ((SRequestObj*) res)->body.pResInfo->length;
}
const char *taos_data_type(int type) {
switch (type) {
case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL";
case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL";
case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT";
case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT";
case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT";
case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT";
case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT";
case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE";
case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY";
case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP";
case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR";
default: return "UNKNOWN";
}
}

187
source/client/src/clientMsgHandler.c
View File

@ -13,11 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include <catalog.h>
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "tmsgtype.h"
#include "trpc.h"
#include "tscLog.h"
int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0};
int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen);
@ -528,7 +529,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) {
if (pCmd->command == TSDB_SQL_SELECT ||
pCmd->command == TSDB_SQL_FETCH ||
pCmd->command == TSDB_SQL_RETRIEVE ||
pCmd->command == TSDB_SQL_RETRIEVE_MNODE ||
pCmd->command == TSDB_SQL_INSERT ||
pCmd->command == TSDB_SQL_CONNECT ||
pCmd->command == TSDB_SQL_HB ||
@ -2699,7 +2700,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) {
}
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if ((pCmd->command == TSDB_SQL_RETRIEVE) ||
if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) ||
((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) &&
!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) ||
(tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) &&
@ -3091,6 +3092,36 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
#endif
int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT;
pMsgBody->msgLen = sizeof(SConnectMsg);
pMsgBody->requestObjRefId = pRequest->self;
SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));
if (pConnect == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return -1;
}
// TODO refactor full_name
char *db; // ugly code to move the space
STscObj *pObj = pRequest->pTscObj;
pthread_mutex_lock(&pObj->mutex);
db = strstr(pObj->db, TS_PATH_DELIMITER);
db = (db == NULL) ? pObj->db : db + 1;
tstrncpy(pConnect->db, db, sizeof(pConnect->db));
pthread_mutex_unlock(&pObj->mutex);
pConnect->pid = htonl(appInfo.pid);
pConnect->startTime = htobe64(appInfo.startTime);
tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app));
pMsgBody->pData = pConnect;
return 0;
}
int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
STscObj *pTscObj = pRequest->pTscObj;
@ -3099,6 +3130,11 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
pConnect->connId = htonl(pConnect->connId);
pConnect->clusterId = htonl(pConnect->clusterId);
assert(pConnect->epSet.numOfEps > 0);
for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) {
pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]);
}
// TODO refactor
pthread_mutex_lock(&pTscObj->mutex);
char temp[TSDB_TABLE_FNAME_LEN * 2] = {0};
@ -3108,13 +3144,12 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db));
pthread_mutex_unlock(&pTscObj->mutex);
assert(pConnect->epSet.numOfEps > 0);
if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) {
updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet);
}
for (int i = 0; i < pConnect->epSet.numOfEps; ++i) {
tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id);
tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id);
}
pTscObj->connId = pConnect->connId;
@ -3123,14 +3158,132 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
pTscObj->pAppInfo->clusterId = pConnect->clusterId;
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns);
// createHbObj(pTscObj);
pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo));
pRequest->body.pResInfo->pMsg = pMsg;
// launch a timer to send heartbeat to maintain the connection and send status to mnode
// taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer);
tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns);
return 0;
}
int32_t buildCreateUserMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER;
pMsgBody->msgLen = sizeof(SCreateUserMsg);
pMsgBody->requestObjRefId = pRequest->self;
pMsgBody->pData = pRequest->body.param;
return 0;
}
int32_t buildShowMsg(SRequestObj* pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_SHOW;
pMsgBody->msgLen = pRequest->body.paramLen;
pMsgBody->requestObjRefId = pRequest->self;
pMsgBody->pData = pRequest->body.param;
}
STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2);
size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
pTableMeta->tableType = pTableMetaMsg->tableType;
pTableMeta->vgId = pTableMetaMsg->vgId;
pTableMeta->suid = pTableMetaMsg->suid;
pTableMeta->uid = pTableMetaMsg->tuid;
pTableMeta->tableInfo = (STableComInfo) {
.numOfTags = pTableMetaMsg->numOfTags,
.precision = pTableMetaMsg->precision,
.numOfColumns = pTableMetaMsg->numOfColumns,
};
pTableMeta->sversion = pTableMetaMsg->sversion;
pTableMeta->tversion = pTableMetaMsg->tversion;
memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize);
int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns;
for(int32_t i = 0; i < numOfTotalCols; ++i) {
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
return pTableMeta;
}
int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
SShowRsp* pShow = (SShowRsp *)pMsg;
pShow->showId = htonl(pShow->showId);
STableMetaMsg *pMetaMsg = &(pShow->tableMeta);
pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns);
SSchema* pSchema = pMetaMsg->pSchema;
pMetaMsg->tuid = htobe64(pMetaMsg->tuid);
for (int i = 0; i < pMetaMsg->numOfColumns; ++i) {
pSchema->bytes = htonl(pSchema->bytes);
pSchema++;
}
pSchema = pMetaMsg->pSchema;
TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD));
for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) {
tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name));
pFields[i].type = pSchema[i].type;
pFields[i].bytes = pSchema[i].bytes;
}
if (pRequest->body.pResInfo == NULL) {
pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo));
}
pRequest->body.pResInfo->pMsg = pMsg;
SClientResultInfo* pResInfo = pRequest->body.pResInfo;
pResInfo->fields = pFields;
pResInfo->numOfCols = pMetaMsg->numOfColumns;
pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES);
pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES);
pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t));
pRequest->body.execId = pShow->showId;
return 0;
}
int buildRetrieveMnodeMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE;
pMsgBody->msgLen = sizeof(SRetrieveTableMsg);
pMsgBody->requestObjRefId = pRequest->self;
SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg));
pRetrieveMsg->showId = htonl(pRequest->body.execId);
pMsgBody->pData = pRetrieveMsg;
return TSDB_CODE_SUCCESS;
}
int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
assert(msgLen >= sizeof(SRetrieveTableRsp));
tfree(pRequest->body.pResInfo->pMsg);
pRequest->body.pResInfo->pMsg = pMsg;
SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg;
pRetrieve->numOfRows = htonl(pRetrieve->numOfRows);
pRetrieve->precision = htons(pRetrieve->precision);
SClientResultInfo* pResInfo = pRequest->body.pResInfo;
pResInfo->numOfRows = pRetrieve->numOfRows;
pResInfo->pData = pRetrieve->data; // todo fix this in async model
pResInfo->current = 0;
setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows);
tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows,
pRetrieve->completed, pRequest->body.execId);
return 0;
}
void initMsgHandleFp() {
#if 0
tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg;
@ -3167,7 +3320,7 @@ void initMsgHandleFp() {
tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg;
tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg;
tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg;
tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg;
tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg;
tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg;
tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg;
@ -3185,7 +3338,7 @@ void initMsgHandleFp() {
tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp;
tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp;
tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function.
tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function.
tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp;
tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp;
@ -3207,6 +3360,14 @@ void initMsgHandleFp() {
tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp;
#endif
// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg;
buildRequestMsgFp[TSDB_SQL_CONNECT] = buildConnectMsg;
handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp;
buildRequestMsgFp[TSDB_SQL_CREATE_USER] = buildCreateUserMsg;
buildRequestMsgFp[TSDB_SQL_SHOW] = buildShowMsg;
handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp;
buildRequestMsgFp[TSDB_SQL_RETRIEVE_MNODE] = buildRetrieveMnodeMsg;
handleRequestRspFp[TSDB_SQL_RETRIEVE_MNODE]= processRetrieveMnodeRsp;
}

85
source/client/src/clientmain.c
View File

@ -1,85 +0,0 @@
#include "clientInt.h"
#include "trpc.h"
#include "os.h"
#include "taosmsg.h"
#include "tcache.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tnote.h"
#include "tref.h"
#include "tscLog.h"
#include "tsched.h"
#include "ttime.h"
#include "ttimezone.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
static int32_t sentinel = TSC_VAR_NOT_RELEASE;
static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
extern int32_t tscInitRes;
int taos_options(TSDB_OPTION option, const void *arg, ...) {
static int32_t lock = 0;
for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) {
if (i % 1000 == 0) {
tscInfo("haven't acquire lock after spin %d times.", i);
sched_yield();
}
}
int ret = taos_options_imp(option, (const char*)arg);
atomic_store_32(&lock, 0);
return ret;
}
int taos_init() {
pthread_once(&tscinit, taos_init_imp);
return tscInitRes;
}
// this function may be called by user or system, or by both simultaneously.
void taos_cleanup(void) {
tscDebug("start to cleanup client environment");
if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
return;
}
int32_t id = tscReqRef;
tscReqRef = -1;
taosCloseRef(id);
void* p = tscQhandle;
tscQhandle = NULL;
taosCleanUpScheduler(p);
id = tscConnRef;
tscConnRef = -1;
taosCloseRef(id);
rpcCleanup();
taosCloseLog();
}
void taos_close(TAOS* taos) {
if (taos == NULL) {
return;
}
STscObj *pTscObj = (STscObj *)taos;
tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs);
taosRemoveRef(tscConnRef, pTscObj->id);
}
const char *taos_errstr(TAOS_RES *res) {
}
void taos_free_result(TAOS_RES *res) {
}

23
source/client/src/tscEnv.c
View File

@ -13,6 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "taosmsg.h"
#include "tcache.h"
@ -20,12 +22,10 @@
#include "tglobal.h"
#include "tnote.h"
#include "tref.h"
#include "tscLog.h"
#include "trpc.h"
#include "tsched.h"
#include "ttime.h"
#include "trpc.h"
#include "ttimezone.h"
#include "clientInt.h"
#define TSC_VAR_NOT_RELEASE 1
#define TSC_VAR_RELEASED 0
@ -35,6 +35,7 @@ int32_t tscReqRef = -1;
int32_t tscConnRef = -1;
void *tscQhandle = NULL;
static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
int32_t tsNumOfThreads = 1;
volatile int32_t tscInitRes = 0;
@ -64,7 +65,7 @@ static void deregisterRequest(SRequestObj* pRequest) {
SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary;
int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1);
int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1);
tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst);
taosReleaseRef(tscConnRef, pTscObj->id);
@ -126,8 +127,6 @@ void destroyTscObj(void *pObj) {
atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns);
closeTransporter(pTscObj);
pthread_mutex_destroy(&pTscObj->mutex);
tfree(pTscObj);
}
@ -172,6 +171,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
pRequest->pTscObj = pObj;
pRequest->body.fp = fp;
pRequest->body.param = param;
pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
tsem_init(&pRequest->body.rspSem, 0, 0);
registerRequest(pRequest);
@ -188,6 +188,12 @@ static void doDestroyRequest(void* p) {
tfree(pRequest->sqlstr);
tfree(pRequest->pInfo);
if (pRequest->body.pResInfo != NULL) {
tfree(pRequest->body.pResInfo->pData);
tfree(pRequest->body.pResInfo->pMsg);
tfree(pRequest->body.pResInfo);
}
deregisterRequest(pRequest);
tfree(pRequest);
}
@ -250,6 +256,11 @@ void taos_init_imp(void) {
tscDebug("client is initialized successfully");
}
int taos_init() {
pthread_once(&tscinit, taos_init_imp);
return tscInitRes;
}
int taos_options_imp(TSDB_OPTION option, const char *str) {
SGlobalCfg *cfg = NULL;

58
source/client/test/clientTests.cpp
View File

@ -14,6 +14,7 @@
*/
#include <gtest/gtest.h>
#include <taoserror.h>
#include <iostream>
#include "tglobal.h"
#pragma GCC diagnostic ignored "-Wwrite-strings"
@ -22,6 +23,7 @@
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "../inc/clientInt.h"
#include "taos.h"
namespace {
@ -33,8 +35,62 @@ int main(int argc, char** argv) {
}
TEST(testCase, driverInit_Test) {
taos_init();
}
TEST(testCase, connect_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
taos_close(pConn);
}
TEST(testCase, create_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
}
//TEST(testCase, show_user_Test) {
// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "show users");
// TAOS_ROW pRow = NULL;
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
//
// char str[512] = {0};
// while((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// printf("%s\n", str);
// }
//
// taos_close(pConn);
//}
TEST(testCase, show_db_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show databases");
TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0};
while((pRow = taos_fetch_row(pRes)) != NULL) {
int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
printf("%s\n", str);
}
taos_close(pConn);
}

3
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -44,8 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_ACK] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_RESET] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg;
// msg from client to mnode
pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg;

26
source/dnode/mgmt/impl/test/acct/acct.cpp
View File

@ -1,16 +1,12 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
/**
* @file vnodeApiTests.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module acct-msg tests
* @version 0.1
* @date 2021-12-15
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* @copyright Copyright (c) 2021
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"
@ -49,7 +45,7 @@ SServer* DndTestAcct::pServer;
SClient* DndTestAcct::pClient;
int32_t DndTestAcct::connId;
TEST_F(DndTestAcct, CreateAcct) {
TEST_F(DndTestAcct, 01_CreateAcct) {
ASSERT_NE(pClient, nullptr);
SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(sizeof(SCreateAcctMsg));
@ -65,7 +61,7 @@ TEST_F(DndTestAcct, CreateAcct) {
ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED);
}
TEST_F(DndTestAcct, AlterAcct) {
TEST_F(DndTestAcct, 02_AlterAcct) {
ASSERT_NE(pClient, nullptr);
SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(sizeof(SAlterAcctMsg));
@ -81,7 +77,7 @@ TEST_F(DndTestAcct, AlterAcct) {
ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED);
}
TEST_F(DndTestAcct, DropAcct) {
TEST_F(DndTestAcct, 03_DropAcct) {
ASSERT_NE(pClient, nullptr);
SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(sizeof(SDropAcctMsg));
@ -97,7 +93,7 @@ TEST_F(DndTestAcct, DropAcct) {
ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED);
}
TEST_F(DndTestAcct, ShowAcct) {
TEST_F(DndTestAcct, 04_ShowAcct) {
ASSERT_NE(pClient, nullptr);
SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg));

18
source/dnode/mgmt/impl/test/cluster/cluster.cpp
View File

@ -1,16 +1,12 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
/**
* @file vnodeApiTests.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module cluster-msg tests
* @version 0.1
* @date 2021-12-15
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* @copyright Copyright (c) 2021
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"

203
source/dnode/mgmt/impl/test/db/db.cpp
View File

@ -1,16 +1,12 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
/**
* @file vnodeApiTests.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module db-msg tests
* @version 0.1
* @date 2021-12-15
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* @copyright Copyright (c) 2021
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"
@ -30,7 +26,7 @@ class DndTestDb : public ::testing::Test {
const char* firstEp = "localhost:9040";
pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp);
pClient = createClient("root", "taosdata", fqdn, 9040);
taosMsleep(300);
taosMsleep(1100);
}
static void TearDownTestSuite() {
@ -48,11 +44,12 @@ class DndTestDb : public ::testing::Test {
void SetUp() override {}
void TearDown() override {}
void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) {
void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) {
SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg));
pShow->type = showType;
strcpy(pShow->db, "");
if (db != NULL) {
strcpy(pShow->db, db);
}
SRpcMsg showRpcMsg = {0};
showRpcMsg.pCont = pShow;
showRpcMsg.contLen = sizeof(SShowMsg);
@ -67,10 +64,10 @@ class DndTestDb : public ::testing::Test {
ASSERT_NE(pShowRsp, nullptr);
pShowRsp->showId = htonl(pShowRsp->showId);
pMeta = &pShowRsp->tableMeta;
pMeta->numOfTags = htons(pMeta->numOfTags);
pMeta->numOfColumns = htons(pMeta->numOfColumns);
pMeta->sversion = htons(pMeta->sversion);
pMeta->tversion = htons(pMeta->tversion);
pMeta->numOfTags = htonl(pMeta->numOfTags);
pMeta->numOfColumns = htonl(pMeta->numOfColumns);
pMeta->sversion = htonl(pMeta->sversion);
pMeta->tversion = htonl(pMeta->tversion);
pMeta->tuid = htobe64(pMeta->tuid);
pMeta->suid = htobe64(pMeta->suid);
@ -183,31 +180,33 @@ SClient* DndTestDb::pClient;
int32_t DndTestDb::connId;
TEST_F(DndTestDb, 01_ShowDb) {
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);
CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name");
CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time");
CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "replica");
CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "quorum");
CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "days");
CheckSchema(5, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2");
CheckSchema(6, TSDB_DATA_TYPE_INT, 4, "cache(MB)");
CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "blocks");
CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "minrows");
CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "maxrows");
CheckSchema(10, TSDB_DATA_TYPE_TINYINT, 1, "wallevel");
CheckSchema(11, TSDB_DATA_TYPE_INT, 4, "fsync");
CheckSchema(12, TSDB_DATA_TYPE_TINYINT, 1, "comp");
CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "cachelast");
CheckSchema(14, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision");
CheckSchema(15, TSDB_DATA_TYPE_TINYINT, 1, "update");
CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups");
CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica");
CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum");
CheckSchema(5, TSDB_DATA_TYPE_SMALLINT, 2, "days");
CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2");
CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "cache(MB)");
CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "blocks");
CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "minrows");
CheckSchema(10, TSDB_DATA_TYPE_INT, 4, "maxrows");
CheckSchema(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel");
CheckSchema(12, TSDB_DATA_TYPE_INT, 4, "fsync");
CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "comp");
CheckSchema(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast");
CheckSchema(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision");
CheckSchema(16, TSDB_DATA_TYPE_TINYINT, 1, "update");
SendThenCheckShowRetrieveMsg(0);
}
TEST_F(DndTestDb, 02_CreateDb) {
TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) {
{
SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg));
strcpy(pReq->db, "1.d1");
pReq->numOfVgroups = htonl(2);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
@ -238,10 +237,11 @@ TEST_F(DndTestDb, 02_CreateDb) {
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);
SendThenCheckShowRetrieveMsg(1);
CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
CheckTimestamp();
CheckInt16(2); // vgroups
CheckInt16(1); // replica
CheckInt16(1); // quorum
CheckInt16(10); // days
@ -256,9 +256,22 @@ TEST_F(DndTestDb, 02_CreateDb) {
CheckInt8(0); // cachelast
CheckBinary("ms", 3); // precision
CheckInt8(0); // update
}
TEST_F(DndTestDb, 03_AlterDb) {
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "show vgroups", 4, "1.d1");
CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "vgId");
CheckSchema(1, TSDB_DATA_TYPE_INT, 4, "tables");
CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode");
CheckSchema(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status");
SendThenCheckShowRetrieveMsg(2);
CheckInt32(1);
CheckInt32(2);
CheckInt32(0);
CheckInt32(0);
CheckInt16(1);
CheckInt16(1);
CheckBinary("master", 9);
CheckBinary("master", 9);
{
SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg));
strcpy(pReq->db, "1.d1");
@ -282,10 +295,11 @@ TEST_F(DndTestDb, 03_AlterDb) {
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);
SendThenCheckShowRetrieveMsg(1);
CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
CheckTimestamp();
CheckInt16(2); // vgroups
CheckInt16(1); // replica
CheckInt16(2); // quorum
CheckInt16(10); // days
@ -300,9 +314,8 @@ TEST_F(DndTestDb, 03_AlterDb) {
CheckInt8(1); // cachelast
CheckBinary("ms", 3); // precision
CheckInt8(0); // update
}
TEST_F(DndTestDb, 04_RestartDnode) {
// restart
stopServer(pServer);
pServer = NULL;
@ -314,10 +327,11 @@ TEST_F(DndTestDb, 04_RestartDnode) {
uInfo("all server is running");
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);
SendThenCheckShowRetrieveMsg(1);
CheckBinary("d1", TSDB_DB_NAME_LEN - 1);
CheckTimestamp();
CheckInt16(2); // vgroups
CheckInt16(1); // replica
CheckInt16(2); // quorum
CheckInt16(10); // days
@ -332,11 +346,9 @@ TEST_F(DndTestDb, 04_RestartDnode) {
CheckInt8(1); // cachelast
CheckBinary("ms", 3); // precision
CheckInt8(0); // update
}
TEST_F(DndTestDb, 05_DropDb) {
{
SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg));
SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SDropDbMsg));
strcpy(pReq->db, "1.d1");
SRpcMsg rpcMsg = {0};
@ -350,6 +362,103 @@ TEST_F(DndTestDb, 05_DropDb) {
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16);
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);
SendThenCheckShowRetrieveMsg(0);
}
}
TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) {
{
SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg));
strcpy(pReq->db, "1.d2");
pReq->numOfVgroups = htonl(2);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replications = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->ignoreExist = 1;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateDbMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL);
SendThenCheckShowRetrieveMsg(1);
CheckBinary("d2", TSDB_DB_NAME_LEN - 1);
{
SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(sizeof(SUseDbMsg));
strcpy(pReq->db, "1.d2");
pReq->vgVersion = htonl(-1);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SUseDbMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_USE_DB;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont;
EXPECT_STREQ(pRsp->db, "1.d2");
pRsp->vgVersion = htonl(pRsp->vgVersion);
pRsp->vgNum = htonl(pRsp->vgNum);
pRsp->hashMethod = pRsp->hashMethod;
EXPECT_EQ(pRsp->vgVersion, 1);
EXPECT_EQ(pRsp->vgNum, 2);
EXPECT_EQ(pRsp->hashMethod, 1);
{
SVgroupInfo* pInfo = &pRsp->vgroupInfo[0];
pInfo->vgId = htonl(pInfo->vgId);
pInfo->hashBegin = htonl(pInfo->hashBegin);
pInfo->hashEnd = htonl(pInfo->hashEnd);
EXPECT_GT(pInfo->vgId, 0);
EXPECT_EQ(pInfo->hashBegin, 0);
EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1);
EXPECT_EQ(pInfo->inUse, 0);
EXPECT_EQ(pInfo->numOfEps, 1);
SEpAddrMsg* pAddr = &pInfo->epAddr[0];
pAddr->port = htons(pAddr->port);
EXPECT_EQ(pAddr->port, 9040);
EXPECT_STREQ(pAddr->fqdn, "localhost");
}
{
SVgroupInfo* pInfo = &pRsp->vgroupInfo[1];
pInfo->vgId = htonl(pInfo->vgId);
pInfo->hashBegin = htonl(pInfo->hashBegin);
pInfo->hashEnd = htonl(pInfo->hashEnd);
EXPECT_GT(pInfo->vgId, 0);
EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2);
EXPECT_EQ(pInfo->hashEnd, UINT32_MAX);
EXPECT_EQ(pInfo->inUse, 0);
EXPECT_EQ(pInfo->numOfEps, 1);
SEpAddrMsg* pAddr = &pInfo->epAddr[0];
pAddr->port = htons(pAddr->port);
EXPECT_EQ(pAddr->port, 9040);
EXPECT_STREQ(pAddr->fqdn, "localhost");
}
}
}

79
source/dnode/mgmt/impl/test/dnode/dnode.cpp
View File

@ -1,16 +1,12 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
/**
* @file vnodeApiTests.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module dnode-msg tests
* @version 0.1
* @date 2021-12-15
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* @copyright Copyright (c) 2021
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"
@ -82,11 +78,11 @@ class DndTestDnode : public ::testing::Test {
ASSERT_NE(pShowRsp, nullptr);
pShowRsp->showId = htonl(pShowRsp->showId);
pMeta = &pShowRsp->tableMeta;
pMeta->numOfTags = htons(pMeta->numOfTags);
pMeta->numOfColumns = htons(pMeta->numOfColumns);
pMeta->sversion = htons(pMeta->sversion);
pMeta->numOfTags = htonl(pMeta->numOfTags);
pMeta->numOfColumns = htonl(pMeta->numOfColumns);
pMeta->sversion = htonl(pMeta->sversion);
pMeta->tversion = htons(pMeta->tversion);
pMeta->tuid = htobe64(pMeta->tuid);
pMeta->tuid = htonl(pMeta->tuid);
pMeta->suid = htobe64(pMeta->suid);
showId = pShowRsp->showId;
@ -224,19 +220,21 @@ TEST_F(DndTestDnode, 02_ConfigDnode) {
ASSERT_EQ(pMsg->code, 0);
}
TEST_F(DndTestDnode, 03_CreateDnode) {
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
strcpy(pReq->ep, "localhost:9042");
TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) {
{
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
strcpy(pReq->ep, "localhost:9042");
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateDnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateDnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
taosMsleep(1300);
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
@ -255,21 +253,21 @@ TEST_F(DndTestDnode, 03_CreateDnode) {
CheckTimestamp();
CheckBinary("", 24);
CheckBinary("", 24);
}
TEST_F(DndTestDnode, 04_DropDnode) {
SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg));
pReq->dnodeId = htonl(2);
{
SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg));
pReq->dnodeId = htonl(2);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SDropDnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SDropDnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7);
SendThenCheckShowRetrieveMsg(1);
@ -280,9 +278,7 @@ TEST_F(DndTestDnode, 04_DropDnode) {
CheckBinary("ready", 10);
CheckTimestamp();
CheckBinary("", 24);
}
TEST_F(DndTestDnode, 05_CreateDnode) {
{
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg));
strcpy(pReq->ep, "localhost:9043");
@ -359,9 +355,8 @@ TEST_F(DndTestDnode, 05_CreateDnode) {
CheckBinary("", 24);
CheckBinary("", 24);
CheckBinary("", 24);
}
TEST_F(DndTestDnode, 06_RestartDnode) {
// restart
uInfo("stop all server");
stopServer(pServer1);
stopServer(pServer2);

18
source/dnode/mgmt/impl/test/profile/profile.cpp
View File

@ -1,16 +1,12 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
/**
* @file vnodeApiTests.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module profile-msg tests
* @version 0.1
* @date 2021-12-15
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* @copyright Copyright (c) 2021
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"

18
source/dnode/mgmt/impl/test/show/show.cpp
View File

@ -1,16 +1,12 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
/**
* @file vnodeApiTests.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module show-msg tests
* @version 0.1
* @date 2021-12-15
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* @copyright Copyright (c) 2021
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"

83
source/dnode/mgmt/impl/test/user/user.cpp
View File

@ -1,16 +1,12 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
/**
* @file vnodeApiTests.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module user-msg tests
* @version 0.1
* @date 2021-12-15
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* @copyright Copyright (c) 2021
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "deploy.h"
@ -67,10 +63,10 @@ class DndTestUser : public ::testing::Test {
ASSERT_NE(pShowRsp, nullptr);
pShowRsp->showId = htonl(pShowRsp->showId);
pMeta = &pShowRsp->tableMeta;
pMeta->numOfTags = htons(pMeta->numOfTags);
pMeta->numOfColumns = htons(pMeta->numOfColumns);
pMeta->sversion = htons(pMeta->sversion);
pMeta->tversion = htons(pMeta->tversion);
pMeta->numOfTags = htonl(pMeta->numOfTags);
pMeta->numOfColumns = htonl(pMeta->numOfColumns);
pMeta->sversion = htonl(pMeta->sversion);
pMeta->tversion = htonl(pMeta->tversion);
pMeta->tuid = htobe64(pMeta->tuid);
pMeta->suid = htobe64(pMeta->suid);
@ -184,7 +180,7 @@ TEST_F(DndTestUser, 01_ShowUser) {
CheckBinary("root", TSDB_USER_LEN);
}
TEST_F(DndTestUser, 02_CreateUser) {
TEST_F(DndTestUser, 02_Create_Drop_Alter_User) {
{
SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg));
strcpy(pReq->user, "u1");
@ -231,23 +227,22 @@ TEST_F(DndTestUser, 02_CreateUser) {
CheckBinary("root", TSDB_USER_LEN);
CheckBinary("root", TSDB_USER_LEN);
CheckBinary("root", TSDB_USER_LEN);
}
TEST_F(DndTestUser, 03_AlterUser) {
SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg));
strcpy(pReq->user, "u1");
strcpy(pReq->pass, "p2");
{
SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg));
strcpy(pReq->user, "u1");
strcpy(pReq->pass, "p2");
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SAlterUserMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SAlterUserMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4);
SendThenCheckShowRetrieveMsg(3);
CheckBinary("u1", TSDB_USER_LEN);
@ -262,22 +257,21 @@ TEST_F(DndTestUser, 03_AlterUser) {
CheckBinary("root", TSDB_USER_LEN);
CheckBinary("root", TSDB_USER_LEN);
CheckBinary("root", TSDB_USER_LEN);
}
TEST_F(DndTestUser, 04_DropUser) {
SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg));
strcpy(pReq->user, "u1");
{
SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg));
strcpy(pReq->user, "u1");
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SDropUserMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SDropUserMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4);
SendThenCheckShowRetrieveMsg(2);
CheckBinary("root", TSDB_USER_LEN);
@ -288,9 +282,8 @@ TEST_F(DndTestUser, 04_DropUser) {
CheckTimestamp();
CheckBinary("root", TSDB_USER_LEN);
CheckBinary("root", TSDB_USER_LEN);
}
TEST_F(DndTestUser, 05_RestartDnode) {
// restart
stopServer(pServer);
pServer = NULL;

32
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -42,13 +42,6 @@ extern int32_t mDebugFlag;
#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }}
typedef struct SClusterObj SClusterObj;
typedef struct SMnodeObj SMnodeObj;
typedef struct SAcctObj SAcctObj;
typedef struct SVgObj SVgObj;
typedef struct SFuncObj SFuncObj;
typedef struct SOperObj SOperObj;
typedef enum {
MND_AUTH_ACCT_START = 0,
MND_AUTH_ACCT_USER,
@ -99,7 +92,7 @@ typedef enum {
DND_REASON_OTHERS
} EDndReason;
typedef struct STrans {
typedef struct {
int32_t id;
ETrnStage stage;
ETrnPolicy policy;
@ -111,7 +104,7 @@ typedef struct STrans {
SArray *undoActions;
} STrans;
typedef struct SClusterObj {
typedef struct {
int32_t id;
char name[TSDB_CLUSTER_ID_LEN];
int64_t createdTime;
@ -139,7 +132,7 @@ typedef struct {
char ep[TSDB_EP_LEN];
} SDnodeObj;
typedef struct SMnodeObj {
typedef struct {
int32_t id;
int64_t createdTime;
int64_t updateTime;
@ -167,7 +160,7 @@ typedef struct {
int64_t compStorage; // Compressed storage on disk
} SAcctInfo;
typedef struct SAcctObj {
typedef struct {
char acct[TSDB_USER_LEN];
int64_t createdTime;
int64_t updateTime;
@ -195,8 +188,8 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t minRows;
int32_t maxRows;
int32_t commitTime;
int32_t fsyncPeriod;
int8_t walLevel;
@ -214,7 +207,10 @@ typedef struct {
int64_t createdTime;
int64_t updateTime;
int64_t uid;
int32_t version;
int32_t cfgVersion;
int32_t vgVersion;
int32_t numOfVgroups;
int8_t hashMethod; // default is 1
SDbCfg cfg;
} SDbObj;
@ -223,12 +219,15 @@ typedef struct {
ESyncState role;
} SVnodeGid;
typedef struct SVgObj {
typedef struct {
int32_t vgId;
int64_t createdTime;
int64_t updateTime;
int32_t version;
uint32_t hashBegin;
uint32_t hashEnd;
char dbName[TSDB_FULL_DB_NAME_LEN];
int64_t dbUid;
int32_t numOfTables;
int32_t numOfTimeSeries;
int64_t totalStorage;
@ -245,6 +244,7 @@ typedef struct {
int64_t createdTime;
int64_t updateTime;
uint64_t uid;
uint64_t dbUid;
int32_t version;
int32_t numOfColumns;
int32_t numOfTags;
@ -252,7 +252,7 @@ typedef struct {
SSchema *pSchema;
} SStbObj;
typedef struct SFuncObj {
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
int64_t createdTime;
int8_t funcType;

1
source/dnode/mnode/impl/inc/mndDnode.h
View File

@ -28,6 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId);
void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode);
SEpSet mndGetDnodeEpset(SDnodeObj *pDnode);
int32_t mndGetDnodeSize(SMnode *pMnode);
bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode);
#ifdef __cplusplus
}

11
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -22,10 +22,13 @@
extern "C" {
#endif
int32_t mndInitVgroup(SMnode *pMnode);
void mndCleanupVgroup(SMnode *pMnode);
SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId);
void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
int32_t mndInitVgroup(SMnode *pMnode);
void mndCleanupVgroup(SMnode *pMnode);
SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId);
void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
#ifdef __cplusplus
}

8
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -143,22 +143,22 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "id");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
strcpy(pMeta->tbFname, mndShowStr(pShow->type));
pShow->numOfColumns = cols;

414
source/dnode/mnode/impl/src/mndDb.c
View File

@ -20,6 +20,7 @@
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#define TSDB_DB_VER_NUM 1
#define TSDB_DB_RESERVE_SIZE 64
@ -74,15 +75,18 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT64(pRaw, dataPos, pDb->createdTime)
SDB_SET_INT64(pRaw, dataPos, pDb->updateTime)
SDB_SET_INT64(pRaw, dataPos, pDb->uid)
SDB_SET_INT32(pRaw, dataPos, pDb->version)
SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion)
SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion)
SDB_SET_INT32(pRaw, dataPos, pDb->numOfVgroups)
SDB_SET_INT8(pRaw, dataPos, pDb->hashMethod)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRowsPerFileBlock)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRowsPerFileBlock)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel)
@ -118,15 +122,18 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->version)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfgVersion)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->vgVersion)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->numOfVgroups)
SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->hashMethod)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep0)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep1)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep2)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRowsPerFileBlock)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRowsPerFileBlock)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRows)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRows)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.commitTime)
SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.fsyncPeriod)
SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.walLevel)
@ -154,6 +161,9 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) {
static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
mTrace("db:%s, perform update action", pOldDb->name);
pOldDb->updateTime = pNewDb->createdTime;
pOldDb->cfgVersion = pNewDb->cfgVersion;
pOldDb->vgVersion = pNewDb->vgVersion;
pOldDb->numOfVgroups = pNewDb->numOfVgroups;
memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg));
return 0;
}
@ -184,133 +194,30 @@ static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) {
return 0;
}
static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg, char *errMsg, int32_t len) {
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database cache block size option", len);
return -1;
}
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database total blocks option", len);
return -1;
}
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database days option", len);
return -1;
}
if (pCfg->daysToKeep0 < pCfg->daysPerFile) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database days option", len);
return -1;
}
if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database keep0 option", len);
return -1;
}
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database keep1 option", len);
return -1;
}
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database keep2 option", len);
return -1;
}
if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database minrows option", len);
return -1;
}
if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database maxrows option", len);
return -1;
}
if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database minrows option", len);
return -1;
}
if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database commit option", len);
return -1;
}
if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database fsync option", len);
return -1;
}
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database wal level option", len);
return -1;
}
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid precision option", len);
return -1;
}
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database compression option", len);
return -1;
}
if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database replication option", len);
return -1;
}
if (pCfg->replications > mndGetDnodeSize(pMnode)) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database replication option", len);
return -1;
}
if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database quorum option", len);
return -1;
}
if (pCfg->quorum > pCfg->replications) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database quorum option", len);
return -1;
}
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database update option", len);
return -1;
}
if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
tstrncpy(errMsg, "Invalid database cachelast option", len);
return -1;
}
static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1;
if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1;
if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1;
if (pCfg->minRows < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRows > TSDB_MAX_MIN_ROW_FBLOCK) return -1;
if (pCfg->maxRows < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRows > TSDB_MAX_MAX_ROW_FBLOCK) return -1;
if (pCfg->minRows > pCfg->maxRows) return -1;
if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1;
if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) return -1;
if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1;
if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) return -1;
if (pCfg->quorum > pCfg->replications) return -1;
if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1;
if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1;
return TSDB_CODE_SUCCESS;
}
@ -321,8 +228,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP;
if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = TSDB_DEFAULT_KEEP;
if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = TSDB_DEFAULT_KEEP;
if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MIN_ROW_FBLOCK;
if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK;
if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME;
if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL;
@ -334,6 +241,56 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
}
static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING);
for (int v = 0; v < pDb->numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING);
}
return 0;
}
static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED);
for (int v = 0; v < pDb->numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED);
}
return 0;
}
static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_READY);
for (int v = 0; v < pDb->numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
}
return 0;
}
static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
return 0;
}
static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
return 0;
}
static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) {
SDbObj dbObj = {0};
tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN);
@ -341,14 +298,18 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
dbObj.createdTime = taosGetTimestampMs();
dbObj.updateTime = dbObj.createdTime;
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN);
dbObj.numOfVgroups = pCreate->numOfVgroups;
dbObj.hashMethod = 1;
dbObj.cfgVersion = 1;
dbObj.vgVersion = 1;
dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize,
.totalBlocks = pCreate->totalBlocks,
.daysPerFile = pCreate->daysPerFile,
.daysToKeep0 = pCreate->daysToKeep0,
.daysToKeep1 = pCreate->daysToKeep1,
.daysToKeep2 = pCreate->daysToKeep2,
.minRowsPerFileBlock = pCreate->minRowsPerFileBlock,
.maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock,
.minRows = pCreate->minRowsPerFileBlock,
.maxRows = pCreate->maxRowsPerFileBlock,
.fsyncPeriod = pCreate->fsyncPeriod,
.commitTime = pCreate->commitTime,
.precision = pCreate->precision,
@ -366,57 +327,69 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
return -1;
}
char errMsg[TSDB_ERROR_MSG_LEN] = {0};
if (mndCheckDbCfg(pMnode, &dbObj.cfg, errMsg, TSDB_ERROR_MSG_LEN) != 0) {
if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
mError("db:%s, failed to create since %s", pCreate->db, terrstr());
return -1;
}
SVgObj *pVgroups = NULL;
if (mndAllocVgroup(pMnode, &dbObj, &pVgroups) != 0) {
mError("db:%s, failed to create since %s", pCreate->db, terrstr());
return -1;
}
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("db:%s, failed to create since %s", pCreate->db, terrstr());
return -1;
goto CREATE_DB_OVER;
}
mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db);
SSdbRaw *pRedoRaw = mndDbActionEncode(&dbObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING);
SSdbRaw *pUndoRaw = mndDbActionEncode(&dbObj);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
goto CREATE_DB_OVER;
}
code = 0;
CREATE_DB_OVER:
free(pVgroups);
mndTransDrop(pTrans);
return 0;
return code;
}
static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont;
pCreate->numOfVgroups = htonl(pCreate->numOfVgroups);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->totalBlocks = htonl(pCreate->totalBlocks);
pCreate->daysPerFile = htonl(pCreate->daysPerFile);
@ -573,7 +546,8 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
return code;
}
dbObj.version++;
dbObj.cfgVersion++;
dbObj.updateTime = taosGetTimestampMs();
code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj);
mndReleaseDb(pMnode, pDb);
@ -658,17 +632,69 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) {
static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SSdb *pSdb = pMnode->pSdb;
SUseDbMsg *pUse = pMsg->rpcMsg.pCont;
pUse->vgVersion = htonl(pUse->vgVersion);
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
if (pDb != NULL) {
strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN);
mndReleaseDb(pMnode, pDb);
return 0;
} else {
mError("db:%s, failed to process use db msg since %s", pMsg->db, terrstr());
SDbObj *pDb = mndAcquireDb(pMnode, pUse->db);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
mError("db:%s, failed to process use db msg since %s", pUse->db, terrstr());
return -1;
}
int32_t contLen = sizeof(SUseDbRsp) + pDb->numOfVgroups * sizeof(SVgroupInfo);
SUseDbRsp *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
int32_t vindex = 0;
if (pUse->vgVersion < pDb->vgVersion) {
void *pIter = NULL;
while (vindex < pDb->numOfVgroups) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid == pDb->uid) {
SVgroupInfo *pInfo = &pRsp->vgroupInfo[vindex];
pInfo->vgId = htonl(pVgroup->vgId);
pInfo->hashBegin = htonl(pVgroup->hashBegin);
pInfo->hashEnd = htonl(pVgroup->hashEnd);
pInfo->numOfEps = pVgroup->replica;
for (int32_t gid = 0; gid < pVgroup->replica; ++gid) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[gid];
SEpAddrMsg *pEpArrr = &pInfo->epAddr[gid];
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode != NULL) {
memcpy(pEpArrr->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
pEpArrr->port = htons(pDnode->port);
}
mndReleaseDnode(pMnode, pDnode);
if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
pInfo->inUse = gid;
}
}
vindex++;
}
sdbRelease(pSdb, pVgroup);
}
}
memcpy(pRsp->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
pRsp->vgVersion = htonl(pDb->vgVersion);
pRsp->vgNum = htonl(vindex);
pRsp->hashMethod = pDb->hashMethod;
pMsg->pCont = pRsp;
pMsg->contLen = contLen;
mndReleaseDb(pMnode, pDb);
return 0;
}
static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) {
@ -709,100 +735,106 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe
pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create time");
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "vgroups");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "replica");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "quorum");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "days");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "keep0,keep1,keep2");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "cache(MB)");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "cache");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "blocks");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "minrows");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "maxrows");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "wallevel");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "fsync");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "comp");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "cachelast");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "precision");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema[cols].name, "update");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
@ -855,6 +887,10 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3
*(int64_t *)pWrite = pDb->createdTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->numOfVgroups;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pDb->cfg.replications;
cols++;
@ -886,11 +922,11 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock;
*(int32_t *)pWrite = pDb->cfg.minRows;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock;
*(int32_t *)pWrite = pDb->cfg.maxRows;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

39
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -218,6 +218,15 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
return sdbGetSize(pSdb, SDB_DNODE);
}
bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) {
int64_t ms = taosGetTimestampMs();
int64_t interval = ABS(pDnode->lastAccessTime - ms);
if (interval > 3000 * pMnode->cfg.statusInterval) {
return false;
}
return true;
}
static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) {
SSdb *pSdb = pMnode->pSdb;
@ -552,16 +561,16 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name));
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name));
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
@ -629,46 +638,46 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "id");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "endpoint");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "vnodes");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "max vnodes");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "max_vnodes");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "offline reason");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "offline_reason");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;

16
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -380,46 +380,46 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p
pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "comment");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "aggregate");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "outputtype");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "code_len");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "bufsize");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;

18
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -368,34 +368,34 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "id");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "end point");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "endpoint");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "role");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "role time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "role_time");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;

74
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -474,48 +474,48 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "connId");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "user");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
// app name
pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "program");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
// app pid
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "pid");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ip:port");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "login_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "last_access");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
@ -602,88 +602,88 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "queryId");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "connId");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "user");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ip:port");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 24;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "qid");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "created_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_BIGINT;
strcpy(pSchema[cols].name, "time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sql_obj_id");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "pid");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ep");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 1;
pSchema[cols].type = TSDB_DATA_TYPE_BOOL;
strcpy(pSchema[cols].name, "stable_query");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "sub_queries");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sub_query_info");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sql");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
@ -818,64 +818,64 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "streamId");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "connId");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "user");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "dest table");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "destination");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "ip:port");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "created time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "exec time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "exec");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_BIGINT;
strcpy(pSchema[cols].name, "time(us)");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "sql");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "cycles");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;

20
source/dnode/mnode/impl/src/mndStb.c
View File

@ -74,10 +74,12 @@ static SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN)
SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN)
SDB_SET_BINARY(pRaw, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN)
SDB_SET_INT64(pRaw, dataPos, pStb->createdTime)
SDB_SET_INT64(pRaw, dataPos, pStb->updateTime)
SDB_SET_INT64(pRaw, dataPos, pStb->uid)
SDB_SET_INT64(pRaw, dataPos, pStb->dbUid)
SDB_SET_INT64(pRaw, dataPos, pStb->version)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags)
@ -113,10 +115,12 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
if (pStb == NULL) return NULL;
int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN)
SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN)
SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN)
SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid)
SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->dbUid)
SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version)
SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns)
SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags)
@ -570,28 +574,28 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM
pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "columns");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
strcpy(pSchema[cols].name, "tags");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;

36
source/dnode/mnode/impl/src/mndSync.c
View File

@ -1,36 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "mndInt.h"
#include "mndTrans.h"
int32_t mndInitSync(SMnode *pMnode) { return 0; }
void mndCleanupSync(SMnode *pMnode) {}
int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) {
int32_t code = 0;
// int32_t len = sdbGetRawTotalSize(pRaw);
// SSdbRaw *pReceived = calloc(1, len);
// memcpy(pReceived, pRaw, len);
// mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg);
// mndTransApply(pMnode, pReceived, code);
return code;
}
bool mndIsMaster(SMnode *pMnode) { return true; }

10
source/dnode/mnode/impl/src/mndUser.c
View File

@ -410,28 +410,28 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p
pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "privilege");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "account");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;

149
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -24,11 +24,9 @@
#define TSDB_VGROUP_VER_NUM 1
#define TSDB_VGROUP_RESERVE_SIZE 64
static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup);
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup);
static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg);
static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg);
@ -70,8 +68,8 @@ int32_t mndInitVgroup(SMnode *pMnode) {
void mndCleanupVgroup(SMnode *pMnode) {}
static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE);
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -79,7 +77,10 @@ static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime)
SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime)
SDB_SET_INT32(pRaw, dataPos, pVgroup->version)
SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin)
SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd)
SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN)
SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid)
SDB_SET_INT8(pRaw, dataPos, pVgroup->replica)
for (int8_t i = 0; i < pVgroup->replica; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
@ -92,7 +93,7 @@ static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
return pRaw;
}
static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
@ -102,7 +103,7 @@ static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
return NULL;
}
SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj));
SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj));
SVgObj *pVgroup = sdbGetRowObj(pRow);
if (pVgroup == NULL) return NULL;
@ -111,7 +112,10 @@ static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->updateTime)
SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->version)
SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashBegin)
SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashEnd)
SDB_GET_BINARY(pRaw, pRow, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN)
SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->dbUid)
SDB_GET_INT8(pRaw, pRow, dataPos, &pVgroup->replica)
for (int8_t i = 0; i < pVgroup->replica; ++i) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[i];
@ -133,12 +137,6 @@ static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) {
return 0;
}
static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) {
mTrace("vgId:%d, perform update action", pOldVgroup->vgId);
pOldVgroup->updateTime = pNewVgroup->updateTime;
@ -158,6 +156,105 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
sdbRelease(pSdb, pVgroup);
}
static int32_t mndGetDefaultVgroupSize(SMnode *pMnode) {
// todo
return 2;
}
static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) {
SSdb *pSdb = pMnode->pSdb;
int32_t allocedVnodes = 0;
void *pIter = NULL;
while (allocedVnodes < pVgroup->replica) {
SDnodeObj *pDnode = NULL;
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
if (pIter == NULL) break;
// todo
if (mndIsDnodeInReadyStatus(pMnode, pDnode)) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[allocedVnodes];
pVgid->dnodeId = pDnode->id;
if (pVgroup->replica == 1) {
pVgid->role = TAOS_SYNC_STATE_LEADER;
} else {
pVgid->role = TAOS_SYNC_STATE_FOLLOWER;
}
allocedVnodes++;
}
sdbRelease(pSdb, pDnode);
}
if (allocedVnodes != pVgroup->replica) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
return 0;
}
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
if (pDb->numOfVgroups != -1 &&
(pDb->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pDb->numOfVgroups > TSDB_MAX_VNODES_PER_DB)) {
terrno = TSDB_CODE_MND_INVALID_DB_OPTION;
return -1;
}
if (pDb->numOfVgroups == -1) {
pDb->numOfVgroups = mndGetDefaultVgroupSize(pMnode);
if (pDb->numOfVgroups < 0) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1;
}
}
SVgObj *pVgroups = calloc(pDb->numOfVgroups, sizeof(SVgObj));
if (pVgroups == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
int32_t alloceVgroups = 0;
int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
uint32_t hashMin = 0;
uint32_t hashMax = UINT32_MAX;
uint32_t hashInterval = (hashMax - hashMin) / pDb->numOfVgroups;
for (uint32_t v = 0; v < pDb->numOfVgroups; v++) {
SVgObj *pVgroup = &pVgroups[v];
pVgroup->vgId = maxVgId++;
pVgroup->createdTime = taosGetTimestampMs();
pVgroup->updateTime = pVgroups->createdTime;
pVgroup->version = 1;
pVgroup->dbUid = pDb->uid;
pVgroup->hashBegin = hashMin + hashInterval * v;
if (v == pDb->numOfVgroups - 1) {
pVgroup->hashEnd = hashMax;
} else {
pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1;
}
memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN);
pVgroup->replica = pDb->cfg.replications;
if (mndGetAvailableDnode(pMnode, pVgroup) != 0) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
free(pVgroups);
return -1;
}
alloceVgroups++;
}
*ppVgroups = pVgroups;
return 0;
}
static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) {
SSdb *pSdb = pMnode->pSdb;
@ -203,30 +300,30 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "vgId");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "tables");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
for (int32_t i = 0; i < pShow->replica; ++i) {
pShow->bytes[cols] = 4;
pShow->bytes[cols] = 2;
pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1);
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
}
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
@ -273,10 +370,6 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data,
cols++;
}
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int8_t *)pWrite = pVgroup->compact;
cols++;
sdbRelease(pSdb, pVgroup);
numOfRows++;
}
@ -309,16 +402,16 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "vgId");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "status");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
pSchema[cols].bytes = htonl(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
pMeta->numOfColumns = htonl(cols);
pShow->numOfColumns = cols;
pShow->offset[0] = 0;

8
source/dnode/vnode/impl/inc/vnodeStateMgr.h
View File

@ -21,13 +21,13 @@ extern "C" {
#endif
typedef struct {
uint64_t processed;
uint64_t committed;
uint64_t applied;
int64_t processed;
int64_t committed;
int64_t applied;
} SVState;
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_STATE_MGR_H_*/
#endif /*_TD_VNODE_STATE_MGR_H_*/

23
source/dnode/vnode/impl/src/vnodeWrite.c
View File

@ -15,16 +15,31 @@
#include "vnodeDef.h"
int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) {
SVnodeReq *pVnodeReq;
switch (pMsg->msgType) {
case TSDB_MSG_TYPE_MQ_SET:
if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) {
// TODO: handle error
}
break;
}
void *pBuf = pMsg->pCont;
return 0;
}
int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
SRpcMsg * pMsg;
SRpcMsg *pMsg;
SVnodeReq *pVnodeReq;
for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
// ser request version
void * pBuf = pMsg->pCont;
uint64_t ver = pVnode->state.processed++;
void *pBuf = pMsg->pCont;
int64_t ver = pVnode->state.processed++;
taosEncodeFixedU64(&pBuf, ver);
if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) {
@ -99,4 +114,4 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
return 0;
}
/* ------------------------ STATIC METHODS ------------------------ */
/* ------------------------ STATIC METHODS ------------------------ */

20
source/dnode/vnode/tq/inc/tqInt.h
View File

@ -22,16 +22,16 @@
extern "C" {
#endif
//create persistent storage for meta info such as consuming offset
//return value > 0: cgId
//return value <= 0: error code
//int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle);
//create ring buffer in memory and load consuming offset
//int tqOpenTCGroup(STQ*, const char* topic, int cgId);
//destroy ring buffer and persist consuming offset
//int tqCloseTCGroup(STQ*, const char* topic, int cgId);
//delete persistent storage for meta info
//int tqDropTCGroup(STQ*, const char* topic, int cgId);
// create persistent storage for meta info such as consuming offset
// return value > 0: cgId
// return value <= 0: error code
// int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle);
// create ring buffer in memory and load consuming offset
// int tqOpenTCGroup(STQ*, const char* topic, int cgId);
// destroy ring buffer and persist consuming offset
// int tqCloseTCGroup(STQ*, const char* topic, int cgId);
// delete persistent storage for meta info
// int tqDropTCGroup(STQ*, const char* topic, int cgId);
#ifdef __cplusplus
}

27
source/dnode/vnode/tq/inc/tqMetaStore.h
View File

@ -23,27 +23,22 @@
extern "C" {
#endif
STqMetaStore* tqStoreOpen(const char* path,
TqSerializeFun pSerializer,
TqDeserializeFun pDeserializer,
TqDeleteFun pDeleter,
int32_t tqConfigFlag
);
STqMetaStore* tqStoreOpen(const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, FTqDelete pDeleter,
int32_t tqConfigFlag);
int32_t tqStoreClose(STqMetaStore*);
//int32_t tqStoreDelete(TqMetaStore*);
//int32_t tqStoreCommitAll(TqMetaStore*);
int32_t tqStorePersist(STqMetaStore*);
//clean deleted idx and data from persistent file
int32_t tqStoreCompact(STqMetaStore*);
// int32_t tqStoreDelete(TqMetaStore*);
// int32_t tqStoreCommitAll(TqMetaStore*);
int32_t tqStorePersist(STqMetaStore*);
// clean deleted idx and data from persistent file
int32_t tqStoreCompact(STqMetaStore*);
void* tqHandleGet(STqMetaStore*, int64_t key);
//make it unpersist
void* tqHandleGet(STqMetaStore*, int64_t key);
// make it unpersist
void* tqHandleTouchGet(STqMetaStore*, int64_t key);
int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value);
int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize);
//delete committed kv pair
//notice that a delete action still needs to be committed
// delete committed kv pair
// notice that a delete action still needs to be committed
int32_t tqHandleDel(STqMetaStore*, int64_t key);
int32_t tqHandleCommit(STqMetaStore*, int64_t key);
int32_t tqHandleAbort(STqMetaStore*, int64_t key);

319
source/dnode/vnode/tq/src/tq.c
View File

@ -16,51 +16,48 @@
#include "tqInt.h"
#include "tqMetaStore.h"
//static
//read next version data
// static
// read next version data
//
//send to fetch queue
// send to fetch queue
//
//handle management message
// handle management message
//
int tqGetgHandleSSize(const TqGroupHandle *gHandle);
int tqGetgHandleSSize(const STqGroupHandle* gHandle);
int tqBufHandleSSize();
int tqBufItemSSize();
TqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
TqGroupHandle* gHandle;
STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
STqGroupHandle* gHandle;
return NULL;
}
void* tqSerializeListHandle(TqListHandle* listHandle, void* ptr);
void* tqSerializeBufHandle(TqBufferHandle* bufHandle, void* ptr);
void* tqSerializeBufItem(TqBufferItem* bufItem, void* ptr);
void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr);
void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr);
void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr);
const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle* bufHandle);
const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem* bufItem);
const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle);
const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem);
STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac) {
STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) {
STQ* pTq = malloc(sizeof(STQ));
if(pTq == NULL) {
//TODO: memory error
if (pTq == NULL) {
// TODO: memory error
return NULL;
}
pTq->path = strdup(path);
pTq->tqConfig = tqConfig;
pTq->tqLogReader = tqLogReader;
pTq->tqMemRef.pAlloctorFactory = allocFac;
// pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
if(pTq->tqMemRef.pAllocator == NULL) {
//TODO
pTq->tqMemRef.pAlloctorFactory = allocFac;
pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
if (pTq->tqMemRef.pAllocator == NULL) {
// TODO
}
pTq->tqMeta = tqStoreOpen(path,
(TqSerializeFun)tqSerializeGroupHandle,
(TqDeserializeFun)tqDeserializeGroupHandle,
free,
0);
if(pTq->tqMeta == NULL) {
//TODO: free STQ
pTq->tqMeta =
tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0);
if (pTq->tqMeta == NULL) {
// TODO: free STQ
return NULL;
}
return pTq;
@ -74,21 +71,21 @@ static int tqProtoCheck(TmqMsgHead *pMsg) {
return pMsg->protoVer == 0;
}
static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** ppQuery) {
//clean old item and move forward
static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) {
// clean old item and move forward
int32_t consumeOffset = pAck->consumeOffset;
int idx = consumeOffset % TQ_BUFFER_SIZE;
int idx = consumeOffset % TQ_BUFFER_SIZE;
ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor);
tfree(bHandle->buffer[idx].content);
if( 1 /* TODO: need to launch new query */) {
TqQueryMsg* pNewQuery = malloc(sizeof(TqQueryMsg));
if(pNewQuery == NULL) {
//TODO: memory insufficient
if (1 /* TODO: need to launch new query */) {
STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg));
if (pNewQuery == NULL) {
// TODO: memory insufficient
return -1;
}
//TODO: lock executor
// TODO: lock executor
pNewQuery->exec->executor = bHandle->buffer[idx].executor;
//TODO: read from wal and assign to src
// TODO: read from wal and assign to src
pNewQuery->exec->src = 0;
pNewQuery->exec->dest = &bHandle->buffer[idx];
pNewQuery->next = *ppQuery;
@ -97,98 +94,94 @@ static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg**
return 0;
}
static int tqAck(TqGroupHandle* gHandle, TmqAcks* pAcks) {
int32_t ackNum = pAcks->ackNum;
TmqOneAck *acks = pAcks->acks;
//double ptr for acks and list
int i = 0;
TqListHandle* node = gHandle->head;
int ackCnt = 0;
TqQueryMsg *pQuery = NULL;
while(i < ackNum && node->next) {
if(acks[i].topicId == node->next->bufHandle.topicId) {
static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) {
int32_t ackNum = pAcks->ackNum;
TmqOneAck* acks = pAcks->acks;
// double ptr for acks and list
int i = 0;
STqListHandle* node = gHandle->head;
int ackCnt = 0;
STqQueryMsg* pQuery = NULL;
while (i < ackNum && node->next) {
if (acks[i].topicId == node->next->bufHandle.topicId) {
ackCnt++;
tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery);
} else if(acks[i].topicId < node->next->bufHandle.topicId) {
} else if (acks[i].topicId < node->next->bufHandle.topicId) {
i++;
} else {
node = node->next;
}
}
if(pQuery) {
//post message
if (pQuery) {
// post message
}
return ackCnt;
}
static int tqCommitTCGroup(TqGroupHandle* handle) {
//persist modification into disk
static int tqCommitTCGroup(STqGroupHandle* handle) {
// persist modification into disk
return 0;
}
int tqCreateTCGroup(STQ *pTq, int64_t topicId, int64_t cgId, int64_t cId, TqGroupHandle** handle) {
//create in disk
TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle));
if(gHandle == NULL) {
//TODO
int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) {
// create in disk
STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle));
if (gHandle == NULL) {
// TODO
return -1;
}
memset(gHandle, 0, sizeof(TqGroupHandle));
memset(gHandle, 0, sizeof(STqGroupHandle));
return 0;
}
TqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
TqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId);
if(gHandle == NULL) {
STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId);
if (gHandle == NULL) {
int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle);
if(code != 0) {
//TODO
if (code != 0) {
// TODO
return NULL;
}
}
//create
//open
// create
// open
return gHandle;
}
int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
return 0;
}
int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; }
int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
//delete from disk
// delete from disk
return 0;
}
static int tqFetch(TqGroupHandle* gHandle, void** msg) {
TqListHandle* head = gHandle->head;
TqListHandle* node = head;
int totSize = 0;
//TODO: make it a macro
int sizeLimit = 4 * 1024;
static int tqFetch(STqGroupHandle* gHandle, void** msg) {
STqListHandle* head = gHandle->head;
STqListHandle* node = head;
int totSize = 0;
// TODO: make it a macro
int sizeLimit = 4 * 1024;
TmqMsgContent* buffer = malloc(sizeLimit);
if(buffer == NULL) {
//TODO:memory insufficient
if (buffer == NULL) {
// TODO:memory insufficient
return -1;
}
//iterate the list to get msgs of all topics
//until all topic iterated or msgs over sizeLimit
while(node->next) {
// iterate the list to get msgs of all topics
// until all topic iterated or msgs over sizeLimit
while (node->next) {
node = node->next;
TqBufferHandle* bufHandle = &node->bufHandle;
int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
if(bufHandle->buffer[idx].content != NULL &&
bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset
) {
STqBufferHandle* bufHandle = &node->bufHandle;
int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) {
totSize += bufHandle->buffer[idx].size;
if(totSize > sizeLimit) {
void *ptr = realloc(buffer, totSize);
if(ptr == NULL) {
if (totSize > sizeLimit) {
void* ptr = realloc(buffer, totSize);
if (ptr == NULL) {
totSize -= bufHandle->buffer[idx].size;
//TODO:memory insufficient
//return msgs already copied
// TODO:memory insufficient
// return msgs already copied
break;
}
}
@ -198,7 +191,7 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size);
buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size);
if(totSize > sizeLimit) {
if (totSize > sizeLimit) {
break;
}
}
@ -206,104 +199,102 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) {
return totSize;
}
TqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) {
return NULL;
}
STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; }
int tqLaunchQuery(TqGroupHandle* gHandle) {
return 0;
}
int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; }
int tqSendLaunchQuery(TqGroupHandle* gHandle) {
return 0;
}
int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; }
/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/
/*return 0;*/
/*return 0;*/
/*}*/
int tqPushMsg(STQ* pTq , void* p, int64_t version) {
//add reference
//judge and launch new query
int tqPushMsg(STQ* pTq, void* p, int64_t version) {
// add reference
// judge and launch new query
return 0;
}
int tqCommit(STQ* pTq) {
//do nothing
// do nothing
return 0;
}
int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) {
if(!tqProtoCheck((TmqMsgHead *)pMsg)) {
//proto version invalid
int tqSetCursor(STQ* pTq, void* msg) {
return 0;
}
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
if (!tqProtoCheck((TmqMsgHead*)pMsg)) {
// proto version invalid
return -1;
}
int64_t clientId = pMsg->head.clientId;
TqGroupHandle *gHandle = tqGetGroupHandle(pTq, clientId);
if(gHandle == NULL) {
//client not connect
int64_t clientId = pMsg->head.clientId;
STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId);
if (gHandle == NULL) {
// client not connect
return -1;
}
if(pMsg->acks.ackNum != 0) {
if(tqAck(gHandle, &pMsg->acks) != 0) {
//ack not success
if (pMsg->acks.ackNum != 0) {
if (tqAck(gHandle, &pMsg->acks) != 0) {
// ack not success
return -1;
}
}
TmqConsumeRsp *pRsp = (TmqConsumeRsp*) pMsg;
STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg;
if(tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
//fetch error
if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
// fetch error
return -1;
}
//judge and launch new query
if(tqLaunchQuery(gHandle)) {
//launch query error
// judge and launch new query
if (tqLaunchQuery(gHandle)) {
// launch query error
return -1;
}
return 0;
}
int tqSerializeGroupHandle(const TqGroupHandle *gHandle, TqSerializedHead** ppHead) {
//calculate size
int sz = tqGetgHandleSSize(gHandle) + sizeof(TqSerializedHead);
if(sz > (*ppHead)->ssize) {
int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) {
// calculate size
int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead);
if (sz > (*ppHead)->ssize) {
void* tmpPtr = realloc(*ppHead, sz);
if(tmpPtr == NULL) {
if (tmpPtr == NULL) {
free(*ppHead);
//TODO: memory err
// TODO: memory err
return -1;
}
*ppHead = tmpPtr;
(*ppHead)->ssize = sz;
}
void* ptr = (*ppHead)->content;
//do serialization
// do serialization
*(int64_t*)ptr = gHandle->cId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = gHandle->cgId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int32_t*)ptr = gHandle->topicNum;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
if(gHandle->topicNum > 0) {
if (gHandle->topicNum > 0) {
tqSerializeListHandle(gHandle->head, ptr);
}
return 0;
}
void* tqSerializeListHandle(TqListHandle *listHandle, void* ptr) {
TqListHandle *node = listHandle;
void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) {
STqListHandle* node = listHandle;
ASSERT(node != NULL);
while(node) {
while (node) {
ptr = tqSerializeBufHandle(&node->bufHandle, ptr);
node = node->next;
}
return ptr;
}
void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) {
void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) {
*(int64_t*)ptr = bufHandle->nextConsumeOffset;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = bufHandle->topicId;
@ -312,21 +303,21 @@ void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) {
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
*(int32_t*)ptr = bufHandle->tail;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for(int i = 0; i < TQ_BUFFER_SIZE; i++) {
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr);
}
return ptr;
}
void* tqSerializeBufItem(TqBufferItem *bufItem, void* ptr) {
//TODO: do we need serialize this?
//mainly for executor
void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) {
// TODO: do we need serialize this?
// mainly for executor
return ptr;
}
const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle **ppGHandle) {
TqGroupHandle *gHandle = *ppGHandle;
const void* ptr = pHead->content;
const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) {
STqGroupHandle* gHandle = *ppGHandle;
const void* ptr = pHead->content;
gHandle->cId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
gHandle->cgId = *(int64_t*)ptr;
@ -335,20 +326,20 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
gHandle->topicNum = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
gHandle->head = NULL;
TqListHandle *node = gHandle->head;
for(int i = 0; i < gHandle->topicNum; i++) {
if(gHandle->head == NULL) {
if((node = malloc(sizeof(TqListHandle))) == NULL) {
//TODO: error
STqListHandle* node = gHandle->head;
for (int i = 0; i < gHandle->topicNum; i++) {
if (gHandle->head == NULL) {
if ((node = malloc(sizeof(STqListHandle))) == NULL) {
// TODO: error
return NULL;
}
node->next= NULL;
ptr = tqDeserializeBufHandle(ptr, &node->bufHandle);
node->next = NULL;
ptr = tqDeserializeBufHandle(ptr, &node->bufHandle);
gHandle->head = node;
} else {
node->next = malloc(sizeof(TqListHandle));
if(node->next == NULL) {
//TODO: error
node->next = malloc(sizeof(STqListHandle));
if (node->next == NULL) {
// TODO: error
return NULL;
}
node->next->next = NULL;
@ -359,7 +350,7 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl
return ptr;
}
const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle) {
const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) {
const void* ptr = pBytes;
bufHandle->nextConsumeOffset = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
@ -369,32 +360,30 @@ const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
bufHandle->tail = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for(int i = 0; i < TQ_BUFFER_SIZE; i++) {
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]);
}
return ptr;
}
const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem *bufItem) {
return pBytes;
const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; }
// TODO: make this a macro
int tqGetgHandleSSize(const STqGroupHandle* gHandle) {
return sizeof(int64_t) * 2 // cId + cgId
+ sizeof(int32_t) // topicNum
+ gHandle->topicNum * tqBufHandleSSize();
}
//TODO: make this a macro
int tqGetgHandleSSize(const TqGroupHandle *gHandle) {
return sizeof(int64_t) * 2 //cId + cgId
+ sizeof(int32_t) //topicNum
+ gHandle->topicNum * tqBufHandleSSize();
}
//TODO: make this a macro
// TODO: make this a macro
int tqBufHandleSSize() {
return sizeof(int64_t) * 2 // nextConsumeOffset + topicId
+ sizeof(int32_t) * 2 // head + tail
+ TQ_BUFFER_SIZE * tqBufItemSSize();
return sizeof(int64_t) * 2 // nextConsumeOffset + topicId
+ sizeof(int32_t) * 2 // head + tail
+ TQ_BUFFER_SIZE * tqBufItemSSize();
}
int tqBufItemSSize() {
//TODO: do this need serialization?
//mainly for executor
// TODO: do this need serialization?
// mainly for executor
return 0;
}

378
source/dnode/vnode/tq/src/tqMetaStore.c
View File

@ -13,20 +13,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tqMetaStore.h"
//TODO:replace by an abstract file layer
#include "osDir.h"
// TODO:replace by an abstract file layer
#include <fcntl.h>
#include <string.h>
#include <unistd.h>
#include "osDir.h"
#define TQ_META_NAME "tq.meta"
#define TQ_IDX_NAME "tq.idx"
#define TQ_IDX_NAME "tq.idx"
static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value);
static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key);
static inline void tqLinkUnpersist(STqMetaStore *pMeta, STqMetaList* pNode) {
if(pNode->unpersistNext == NULL) {
static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) {
if (pNode->unpersistNext == NULL) {
pNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pNode->unpersistPrev = pMeta->unpersistHead;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode;
@ -41,24 +41,24 @@ static inline int tqSeekLastPage(int fd) {
return lseek(fd, curPageOffset, SEEK_SET);
}
//TODO: the struct is tightly coupled with index entry
typedef struct TqIdxPageHead {
// TODO: the struct is tightly coupled with index entry
typedef struct STqIdxPageHead {
int16_t writeOffset;
int8_t unused[14];
} TqIdxPageHead;
} STqIdxPageHead;
typedef struct TqIdxPageBuf {
TqIdxPageHead head;
char buffer[TQ_IDX_PAGE_BODY_SIZE];
} TqIdxPageBuf;
typedef struct STqIdxPageBuf {
STqIdxPageHead head;
char buffer[TQ_IDX_PAGE_BODY_SIZE];
} STqIdxPageBuf;
static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) {
static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd);
int nBytes;
if((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
return -1;
}
if(nBytes == 0) {
if (nBytes == 0) {
memset(pBuf, 0, TQ_PAGE_SIZE);
pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
}
@ -67,28 +67,24 @@ static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) {
return lseek(fd, offset, SEEK_SET);
}
STqMetaStore* tqStoreOpen(const char* path,
TqSerializeFun serializer,
TqDeserializeFun deserializer,
TqDeleteFun deleter,
int32_t tqConfigFlag
) {
STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
if(pMeta == NULL) {
//close
STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter,
int32_t tqConfigFlag) {
STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
if (pMeta == NULL) {
// close
return NULL;
}
memset(pMeta, 0, sizeof(STqMetaStore));
//concat data file name and index file name
// concat data file name and index file name
size_t pathLen = strlen(path);
pMeta->dirPath = malloc(pathLen+1);
if(pMeta->dirPath != NULL) {
//TODO: memory insufficient
pMeta->dirPath = malloc(pathLen + 1);
if (pMeta->dirPath != NULL) {
// TODO: memory insufficient
}
strcpy(pMeta->dirPath, path);
char name[pathLen+10];
char name[pathLen + 10];
strcpy(name, path);
if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
@ -96,98 +92,96 @@ STqMetaStore* tqStoreOpen(const char* path,
}
strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if(idxFd < 0) {
if (idxFd < 0) {
ASSERT(false);
//close file
//free memory
// close file
// free memory
return NULL;
}
pMeta->idxFd = idxFd;
pMeta->unpersistHead = malloc(sizeof(STqMetaList));
if(pMeta->unpersistHead == NULL) {
if (pMeta->unpersistHead == NULL) {
ASSERT(false);
//close file
//free memory
// close file
// free memory
return NULL;
}
memset(pMeta->unpersistHead, 0, sizeof(STqMetaList));
pMeta->unpersistHead->unpersistNext
= pMeta->unpersistHead->unpersistPrev
= pMeta->unpersistHead;
pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
strcpy(name, path);
strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if(fileFd < 0){
if (fileFd < 0) {
ASSERT(false);
return NULL;
}
pMeta->fileFd = fileFd;
pMeta->pSerializer = serializer;
pMeta->pDeserializer = deserializer;
pMeta->pDeleter = deleter;
pMeta->tqConfigFlag = tqConfigFlag;
//read idx file and load into memory
TqIdxPageBuf idxBuf;
TqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
if(serializedObj == NULL) {
//TODO:memory insufficient
// read idx file and load into memory
STqIdxPageBuf idxBuf;
STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
if (serializedObj == NULL) {
// TODO:memory insufficient
}
int idxRead;
int allocated = TQ_PAGE_SIZE;
int idxRead;
int allocated = TQ_PAGE_SIZE;
bool readEnd = false;
while((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
if(idxRead == -1) {
//TODO: handle error
while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) {
// TODO: handle error
ASSERT(false);
}
ASSERT(idxBuf.head.writeOffset == idxRead);
//loop read every entry
for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
STqMetaList *pNode = malloc(sizeof(STqMetaList));
if(pNode == NULL) {
//TODO: free memory and return error
// loop read every entry
for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
STqMetaList* pNode = malloc(sizeof(STqMetaList));
if (pNode == NULL) {
// TODO: free memory and return error
}
memset(pNode, 0, sizeof(STqMetaList));
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
lseek(fileFd, pNode->handle.offset, SEEK_SET);
if(allocated < pNode->handle.serializedSize) {
void *ptr = realloc(serializedObj, pNode->handle.serializedSize);
if(ptr == NULL) {
//TODO: memory insufficient
if (allocated < pNode->handle.serializedSize) {
void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) {
// TODO: memory insufficient
}
serializedObj = ptr;
allocated = pNode->handle.serializedSize;
}
serializedObj->ssize = pNode->handle.serializedSize;
if(read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
//TODO: read error
if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) {
// TODO: read error
}
if(serializedObj->action == TQ_ACTION_INUSE) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) {
if (serializedObj->action == TQ_ACTION_INUSE) {
if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
} else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN;
}
} else if(serializedObj->action == TQ_ACTION_INTXN) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) {
} else if (serializedObj->action == TQ_ACTION_INTXN) {
if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn);
} else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
}
} else if(serializedObj->action == TQ_ACTION_INUSE_CONT) {
if(serializedObj->ssize != sizeof(TqSerializedHead)) {
} else if (serializedObj->action == TQ_ACTION_INUSE_CONT) {
if (serializedObj->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse);
} else {
pNode->handle.valueInUse = TQ_DELETE_TOKEN;
}
TqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
if(ptr->ssize != sizeof(TqSerializedHead)) {
STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize);
if (ptr->ssize != sizeof(STqSerializedHead)) {
pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn);
} else {
pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
@ -196,22 +190,21 @@ STqMetaStore* tqStoreOpen(const char* path,
ASSERT(0);
}
//put into list
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
// put into list
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
STqMetaList* pBucketNode = pMeta->bucket[bucketKey];
if(pBucketNode == NULL) {
if (pBucketNode == NULL) {
pMeta->bucket[bucketKey] = pNode;
} else if(pBucketNode->handle.key == pNode->handle.key) {
} else if (pBucketNode->handle.key == pNode->handle.key) {
pNode->next = pBucketNode->next;
pMeta->bucket[bucketKey] = pNode;
} else {
while(pBucketNode->next &&
pBucketNode->next->handle.key != pNode->handle.key) {
pBucketNode = pBucketNode->next;
while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) {
pBucketNode = pBucketNode->next;
}
if(pBucketNode->next) {
if (pBucketNode->next) {
ASSERT(pBucketNode->next->handle.key == pNode->handle.key);
STqMetaList *pNodeFound = pBucketNode->next;
STqMetaList* pNodeFound = pBucketNode->next;
pNode->next = pNodeFound->next;
pBucketNode->next = pNode;
pBucketNode = pNodeFound;
@ -221,13 +214,11 @@ STqMetaStore* tqStoreOpen(const char* path,
pBucketNode = NULL;
}
}
if(pBucketNode) {
if(pBucketNode->handle.valueInUse
&& pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
if (pBucketNode) {
if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInUse);
}
if(pBucketNode->handle.valueInTxn
&& pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pBucketNode->handle.valueInTxn);
}
free(pBucketNode);
@ -239,23 +230,21 @@ STqMetaStore* tqStoreOpen(const char* path,
}
int32_t tqStoreClose(STqMetaStore* pMeta) {
//commit data and idx
// commit data and idx
tqStorePersist(pMeta);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next==NULL);
ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL);
close(pMeta->fileFd);
close(pMeta->idxFd);
//free memory
for(int i = 0; i < TQ_BUCKET_SIZE; i++) {
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
while(pNode) {
while (pNode) {
ASSERT(pNode->unpersistNext == NULL);
ASSERT(pNode->unpersistPrev == NULL);
if(pNode->handle.valueInTxn
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
if(pNode->handle.valueInUse
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
STqMetaList* next = pNode->next;
@ -272,17 +261,15 @@ int32_t tqStoreClose(STqMetaStore* pMeta) {
int32_t tqStoreDelete(STqMetaStore* pMeta) {
close(pMeta->fileFd);
close(pMeta->idxFd);
//free memory
for(int i = 0; i < TQ_BUCKET_SIZE; i++) {
// free memory
for (int i = 0; i < TQ_BUCKET_SIZE; i++) {
STqMetaList* pNode = pMeta->bucket[i];
pMeta->bucket[i] = NULL;
while(pNode) {
if(pNode->handle.valueInTxn
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
while (pNode) {
if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
if(pNode->handle.valueInUse
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
STqMetaList* next = pNode->next;
@ -297,26 +284,26 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) {
return 0;
}
//TODO: wrap in tfile
// TODO: wrap in tfile
int32_t tqStorePersist(STqMetaStore* pMeta) {
TqIdxPageBuf idxBuf;
int64_t* bufPtr = (int64_t*)idxBuf.buffer;
STqMetaList *pHead = pMeta->unpersistHead;
STqMetaList *pNode = pHead->unpersistNext;
TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead));
if(pSHead == NULL) {
//TODO: memory error
STqIdxPageBuf idxBuf;
int64_t* bufPtr = (int64_t*)idxBuf.buffer;
STqMetaList* pHead = pMeta->unpersistHead;
STqMetaList* pNode = pHead->unpersistNext;
STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead));
if (pSHead == NULL) {
// TODO: memory error
return -1;
}
pSHead->ver = TQ_SVER;
pSHead->checksum = 0;
pSHead->ssize = sizeof(TqSerializedHead);
int allocatedSize = sizeof(TqSerializedHead);
pSHead->ssize = sizeof(STqSerializedHead);
int allocatedSize = sizeof(STqSerializedHead);
int offset = lseek(pMeta->fileFd, 0, SEEK_CUR);
tqReadLastPage(pMeta->idxFd, &idxBuf);
if(idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) {
lseek(pMeta->idxFd, 0, SEEK_END);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
@ -324,18 +311,18 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset);
}
while(pHead != pNode) {
while (pHead != pNode) {
int nBytes = 0;
if(pNode->handle.valueInUse) {
if(pNode->handle.valueInTxn) {
if (pNode->handle.valueInUse) {
if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INUSE_CONT;
} else {
pSHead->action = TQ_ACTION_INUSE;
}
if(pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(TqSerializedHead);
if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(STqSerializedHead);
} else {
pMeta->pSerializer(pNode->handle.valueInUse, &pSHead);
}
@ -343,10 +330,10 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
ASSERT(nBytes == pSHead->ssize);
}
if(pNode->handle.valueInTxn) {
if (pNode->handle.valueInTxn) {
pSHead->action = TQ_ACTION_INTXN;
if(pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(TqSerializedHead);
if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) {
pSHead->ssize = sizeof(STqSerializedHead);
} else {
pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead);
}
@ -357,42 +344,39 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
pNode->handle.offset = offset;
offset += nBytes;
//write idx file
//TODO: endian check and convert
// write idx file
// TODO: endian check and convert
*(bufPtr++) = pNode->handle.key;
*(bufPtr++) = pNode->handle.offset;
*(bufPtr++) = (int64_t)nBytes;
idxBuf.head.writeOffset += TQ_IDX_SIZE;
if(idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) {
nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE);
//TODO: handle error with tfile
// TODO: handle error with tfile
ASSERT(nBytes == TQ_PAGE_SIZE);
memset(&idxBuf, 0, TQ_PAGE_SIZE);
idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE;
bufPtr = (int64_t*)&idxBuf.buffer;
}
//remove from unpersist list
// remove from unpersist list
pHead->unpersistNext = pNode->unpersistNext;
pHead->unpersistNext->unpersistPrev = pHead;
pNode->unpersistPrev = pNode->unpersistNext = NULL;
pNode = pHead->unpersistNext;
//remove from bucket
if(pNode->handle.valueInUse == TQ_DELETE_TOKEN &&
pNode->handle.valueInTxn == NULL
) {
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
// remove from bucket
if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) {
int bucketKey = pNode->handle.key & TQ_BUCKET_MASK;
STqMetaList* pBucketHead = pMeta->bucket[bucketKey];
if(pBucketHead == pNode) {
if (pBucketHead == pNode) {
pMeta->bucket[bucketKey] = pNode->next;
} else {
STqMetaList* pBucketNode = pBucketHead;
while(pBucketNode->next != NULL
&& pBucketNode->next != pNode) {
pBucketNode = pBucketNode->next;
while (pBucketNode->next != NULL && pBucketNode->next != pNode) {
pBucketNode = pBucketNode->next;
}
//impossible for pBucket->next == NULL
// impossible for pBucket->next == NULL
ASSERT(pBucketNode->next == pNode);
pBucketNode->next = pNode->next;
}
@ -400,46 +384,45 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
}
}
//write left bytes
// write left bytes
free(pSHead);
//TODO: write new version in tfile
if((char*)bufPtr != idxBuf.buffer) {
// TODO: write new version in tfile
if ((char*)bufPtr != idxBuf.buffer) {
int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset);
//TODO: handle error in tfile
// TODO: handle error in tfile
ASSERT(nBytes == idxBuf.head.writeOffset);
}
//TODO: using fsync in tfile
// TODO: using fsync in tfile
fsync(pMeta->idxFd);
fsync(pMeta->fileFd);
return 0;
}
static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.key == key) {
//TODO: think about thread safety
if(pNode->handle.valueInUse
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
while (pNode) {
if (pNode->handle.key == key) {
// TODO: think about thread safety
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
//change pointer ownership
// change pointer ownership
pNode->handle.valueInUse = value;
return 0;
} else {
pNode = pNode->next;
}
}
STqMetaList *pNewNode = malloc(sizeof(STqMetaList));
if(pNewNode == NULL) {
//TODO: memory error
STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if (pNewNode == NULL) {
// TODO: memory error
return -1;
}
memset(pNewNode, 0, sizeof(STqMetaList));
pNewNode->handle.key = key;
pNewNode->handle.valueInUse = value;
//put into unpersist list
// put into unpersist list
pNewNode->unpersistPrev = pMeta->unpersistHead;
pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext;
pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode;
@ -448,12 +431,11 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu
}
void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.key == key) {
if(pNode->handle.valueInUse != NULL
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
return pNode->handle.valueInUse;
} else {
return NULL;
@ -466,12 +448,11 @@ void* tqHandleGet(STqMetaStore* pMeta, int64_t key) {
}
void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.key == key) {
if(pNode->handle.valueInUse != NULL
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
tqLinkUnpersist(pMeta, pNode);
return pNode->handle.valueInUse;
} else {
@ -485,16 +466,16 @@ void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) {
}
static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.key == key) {
//TODO: think about thread safety
if(pNode->handle.valueInTxn) {
if(TqDupIntxnReject(pMeta->tqConfigFlag)) {
while (pNode) {
if (pNode->handle.key == key) {
// TODO: think about thread safety
if (pNode->handle.valueInTxn) {
if (TqDupIntxnReject(pMeta->tqConfigFlag)) {
return -2;
}
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
}
@ -505,9 +486,9 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
pNode = pNode->next;
}
}
STqMetaList *pNewNode = malloc(sizeof(STqMetaList));
if(pNewNode == NULL) {
//TODO: memory error
STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if (pNewNode == NULL) {
// TODO: memory error
return -1;
}
memset(pNewNode, 0, sizeof(STqMetaList));
@ -519,14 +500,12 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
return 0;
}
int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) {
return tqHandlePutImpl(pMeta, key, value);
}
int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); }
int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
void *vmem = malloc(vsize);
if(vmem == NULL) {
//TODO: memory error
void* vmem = malloc(vsize);
if (vmem == NULL) {
// TODO: memory error
return -1;
}
memcpy(vmem, value, vsize);
@ -534,12 +513,11 @@ int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vs
}
static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.key == key) {
if(pNode->handle.valueInTxn != NULL
&& pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
return pNode->handle.valueInTxn;
} else {
return NULL;
@ -552,15 +530,14 @@ static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) {
}
int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.key == key) {
if(pNode->handle.valueInTxn == NULL) {
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn == NULL) {
return -1;
}
if(pNode->handle.valueInUse
&& pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
pNode->handle.valueInUse = pNode->handle.valueInTxn;
@ -575,12 +552,12 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
}
int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.key == key) {
if(pNode->handle.valueInTxn) {
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn) {
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
pNode->handle.valueInTxn = NULL;
@ -596,13 +573,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
}
int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
int64_t bucketKey = key & TQ_BUCKET_MASK;
int64_t bucketKey = key & TQ_BUCKET_MASK;
STqMetaList* pNode = pMeta->bucket[bucketKey];
while(pNode) {
if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if(pNode->handle.valueInTxn) {
while (pNode) {
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
if (pNode->handle.valueInTxn) {
pMeta->pDeleter(pNode->handle.valueInTxn);
}
pNode->handle.valueInTxn = TQ_DELETE_TOKEN;
tqLinkUnpersist(pMeta, pNode);
return 0;
@ -610,11 +588,9 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
pNode = pNode->next;
}
}
//no such key
// no such key
return -1;
}
//TODO: clean deleted idx and data from persistent file
int32_t tqStoreCompact(STqMetaStore *pMeta) {
return 0;
}
// TODO: clean deleted idx and data from persistent file
int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; }

10
source/dnode/vnode/tq/test/tqMetaTest.cpp
View File

@ -9,17 +9,17 @@ struct Foo {
int32_t a;
};
int FooSerializer(const void* pObj, TqSerializedHead** ppHead) {
int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
Foo* foo = (Foo*) pObj;
if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(TqSerializedHead) + sizeof(int32_t)) {
*ppHead = (TqSerializedHead*)realloc(*ppHead, sizeof(TqSerializedHead) + sizeof(int32_t));
(*ppHead)->ssize = sizeof(TqSerializedHead) + sizeof(int32_t);
if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
*ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
(*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
}
*(int32_t*)(*ppHead)->content = foo->a;
return (*ppHead)->ssize;
}
const void* FooDeserializer(const TqSerializedHead* pHead, void** ppObj) {
const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
if(*ppObj == NULL) {
*ppObj = realloc(*ppObj, sizeof(int32_t));
}

2
source/libs/catalog/CMakeLists.txt
View File

@ -10,3 +10,5 @@ target_link_libraries(
catalog
PRIVATE os util common transport query
)
ADD_SUBDIRECTORY(test)

7
source/libs/catalog/inc/catalogInt.h
View File

@ -70,9 +70,10 @@ extern int32_t ctgDebugFlag;
#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0)
#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0)
#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#ifdef __cplusplus

452
source/libs/catalog/src/catalog.c
View File

@ -20,50 +20,7 @@
SCatalogMgmt ctgMgmt = {0};
int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SVgroupListInfo** pVgroup) {
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen);
if (code) {
return code;
}
SRpcMsg rpcMsg = {
.msgType = TSDB_MSG_TYPE_VGROUP_LIST,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
code = queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
return code;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetVgroupFromCache(struct SCatalog* pCatalog, SHashObj** pVgroupList, int32_t* exist) {
if (NULL == pCatalog->vgroupCache.cache || pCatalog->vgroupCache.vgroupVersion < 0) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
if (pVgroupList) {
*pVgroupList = pCatalog->vgroupCache.cache;
}
*exist = 1;
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) {
int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo *dbInfo, int32_t *exist) {
if (NULL == pCatalog->dbCache.cache) {
*exist = 0;
return TSDB_CODE_SUCCESS;
@ -71,28 +28,13 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S
SDBVgroupInfo *info = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
if (NULL == info || info->vgroupVersion < pCatalog->vgroupCache.vgroupVersion) {
if (NULL == info) {
*exist = 0;
return TSDB_CODE_SUCCESS;
}
if (dbInfo) {
*dbInfo = calloc(1, sizeof(**dbInfo));
if (NULL == *dbInfo) {
ctgError("calloc size[%d] failed", (int32_t)sizeof(**dbInfo));
return TSDB_CODE_CTG_MEM_ERROR;
}
(*dbInfo)->vgId = taosArrayDup(info->vgId);
if (NULL == (*dbInfo)->vgId) {
ctgError("taos array duplicate failed");
tfree(*dbInfo);
return TSDB_CODE_CTG_MEM_ERROR;
}
(*dbInfo)->vgroupVersion = info->vgroupVersion;
(*dbInfo)->hashRange = info->hashRange;
(*dbInfo)->hashType = info->hashType;
*dbInfo = *info;
}
*exist = 1;
@ -107,26 +49,35 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen);
if (code) {
return code;
CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen));
char *pMsg = rpcMallocCont(msgLen);
if (NULL == pMsg) {
ctgError("rpc malloc %d failed", msgLen);
tfree(msg);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(pMsg, msg, msgLen);
tfree(msg);
SRpcMsg rpcMsg = {
.msgType = TSDB_MSG_TYPE_USE_DB,
.pCont = msg,
.pCont = pMsg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
code = queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
return code;
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("error rsp for use db, code:%x", rpcRsp.code);
CTG_ERR_RET(rpcRsp.code);
}
CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS;
}
@ -157,14 +108,14 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName,
if ((*stbMeta)->suid != tbMeta->suid) {
ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid);
return TSDB_CODE_CTG_INTERNAL_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema);
*pTableMeta = calloc(1, metaSize);
if (NULL == *pTableMeta) {
ctgError("calloc size[%d] failed", metaSize);
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta));
@ -174,7 +125,7 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName,
*pTableMeta = calloc(1, metaSize);
if (NULL == *pTableMeta) {
ctgError("calloc size[%d] failed", metaSize);
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(*pTableMeta, tbMeta, metaSize);
@ -198,7 +149,7 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
char tbFullName[TSDB_TABLE_FNAME_LEN];
@ -210,10 +161,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen);
if (code) {
return code;
}
CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen));
SRpcMsg rpcMsg = {
.msgType = TSDB_MSG_TYPE_TABLE_META,
@ -227,23 +175,20 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
ctgGenEpSet(&epSet, vgroupInfo);
rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("get table meta from mnode failed, error code:%d", rpcRsp.code);
return rpcRsp.code;
ctgError("error rsp for table meta, code:%x", rpcRsp.code);
CTG_ERR_RET(rpcRsp.code);
}
code = queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen);
if (code) {
return code;
}
CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) {
switch (hashType) {
int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) {
switch (hashMethod) {
default:
*fp = MurmurHash3_32;
break;
@ -252,129 +197,68 @@ int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetVgroupFromVgId(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, int32_t vgId, SVgroupInfo *pVgroup) {
int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray* vgroupList) {
SHashObj *vgroupHash = NULL;
CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash));
if (NULL == vgroupHash) {
ctgError("get empty vgroup cache");
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
SVgroupInfo *vgInfo = NULL;
if (NULL == taosHashGetClone(vgroupHash, &vgId, sizeof(vgId), pVgroup)) {
ctgError("vgId[%d] not found in vgroup list", vgId);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
void *pIter = taosHashIterate(dbInfo->vgInfo, NULL);
while (pIter) {
vgInfo = pIter;
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetVgroupFromVgIdBatch(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SArray* vgIds, SArray* vgroupList) {
SHashObj *vgroupHash = NULL;
SVgroupInfo pVgroup = {0};
int32_t vgIdNum = taosArrayGetSize(vgIds);
CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash));
if (NULL == vgroupHash) {
ctgError("get empty vgroup cache");
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
for (int32_t i = 0; i < vgIdNum; ++i) {
int32_t *vgId = taosArrayGet(vgIds, i);
if (NULL == taosArrayPush(vgroupList, vgInfo)) {
ctgError("taosArrayPush failed");
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
if (NULL == taosHashGetClone(vgroupHash, vgId, sizeof(*vgId), &pVgroup)) {
ctgError("vgId[%d] not found in vgroup list", vgId);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
if (NULL == taosArrayPush(vgroupList, &pVgroup)) {
ctgError("push vgroup to array failed, idx:%d", i);
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
pIter = taosHashIterate(dbInfo->vgInfo, pIter);
vgInfo = NULL;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) {
SDBVgroupInfo *dbInfo = NULL;
int32_t code = 0;
CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo));
if (NULL == dbInfo) {
ctgWarn("db[%s] vgroup info not found", pDBName);
return TSDB_CODE_TSC_DB_NOT_SELECTED;
}
if (dbInfo->vgroupVersion < 0 || NULL == dbInfo->vgId) {
ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgId:%p", pDBName, dbInfo->vgroupVersion, dbInfo->vgId);
CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
int32_t vgNum = taosArrayGetSize(dbInfo->vgId);
int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) {
int32_t vgNum = taosHashGetSize(dbInfo->vgInfo);
if (vgNum <= 0) {
ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum);
CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
tableNameHashFp fp = NULL;
SVgroupInfo *vgInfo = NULL;
CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashType, &fp));
CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp));
char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName));
uint32_t hashUnit = dbInfo->hashRange / vgNum;
uint32_t vgId = hashValue / hashUnit;
CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, vgId, pVgroup));
_return:
if (dbInfo && dbInfo->vgId) {
taosArrayDestroy(dbInfo->vgId);
dbInfo->vgId = NULL;
}
tfree(dbInfo);
return code;
}
STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) {
assert(pChild != NULL);
int32_t total = pChild->numOfColumns + pChild->numOfTags;
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total);
pTableMeta->tableType = TSDB_SUPER_TABLE;
pTableMeta->tableInfo.numOfTags = pChild->numOfTags;
pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns;
pTableMeta->tableInfo.precision = pChild->precision;
pTableMeta->uid = pChild->suid;
pTableMeta->tversion = pChild->tversion;
pTableMeta->sversion = pChild->sversion;
memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total);
int32_t num = pTableMeta->tableInfo.numOfColumns;
for(int32_t i = 0; i < num; ++i) {
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
void *pIter = taosHashIterate(dbInfo->vgInfo, NULL);
while (pIter) {
vgInfo = pIter;
if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) {
break;
}
pIter = taosHashIterate(dbInfo->vgInfo, pIter);
vgInfo = NULL;
}
return pTableMeta;
if (NULL == vgInfo) {
ctgError("no hash range found for hashvalue[%u]", hashValue);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
*pVgroup = *vgInfo;
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) {
if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t exist = 0;
@ -393,7 +277,7 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet*
if (0 == exist) {
ctgError("get table meta from cache failed, but fetch succeed");
return TSDB_CODE_CTG_INTERNAL_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
return TSDB_CODE_SUCCESS;
@ -403,19 +287,19 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet*
int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) {
if (output->metaNum != 1 && output->metaNum != 2) {
ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum);
return TSDB_CODE_CTG_INTERNAL_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (NULL == output->tbMeta) {
ctgError("no valid table meta got from meta rsp");
return TSDB_CODE_CTG_INTERNAL_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (NULL == pCatalog->tableCache.cache) {
pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.cache) {
ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
}
@ -423,13 +307,13 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.cache) {
ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.stableCache) {
ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
}
@ -467,7 +351,7 @@ error_exit:
pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
return TSDB_CODE_CTG_INTERNAL_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t catalogInit(SCatalogCfg *cfg) {
@ -489,12 +373,12 @@ int32_t catalogInit(SCatalogCfg *cfg) {
int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) {
if (NULL == clusterId || NULL == catalogHandle) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
if (NULL == ctgMgmt.pCluster) {
ctgError("cluster cache are not ready");
return TSDB_CODE_CTG_NOT_READY;
CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY);
}
size_t clen = strlen(clusterId);
@ -508,7 +392,7 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle)
clusterCtg = calloc(1, sizeof(*clusterCtg));
if (NULL == clusterCtg) {
ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg));
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
@ -516,7 +400,7 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle)
if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) {
ctgError("put cluster %s cache to hash failed", clusterId);
tfree(clusterCtg);
return TSDB_CODE_CTG_INTERNAL_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
*catalogHandle = clusterCtg;
@ -524,98 +408,9 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle)
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) {
if (NULL == pCatalog || NULL == version) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
*version = pCatalog->vgroupCache.vgroupVersion;
return TSDB_CODE_SUCCESS;
}
int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) {
if (NULL == pVgroup) {
ctgError("no valid vgroup list info to update");
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
if (pVgroup->vgroupVersion < 0) {
ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion);
return TSDB_CODE_CTG_INVALID_INPUT;
}
if (NULL == pCatalog->vgroupCache.cache) {
pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_CACHE_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->vgroupCache.cache) {
ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_CACHE_VGROUP_NUMBER);
return TSDB_CODE_CTG_MEM_ERROR;
}
} else {
taosHashClear(pCatalog->vgroupCache.cache);
}
SVgroupInfo *vInfo = NULL;
for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) {
if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &pVgroup->vgroupInfo[i], sizeof(pVgroup->vgroupInfo[i])) != 0) {
ctgError("push to vgroup hash cache failed");
goto error_exit;
}
}
pCatalog->vgroupCache.vgroupVersion = pVgroup->vgroupVersion;
return TSDB_CODE_SUCCESS;
error_exit:
if (pCatalog->vgroupCache.cache) {
taosHashCleanup(pCatalog->vgroupCache.cache);
pCatalog->vgroupCache.cache = NULL;
}
pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash) {
if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) {
return TSDB_CODE_CTG_INVALID_INPUT;
}
int32_t exist = 0;
CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist));
if (exist) {
return TSDB_CODE_SUCCESS;
}
SVgroupListInfo *pVgroup = NULL;
CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup));
CTG_ERR_RET(catalogUpdateVgroupCache(pCatalog, pVgroup));
if (pVgroupHash) {
CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist));
}
if (0 == exist) {
ctgError("catalog fetched but get from cache failed");
return TSDB_CODE_CTG_INTERNAL_ERROR;
}
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) {
if (NULL == pCatalog || NULL == dbName || NULL == version) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
if (NULL == pCatalog->dbCache.cache) {
@ -629,17 +424,17 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName,
return TSDB_CODE_SUCCESS;
}
*version = dbInfo->vgroupVersion;
*version = dbInfo->vgVersion;
return TSDB_CODE_SUCCESS;
}
int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) {
if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
if (dbInfo->vgroupVersion < 0) {
if (dbInfo->vgVersion < 0) {
if (pCatalog->dbCache.cache) {
taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName));
}
@ -652,13 +447,19 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName
pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->dbCache.cache) {
ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
} else {
SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName));
if (oldInfo && oldInfo->vgInfo) {
taosHashCleanup(oldInfo->vgInfo);
oldInfo->vgInfo = NULL;
}
}
if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) {
ctgError("push to vgroup hash cache failed");
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
return TSDB_CODE_SUCCESS;
@ -667,13 +468,12 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo) {
int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) {
if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t exist = 0;
int32_t code = 0;
if (0 == forceUpdate) {
CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist));
@ -688,29 +488,17 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet*
strncpy(input.db, dbName, sizeof(input.db));
input.db[sizeof(input.db) - 1] = 0;
input.vgroupVersion = pCatalog->vgroupCache.vgroupVersion;
input.dbGroupVersion = CTG_DEFAULT_INVALID_VERSION;
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut));
if (DbOut.vgroupList) {
CTG_ERR_JRET(catalogUpdateVgroupCache(pCatalog, DbOut.vgroupList));
}
if (DbOut.dbVgroup) {
CTG_ERR_JRET(catalogUpdateDBVgroupCache(pCatalog, dbName, DbOut.dbVgroup));
}
CTG_ERR_RET(catalogUpdateDBVgroupCache(pCatalog, dbName, &DbOut.dbVgroup));
if (dbInfo) {
*dbInfo = DbOut.dbVgroup;
DbOut.dbVgroup = NULL;
}
_return:
tfree(DbOut.dbVgroup);
tfree(DbOut.vgroupList);
return code;
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
@ -719,12 +507,12 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet*
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) {
if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
SVgroupInfo vgroupInfo = {0};
CTG_ERR_RET(ctgGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo));
CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo));
STableMetaOutput output = {0};
@ -741,24 +529,28 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const
return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta);
}
int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) {
int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
STableMeta *tbMeta = NULL;
int32_t code = 0;
SVgroupInfo vgroupInfo = {0};
SDBVgroupInfo *dbVgroup = NULL;
SDBVgroupInfo dbVgroup = {0};
CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta));
if (tbMeta->tableType == TSDB_SUPER_TABLE) {
CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup));
CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup));
CTG_ERR_JRET(ctgGetVgroupFromVgIdBatch(pCatalog, pRpc, pMgmtEps, dbVgroup->vgId, pVgroupList));
if (tbMeta->tableType == TSDB_SUPER_TABLE) {
CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList));
} else {
CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, tbMeta->vgId, &vgroupInfo));
int32_t vgId = tbMeta->vgId;
if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) {
ctgError("vgId[%d] not found in vgroup list", vgId);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) {
ctgError("push vgroupInfo to array failed");
@ -768,20 +560,32 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe
_return:
tfree(tbMeta);
if (dbVgroup && dbVgroup->vgId) {
taosArrayDestroy(dbVgroup->vgId);
dbVgroup->vgId = NULL;
CTG_RET(code);
}
int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) {
SDBVgroupInfo dbInfo = {0};
int32_t code = 0;
int32_t vgId = 0;
CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo));
if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) {
ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo);
CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED);
}
tfree(dbVgroup);
return code;
CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup));
CTG_RET(code);
}
int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) {
return TSDB_CODE_CTG_INVALID_INPUT;
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
@ -793,7 +597,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* p
pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES);
if (NULL == pRsp->pTableMeta) {
ctgError("taosArrayInit num[%d] failed", tbNum);
return TSDB_CODE_CTG_MEM_ERROR;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
}
@ -827,7 +631,7 @@ _return:
taosArrayDestroy(pRsp->pTableMeta);
}
return code;
CTG_RET(code);
}
void catalogDestroy(void) {

18
source/libs/catalog/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
MESSAGE(STATUS "build catalog unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
catalogTest
PUBLIC os util common catalog transport gtest query taos
)
TARGET_INCLUDE_DIRECTORIES(
catalogTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/catalog/inc"
)

125
source/libs/catalog/test/catalogTests.cpp
View File

@ -0,0 +1,125 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <tglobal.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "catalog.h"
#include "tep.h"
#include "trpc.h"
typedef struct SAppInstInfo {
int64_t numOfConns;
SCorEpSet mgmtEp;
} SAppInstInfo;
typedef struct STscObj {
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char acctId[TSDB_ACCT_ID_LEN];
char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
uint32_t connId;
uint64_t id; // ref ID returned by taosAddRef
// struct SSqlObj *sqlList;
void *pTransporter;
pthread_mutex_t mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj from this tscObj
SAppInstInfo *pAppInfo;
} STscObj;
namespace {
void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) {
SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg));
strcpy(pReq->db, "1.db1");
pReq->numOfVgroups = htonl(2);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replications = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->ignoreExist = 1;
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateDbMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB;
SRpcMsg rpcRsp = {0};
rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp);
ASSERT_EQ(rpcRsp.code, 0);
}
}
TEST(testCase, normalCase) {
STscObj* pConn = (STscObj *)taos_connect("127.0.0.1", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
char *clusterId = "cluster1";
char *dbname = "1.db1";
char *tablename = "table1";
struct SCatalog* pCtg = NULL;
void *mockPointer = (void *)0x1;
SVgroupInfo vgInfo = {0};
msgInit();
sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet);
int32_t code = catalogInit(NULL);
ASSERT_EQ(code, 0);
code = catalogGetHandle(clusterId, &pCtg);
ASSERT_EQ(code, 0);
code = catalogGetTableHashVgroup(pCtg, pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet, dbname, tablename, &vgInfo);
ASSERT_EQ(code, 0);
taos_close(pConn);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

11
source/libs/index/inc/indexInt.h
View File

@ -17,7 +17,10 @@
#define _TD_INDEX_INT_H_
#include "index.h"
#include "index_fst.h"
#include "tlog.h"
#include "thash.h"
#include "taos.h"
#ifdef USE_LUCENE
#include <lucene++/Lucene_c.h>
@ -32,12 +35,20 @@ struct SIndex {
#ifdef USE_LUCENE
index_t *index;
#endif
void *cache;
void *tindex;
SHashObj *fieldObj; // <field name, field id>
uint64_t suid;
int fieldId;
pthread_mutex_t mtx;
};
struct SIndexOpts {
#ifdef USE_LUCENE
void *opts;
#endif
int32_t numOfItermLimit;
int8_t mergeInterval;
};
struct SIndexMultiTermQuery {

53
source/libs/index/inc/index_cache.h Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __INDEX_CACHE_H__
#define __INDEX_CACHE_H__
#include "index.h"
#include "tlockfree.h"
// ----------------- row structure in skiplist ---------------------
/* A data row, the format is like below:
* |<--totalLen-->|<-- fieldId-->|<-- value len--->|<-- value-->|<--version--->|<-- itermType -->|
*
*/
#ifdef __cplusplus
extern "C" {
#endif
typedef struct IndexCache {
T_REF_DECLARE()
int cVersion; //
} IndexCache;
//
IndexCache *indexCacheCreate();
void indexCacheDestroy(IndexCache *cache);
int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldVale, int32_t fvlen, uint64_t uid, int8_t operaType);
int indexCacheGet(IndexCache *cache, uint64_t *rst);
int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result);
#ifdef __cplusplus
}
#endif
#endif

47
source/libs/index/src/index.c
View File

@ -15,39 +15,58 @@
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#ifdef USE_LUCENE
#include "lucene++/Lucene_c.h"
#endif
static pthread_once_t isInit = PTHREAD_ONCE_INIT;
typedef struct SIdxFieldInfo {
int id; // generated by index internal
int type; // field type
} SIdxFieldInfo;
static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit();
static int indexMergeCacheIntoTindex(struct SIndex *sIdx) {
if (sIdx == NULL) {
return -1;
}
indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
return 0;
}
SIndex *indexOpen(SIndexOpts *opts, const char *path) {
pthread_once(&isInit, indexInit);
SIndex *sIdx = malloc(sizeof(SIndex));
#ifdef USE_LUCENE
index_t *index = index_open(path);
SIndex *p = malloc(sizeof(SIndex));
p->index = index;
return p;
sIdx->index = index;
#endif
return NULL;
sIdx->cache = (void*)indexCacheCreate();
sIdx->tindex = NULL;
sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pthread_mutex_init(&sIdx->mtx, NULL);
return sIdx;
}
void indexClose(SIndex *index) {
void indexClose(SIndex *sIdx) {
#ifdef USE_LUCENE
index_close(index->index);
index->index = NULL;
index_close(sIdex->index);
sIdx->index = NULL;
#endif
free(index);
indexCacheDestroy(sIdx->cache);
taosHashCleanup(sIdx->fieldObj);
pthread_mutex_destroy(&sIdx->mtx);
free(sIdx);
return;
}
#ifdef USE_LUCENE
#endif
int indexPut(SIndex *index, SArray* field_vals, int uid) {
#ifdef USE_LUCENE
index_document_t *doc = index_document_create();
@ -63,6 +82,8 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) {
index_put(index->index, doc);
index_document_destroy(doc);
#endif
pthread_mutex_lock(&index->mtx);
pthread_mutex_unlock(&index->mtx);
return 1;
}
@ -105,7 +126,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
return 1;
}
int indexDelete(SIndex *index, SIndexMultiTermQuery *query) {
return 1;
}
int indexRebuild(SIndex *index, SIndexOpts *opts);

119
source/libs/index/src/index_cache.c Normal file
View File

@ -0,0 +1,119 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "index_cache.h"
static int32_t compareKey(const void *l, const void *r) {
char *lp = (char *)l;
char *rp = (char *)r;
// skip total len
int32_t ll, rl; // len
memcpy(&ll, lp, sizeof(int32_t));
memcpy(&rl, rp, sizeof(int32_t));
lp += sizeof(int32_t);
rp += sizeof(int32_t);
// compare field id
int32_t lf, rf; // field id
memcpy(&lf, lp, sizeof(lf));
memcpy(&rf, rp, sizeof(rf));
if (lf != rf) {
return lf < rf ? -1: 1;
}
lp += sizeof(lf);
rp += sizeof(rf);
// compare field value
int32_t lfl, rfl;
memcpy(&lfl, lp, sizeof(lfl));
memcpy(&rfl, rp, sizeof(rfl));
lp += sizeof(lfl);
rp += sizeof(rfl);
//refator later
int32_t i, j;
for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) {
if (lp[i] == rp[j]) { continue; }
else { return lp[i] < rp[j] ? -1 : 1;}
}
if (i < lfl) { return 1;}
else if (j < rfl) { return -1; }
lp += lfl;
rp += rfl;
// compare version
int32_t lv, rv;
memcpy(&lv, lp, sizeof(lv));
memcpy(&rv, rp, sizeof(rv));
if (lv != rv) {
return lv > rv ? -1 : 1;
}
lp += sizeof(lv);
rp += sizeof(rv);
return 0;
}
IndexCache *indexCacheCreate() {
IndexCache *cache = calloc(1, sizeof(IndexCache));
return cache;
}
void indexCacheDestroy(IndexCache *cache) {
free(cache);
}
int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
if (cache == NULL) { return -1;}
int32_t version = T_REF_INC(cache);
int32_t total = sizeof(int32_t) + sizeof(fieldId) + 4 + fvlen + sizeof(version) + sizeof(uid) + sizeof(operType);
char *buf = calloc(1, total);
char *p = buf;
memcpy(buf, &total, sizeof(total));
total += total;
memcpy(buf, &fieldId, sizeof(fieldId));
buf += sizeof(fieldId);
memcpy(buf, &fvlen, sizeof(fvlen));
buf += sizeof(fvlen);
memcpy(buf, fieldValue, fvlen);
buf += fvlen;
memcpy(buf, &version, sizeof(version));
buf += sizeof(version);
memcpy(buf, &uid, sizeof(uid));
buf += sizeof(uid);
memcpy(buf, &operType, sizeof(operType));
buf += sizeof(operType);
}
int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result) {
return 0;
}

2
source/libs/index/src/index_fst.c
View File

@ -1330,7 +1330,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb
SArray *nodes = taosArrayInit(8, sizeof(FstNode *));
while (taosArrayGetSize(sws->stack) > 0) {
StreamState *p = (StreamState *)taosArrayPop(sws->stack);
if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) {
if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) {
if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
taosArrayPop(sws->inp);
}

11
source/libs/index/src/index_fst_automation.c
View File

@ -87,9 +87,18 @@ static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) {
if (ssv == NULL || ctx == NULL) {return NULL;}
char *data = ctx->data;
if (ssv->kind == Done) {
return startWithStateValueCreate(Done, FST_INT, &ssv->val);
}
if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
int val = ssv->val + 1;
return startWithStateValueCreate(Running, FST_INT, &val);
StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val);
if (prefixIsMatch(ctx, nsv)) {
nsv->kind = Done;
} else {
nsv->kind = Running;
}
return nsv;
}
return NULL;
}

9
source/libs/parser/inc/astToMsg.h Normal file
View File

@ -0,0 +1,9 @@
#ifndef TDENGINE_ASTTOMSG_H
#define TDENGINE_ASTTOMSG_H
#include "parserInt.h"
#include "taosmsg.h"
SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen);
#endif // TDENGINE_ASTTOMSG_H

9
source/libs/parser/inc/parserInt.h
View File

@ -53,6 +53,15 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t
*/
int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen);
/**
* validate the ddl ast, and convert the ast to the corresponding message format
* @param pSqlInfo
* @param output
* @param type
* @return
*/
int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen);
/**
* Evaluate the numeric and timestamp arithmetic expression in the WHERE clause.
* @param pNode

4
source/libs/parser/inc/parserUtil.h
View File

@ -45,6 +45,8 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo);
SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
int32_t parserValidateIdToken(SToken* pToken);
int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf);
int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr);
@ -63,6 +65,8 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo);
STableMeta* tableMetaDup(const STableMeta* pTableMeta);
bool isDclSqlStatement(SSqlInfo* pSqlInfo);
#ifdef __cplusplus
}
#endif

23
source/libs/parser/src/astToMsg.c Normal file
View File

@ -0,0 +1,23 @@
#include "parserInt.h"
SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) {
SCreateUserMsg *pMsg = (SCreateUserMsg *)calloc(1, sizeof(SCreateUserMsg));
if (pMsg == NULL) {
// tscError("0x%" PRIx64 " failed to malloc for query msg", id);
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return NULL;
}
SUserInfo *pUser = &pInfo->pMiscInfo->user;
strncpy(pMsg->user, pUser->user.z, pUser->user.n);
pMsg->type = pUser->type;
pMsg->superUser = (int8_t)pUser->type;
if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
// pMsg->privilege = (char)pCmd->count;
} else {
strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n);
}
return pMsg;
}

264
source/libs/parser/src/astValidate.c
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <function.h>
#include "astGenerator.h"
#include "function.h"
#include "parserInt.h"
@ -23,6 +22,7 @@
#include "tglobal.h"
#include "tmsgtype.h"
#include "ttime.h"
#include "astToMsg.h"
#define TSQL_TBNAME_L "tbname"
#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@ -3636,11 +3636,14 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf)
}
int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msgBuf, int32_t msgBufLen) {
//1. if it is a query, get the meta info and continue.
assert(pCatalog != NULL && pInfo != NULL);
int32_t code = 0;
#if 0
SMsgBuf m = {.buf = msgBuf, .len = msgBufLen};
SMsgBuf *pMsgBuf = &m;
switch (pInfo->type) {
#if 0
case TSDB_SQL_DROP_TABLE:
case TSDB_SQL_DROP_USER:
case TSDB_SQL_DROP_ACCT:
@ -3651,14 +3654,14 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0);
if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (parserValidateIdToken(pzName) != TSDB_CODE_SUCCESS)) {
return setInvalidOperatorMsg(pMsgBuf, msg2);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
if (pInfo->type == TSDB_SQL_DROP_DB) {
assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1);
code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName);
if (code != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg2);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
} else if (pInfo->type == TSDB_SQL_DROP_TABLE) {
@ -3675,7 +3678,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
strncpy(pCmd->payload, pzName->z, pzName->n);
} else { // drop user/account
if (pzName->n >= TSDB_USER_LEN) {
return setInvalidOperatorMsg(pMsgBuf, msg3);
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
strncpy(pCmd->payload, pzName->z, pzName->n);
@ -3689,12 +3692,12 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken);
if (ret != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
break;
@ -3729,19 +3732,19 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt);
if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) {
return setInvalidOperatorMsg(pMsgBuf, msg2);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
char buf[TSDB_DB_NAME_LEN] = {0};
SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf));
if (tscValidateName(&token) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token);
if (ret != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg2);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) {
@ -3755,7 +3758,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
const char* msg = "invalid host name (ip address)";
if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) {
return setInvalidOperatorMsg(pMsgBuf, msg);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0);
@ -3779,11 +3782,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
}
if (pName->n >= TSDB_USER_LEN) {
return setInvalidOperatorMsg(pMsgBuf, msg3);
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
if (tscValidateName(pName) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg2);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt;
@ -3793,7 +3796,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
} else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) {
} else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) {
} else {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
}
@ -3805,7 +3808,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
// additional msg has been attached already
code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql);
@ -3821,7 +3824,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql);
@ -3836,11 +3839,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0);
if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
if (pToken->n > TSDB_DB_NAME_LEN) {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken);
}
@ -3853,7 +3856,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
/* validate the parameter names and options */
if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg2);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
char* pMsg = pCmd->payload;
@ -3867,7 +3870,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
strncpy(pCfg->ep, t0->z, t0->n);
if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg3);
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
strncpy(pCfg->config, t1->z, t1->n);
@ -3882,65 +3885,13 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
break;
}
case TSDB_SQL_CREATE_USER:
case TSDB_SQL_ALTER_USER: {
const char* msg2 = "invalid user/account name";
const char* msg3 = "name too long";
const char* msg5 = "invalid user rights";
const char* msg7 = "not support options";
pCmd->command = pInfo->type;
SUserInfo* pUser = &pInfo->pMiscInfo->user;
SToken* pName = &pUser->user;
SToken* pPwd = &pUser->passwd;
if (pName->n >= TSDB_USER_LEN) {
return setInvalidOperatorMsg(pMsgBuf, msg3);
}
if (tscValidateName(pName) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg2);
}
if (pCmd->command == TSDB_SQL_CREATE_USER) {
if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
} else {
if (pUser->type == TSDB_ALTER_USER_PASSWD) {
if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
} else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
assert(pPwd->type == TSDB_DATA_TYPE_NULL);
SToken* pPrivilege = &pUser->privilege;
if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) {
pCmd->count = 1;
} else if (strncasecmp(pPrivilege->z, "read", 4) == 0 && pPrivilege->n == 4) {
pCmd->count = 2;
} else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) {
pCmd->count = 3;
} else {
return setInvalidOperatorMsg(pMsgBuf, msg5);
}
} else {
return setInvalidOperatorMsg(pMsgBuf, msg7);
}
}
break;
}
case TSDB_SQL_CFG_LOCAL: {
SMiscInfo *pMiscInfo = pInfo->pMiscInfo;
const char *msg = "invalid configure options or values";
// validate the parameter names and options
if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a);
@ -3995,7 +3946,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size);
if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
// normalizeSqlNode(pSqlNode); // normalize the column name in each function
@ -4061,21 +4012,22 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1);
code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName);
if (code != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg1);
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
break;
}
case TSDB_SQL_COMPACT_VNODE:{
const char* msg = "invalid compact";
if (setCompactVnodeInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) {
return setInvalidOperatorMsg(pMsgBuf, msg);
return buildInvalidOperationMsg(pMsgBuf, msg);
}
break;
}
#endif
default:
return setInvalidOperatorMsg(pMsgBuf, "not support sql expression");
return buildInvalidOperationMsg(pMsgBuf, "not support sql expression");
}
#endif
SCatalogReq req = {0};
SMetaData data = {0};
@ -4114,5 +4066,159 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer
}
return code;
}
// todo remove it
static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLen, SMsgBuf* pMsgBuf) {
const char* msg1 = "invalid name";
const char* msg2 = "wildcard string should be less than %d characters";
const char* msg3 = "database name too long";
const char* msg4 = "pattern is invalid";
const char* msg5 = "database name is empty";
const char* msg6 = "pattern string is empty";
/*
* database prefix in pInfo->pMiscInfo->a[0]
* wildcard in like clause in pInfo->pMiscInfo->a[1]
*/
SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt;
int16_t showType = pShowInfo->showType;
if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_VGROUP) {
SToken* pDbPrefixToken = &pShowInfo->prefix;
if (pDbPrefixToken->type != 0) {
if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
if (pDbPrefixToken->n <= 0) {
return buildInvalidOperationMsg(pMsgBuf, msg5);
}
if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
// int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken);
// if (ret != TSDB_CODE_SUCCESS) {
// return buildInvalidOperationMsg(pMsgBuf, msg1);
// }
}
// show table/stable like 'xxxx', set the like pattern for show tables
SToken* pPattern = &pShowInfo->pattern;
if (pPattern->type != 0) {
if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) {
return buildInvalidOperationMsg(pMsgBuf, msg4);
}
pPattern->n = strdequote(pPattern->z);
if (pPattern->n <= 0) {
return buildInvalidOperationMsg(pMsgBuf, msg6);
}
if (pPattern->n > tsMaxWildCardsLen) {
char tmp[64] = {0};
sprintf(tmp, msg2, tsMaxWildCardsLen);
return buildInvalidOperationMsg(pMsgBuf, tmp);
}
}
} else if (showType == TSDB_MGMT_TABLE_VNODES) {
if (pShowInfo->prefix.type == 0) {
return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep");
}
if (pShowInfo->prefix.type == TK_STRING) {
pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z);
}
}
SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg));
pShowMsg->type = pShowInfo->showType;
if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
SToken* pPattern = &pShowInfo->pattern;
if (pPattern->type > 0) { // only show tables support wildcard query
strncpy(pShowMsg->payload, pPattern->z, pPattern->n);
pShowMsg->payloadLen = htons(pPattern->n);
}
} else {
SToken* pEpAddr = &pShowInfo->prefix;
assert(pEpAddr->n > 0 && pEpAddr->type > 0);
strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n);
pShowMsg->payloadLen = htons(pEpAddr->n);
}
*output = pShowMsg;
*msgLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen);
return TSDB_CODE_SUCCESS;
}
int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) {
int32_t code = 0;
SMsgBuf m = {.buf = msgBuf, .len = msgBufLen};
SMsgBuf *pMsgBuf = &m;
*type = pInfo->type;
switch (pInfo->type) {
case TSDB_SQL_CREATE_USER:
case TSDB_SQL_ALTER_USER: {
const char* msg1 = "not support options";
const char* msg2 = "invalid user/account name";
const char* msg3 = "name too long";
const char* msg4 = "invalid user rights";
SUserInfo* pUser = &pInfo->pMiscInfo->user;
SToken* pName = &pUser->user;
SToken* pPwd = &pUser->passwd;
if (pName->n >= TSDB_USER_LEN) {
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
if (pInfo->type == TSDB_SQL_CREATE_USER) {
if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
} else {
if (pUser->type == TSDB_ALTER_USER_PASSWD) {
if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_OPERATION;
}
} else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) {
assert(pPwd->type == TSDB_DATA_TYPE_NULL);
SToken* pPrivilege = &pUser->privilege;
if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) {
// pCmd->count = 1;
} else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) {
// pCmd->count = 2;
} else {
return buildInvalidOperationMsg(pMsgBuf, msg4);
}
} else {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
}
*output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen);
break;
}
case TSDB_SQL_SHOW: {
code = setShowInfo(pInfo, output, outputLen, pMsgBuf);
break;
}
default:
break;
}
return code;
}

36
source/libs/parser/src/insertParser.c
View File

@ -71,8 +71,7 @@ typedef struct SInsertParseContext {
const char* pSql;
SMsgBuf msg;
struct SCatalog* pCatalog;
SMetaData meta; // need release
const STableMeta* pTableMeta;
STableMeta* pTableMeta;
SHashObj* pTableBlockHashObj; // data block for each table. need release
int32_t totalNum;
SInsertStmtInfo* pOutput;
@ -165,29 +164,29 @@ static int32_t skipInsertInto(SInsertParseContext* pCxt) {
return TSDB_CODE_SUCCESS;
}
static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) {
static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullDbName, char* tableName) {
if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) {
return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z);
}
SName name = {0};
strcpy(name.dbname, pCxt->pComCxt->pDbname);
strncpy(name.tname, pStname->z, pStname->n);
taosArrayPush(tableNameList, &name);
char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false);
if (NULL != p) { // db.table
strcpy(fullDbName, pCxt->pComCxt->pAcctId);
fullDbName[strlen(pCxt->pComCxt->pAcctId)] = TS_PATH_DELIMITER[0];
strncpy(fullDbName, pStname->z, p - pStname->z);
strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1);
} else {
snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%s.%s", pCxt->pComCxt->pAcctId, pCxt->pComCxt->pDbname);
strncpy(tableName, pStname->z, pStname->n);
}
return TSDB_CODE_SUCCESS;
}
static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalogReq* pMetaReq) {
pMetaReq->pTableName = taosArrayInit(4, sizeof(SName));
return buildTableName(pCxt, pStname, pMetaReq->pTableName);
}
static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
SCatalogReq req;
CHECK_CODE(buildMetaReq(pCxt, pTname, &req));
CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, NULL, NULL)); //TODO
pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0);
char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0};
char tableName[TSDB_TABLE_NAME_LEN] = {0};
CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName));
CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta));
return TSDB_CODE_SUCCESS;
}
@ -868,12 +867,11 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
.pOutput = *pInfo
};
CHECK_CODE(catalogGetHandle(NULL, &context.pCatalog)); //TODO
if (NULL == context.pTableBlockHashObj) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog));
CHECK_CODE(skipInsertInto(&context));
CHECK_CODE(parseInsertBody(&context));

37
source/libs/parser/src/parser.c
View File

@ -31,25 +31,36 @@ bool qIsInsertSql(const char* pStr, size_t length) {
} while (1);
}
int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen) {
*pQueryInfo = calloc(1, sizeof(SQueryStmtInfo));
if (*pQueryInfo == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code.
}
int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) {
SSqlInfo info = doGenerateAST(pStr);
if (!info.valid) {
strncpy(msg, info.msg, msgLen);
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
return terrno;
}
struct SCatalog* pCatalog = NULL;
int32_t code = catalogGetHandle(NULL, &pCatalog);
if (code) {
return code;
if (isDclSqlStatement(&info)) {
int32_t code = qParserValidateDclSqlNode(&info, id, pOutput, outputLen, type, msg, msgLen);
if (code == TSDB_CODE_SUCCESS) {
// do nothing
}
} else {
SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo));
if (pQueryInfo == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code.
return terrno;
}
struct SCatalog* pCatalog = NULL;
int32_t code = catalogGetHandle(NULL, &pCatalog);
code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen);
if (code == TSDB_CODE_SUCCESS) {
*pOutput = pQueryInfo;
}
}
return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen);
destroySqlInfo(&info);
return TSDB_CODE_SUCCESS;
}
int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {

48
source/libs/parser/src/parserUtil.c
View File

@ -12,7 +12,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "parserUtil.h"
#include "taosmsg.h"
#include "parser.h"
@ -22,6 +22,8 @@
#include "thash.h"
#include "tbuffer.h"
#include "parserInt.h"
#include "parserUtil.h"
#include "tmsgtype.h"
#include "queryInfoUtil.h"
#include "function.h"
@ -97,12 +99,35 @@ int32_t parserValidateIdToken(SToken* pToken) {
return TSDB_CODE_SUCCESS;
}
int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) {
const char* msg1 = "password can not be empty";
const char* msg2 = "name or password too long";
const char* msg3 = "password needs single quote marks enclosed";
if (pToken->type != TK_STRING) {
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
strdequote(pToken->z);
pToken->n = (uint32_t)strtrim(pToken->z); // trim space before and after passwords
if (pToken->n <= 0) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
if (pToken->n >= TSDB_USET_PASSWORD_LEN) {
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
return TSDB_CODE_SUCCESS;
}
int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) {
strncpy(pBuf->buf, msg, pBuf->len);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) {
int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) {
const char* msgFormat1 = "syntax error near \'%s\'";
const char* msgFormat2 = "syntax error near \'%s\' (%s)";
const char* msgFormat3 = "%s";
@ -584,21 +609,6 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo) {
return pFieldInfo->numOfOutput;
}
int32_t getFirstInvisibleFieldPos(SQueryStmtInfo* pQueryInfo) {
if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) {
return 0;
}
for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) {
SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i);
if (!pField->visible) {
return i;
}
}
return pQueryInfo->fieldsInfo.numOfOutput;
}
SInternalField* appendFieldInfo(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) {
assert(pFieldInfo != NULL);
pFieldInfo->numOfOutput++;
@ -1583,6 +1593,10 @@ uint32_t convertRelationalOperator(SToken *pToken) {
}
}
bool isDclSqlStatement(SSqlInfo* pSqlInfo) {
return (pSqlInfo->type != TSDB_SQL_SELECT);
}
#if 0
int32_t tscCreateQueryFromQueryInfo(SQueryStmtInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) {
memset(pQueryAttr, 0, sizeof(SQueryAttr));

11
source/libs/parser/test/CMakeLists.txt
View File

@ -6,13 +6,16 @@ SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(parserTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
parserTest
PUBLIC os util common parser catalog transport gtest function planner query
)
TARGET_INCLUDE_DIRECTORIES(
parserTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc"
)
TARGET_LINK_LIBRARIES(
parserTest
PUBLIC os util common parser catalog transport gtest function planner query
)
TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc)

21
source/libs/parser/test/insertTest.cpp
View File

@ -27,6 +27,27 @@ namespace {
}
}
extern "C" {
#include <execinfo.h>
void *__real_malloc(size_t);
void *__wrap_malloc(size_t c) {
// printf("My MALLOC called: %d\n", c);
// void *array[32];
// int size = backtrace(array, 32);
// char **symbols = backtrace_symbols(array, size);
// for (int i = 0; i < size; ++i) {
// cout << symbols[i] << endl;
// }
// free(symbols);
return __real_malloc(c);
}
}
// syntax:
// INSERT INTO
// tb_name

43
source/libs/parser/test/parserTests.cpp
View File

@ -29,6 +29,7 @@
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
#include "parserUtil.h"
namespace {
void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_t colId) {
@ -700,5 +701,47 @@ TEST(testCase, function_Test6) {
destroyQueryInfo(pQueryInfo);
qParserClearupMetaRequestInfo(&req);
destroySqlInfo(&info1);
}
TEST(testCase, show_user_Test) {
char msg[128] = {0};
SMsgBuf buf;
buf.len = 128;
buf.buf = msg;
char sql1[] = "show users";
SSqlInfo info1 = doGenerateAST(sql1);
ASSERT_EQ(info1.valid, true);
void* output = NULL;
int32_t type = 0;
int32_t len = 0;
int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len);
ASSERT_EQ(code, 0);
// convert the show command to be the select query
// select name, privilege, create_time, account from information_schema.users;
}
TEST(testCase, create_user_Test) {
char msg[128] = {0};
SMsgBuf buf;
buf.len = 128;
buf.buf = msg;
char sql[] = {"create user abc pass 'abc'"};
SSqlInfo info1 = doGenerateAST(sql);
ASSERT_EQ(info1.valid, true);
ASSERT_EQ(isDclSqlStatement(&info1), true);
void* output = NULL;
int32_t type = 0;
int32_t len = 0;
int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len);
ASSERT_EQ(code, 0);
destroySqlInfo(&info1);
}

85
source/libs/planner/inc/plannerInt.h
View File

@ -23,25 +23,23 @@ extern "C" {
#include "common.h"
#include "tarray.h"
#include "planner.h"
#include "parser.h"
#include "taosmsg.h"
enum LOGIC_PLAN_E {
LP_SCAN = 1,
LP_SESSION = 2,
LP_STATE = 3,
LP_INTERVAL = 4,
LP_FILL = 5,
LP_AGG = 6,
LP_JOIN = 7,
LP_PROJECT = 8,
LP_DISTINCT = 9,
LP_ORDER = 10
};
typedef struct SQueryNodeBasicInfo {
int32_t type; // operator type
char *name; // operator name
} SQueryNodeBasicInfo;
#define QNODE_TAGSCAN 1
#define QNODE_TABLESCAN 2
#define QNODE_PROJECT 3
#define QNODE_AGGREGATE 4
#define QNODE_GROUPBY 5
#define QNODE_LIMIT 6
#define QNODE_JOIN 7
#define QNODE_DISTINCT 8
#define QNODE_SORT 9
#define QNODE_UNION 10
#define QNODE_TIMEWINDOW 11
#define QNODE_SESSIONWINDOW 12
#define QNODE_STATEWINDOW 13
#define QNODE_FILL 14
typedef struct SQueryDistPlanNodeInfo {
bool stableQuery; // super table query or not
@ -52,8 +50,9 @@ typedef struct SQueryDistPlanNodeInfo {
} SQueryDistPlanNodeInfo;
typedef struct SQueryTableInfo {
char *tableName;
uint64_t uid;
char *tableName; // to be deleted
uint64_t uid; // to be deleted
STableMetaInfo* pMeta;
STimeWindow window;
} SQueryTableInfo;
@ -64,50 +63,12 @@ typedef struct SQueryPlanNode {
SArray *pExpr; // the query functions or sql aggregations
int32_t numOfExpr; // number of result columns, which is also the number of pExprs
void *pExtInfo; // additional information
// previous operator to generated result for current node to process
// children operator to generated result for current node to process
// in case of join, multiple prev nodes exist.
SArray *pPrevNodes; // upstream nodes
struct SQueryPlanNode *nextNode;
SArray *pChildren; // upstream nodes
struct SQueryPlanNode *pParent;
} SQueryPlanNode;
typedef SSchema SSlotSchema;
typedef struct SDataBlockSchema {
int32_t index;
SSlotSchema *pSchema;
int32_t numOfCols; // number of columns
} SDataBlockSchema;
typedef struct SPhyNode {
SQueryNodeBasicInfo info;
SArray *pTargets; // target list to be computed or scanned at this node
SArray *pConditions; // implicitly-ANDed qual conditions
SDataBlockSchema targetSchema;
// children plan to generated result for current node to process
// in case of join, multiple plan nodes exist.
SArray *pChildren;
} SPhyNode;
typedef struct SScanPhyNode {
SPhyNode node;
uint64_t uid; // unique id of the table
} SScanPhyNode;
typedef SScanPhyNode STagScanPhyNode;
typedef SScanPhyNode SSystemTableScanPhyNode;
typedef struct SMultiTableScanPhyNode {
SScanPhyNode scan;
SArray *pTagsConditions; // implicitly-ANDed tag qual conditions
} SMultiTableScanPhyNode;
typedef SMultiTableScanPhyNode SMultiTableSeqScanPhyNode;
typedef struct SProjectPhyNode {
SPhyNode node;
} SProjectPhyNode;
/**
* Optimize the query execution plan, currently not implement yet.
* @param pQueryNode
@ -138,6 +99,8 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
*/
int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag);
/**
* Convert to physical plan to string to enable to print it out in the shell.
* @param pPhyNode
@ -150,7 +113,7 @@ int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str);
* Destroy the query plan object.
* @return
*/
void* destroyQueryPlan(struct SQueryPlanNode* pQueryNode);
void destroyQueryPlan(struct SQueryPlanNode* pQueryNode);
/**
* Destroy the physical plan.

604
source/libs/planner/src/logicPlan.c Normal file
View File

@ -0,0 +1,604 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "function.h"
#include "os.h"
#include "parser.h"
#include "plannerInt.h"
typedef struct SFillEssInfo {
int32_t fillType; // fill type
int64_t *val; // fill value
} SFillEssInfo;
typedef struct SJoinCond {
bool tagExists; // denote if tag condition exists or not
SColumn *tagCond[2];
SColumn *colCond[2];
} SJoinCond;
static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo);
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode);
int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len);
int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) {
return 0;
}
int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) {
SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo);
assert(taosArrayGetSize(upstream) == 1);
*pQueryNode = taosArrayGetP(upstream, 0);
taosArrayDestroy(upstream);
return TSDB_CODE_SUCCESS;
}
int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) {
return 0;
}
void destroyQueryPlan(SQueryPlanNode* pQueryNode) {
if (pQueryNode == NULL) {
return;
}
doDestroyQueryNode(pQueryNode);
}
//======================================================================================================================
static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev,
SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) {
SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode));
pNode->info.type = type;
pNode->info.name = strdup(name);
pNode->numOfExpr = numOfOutput;
pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES);
for(int32_t i = 0; i < numOfOutput; ++i) {
taosArrayPush(pNode->pExpr, &pExpr[i]);
}
pNode->pChildren = taosArrayInit(4, POINTER_BYTES);
for(int32_t i = 0; i < numOfPrev; ++i) {
taosArrayPush(pNode->pChildren, &prev[i]);
}
switch(type) {
case QNODE_TAGSCAN:
case QNODE_TABLESCAN: {
SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo));
memcpy(info, pExtInfo, sizeof(SQueryTableInfo));
info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName);
pNode->pExtInfo = info;
break;
}
case QNODE_TIMEWINDOW: {
SInterval* pInterval = calloc(1, sizeof(SInterval));
pNode->pExtInfo = pInterval;
memcpy(pInterval, pExtInfo, sizeof(SInterval));
break;
}
case QNODE_STATEWINDOW: {
SColumn* psw = calloc(1, sizeof(SColumn));
pNode->pExtInfo = psw;
memcpy(psw, pExtInfo, sizeof(SColumn));
break;
}
case QNODE_SESSIONWINDOW: {
SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow));
pNode->pExtInfo = pSessionWindow;
memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow));
break;
}
case QNODE_GROUPBY: {
SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo;
SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr));
pGroupbyExpr->groupbyTag = p->groupbyTag;
pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo);
pNode->pExtInfo = pGroupbyExpr;
break;
}
case QNODE_FILL: { // todo !!
pNode->pExtInfo = pExtInfo;
break;
}
case QNODE_LIMIT: {
pNode->pExtInfo = calloc(1, sizeof(SLimit));
memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit));
break;
}
case QNODE_SORT: {
pNode->pExtInfo = taosArrayDup(pExtInfo);
break;
}
default:
break;
}
return pNode;
}
static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info,
SArray* pExprs, SArray* tableCols) {
if (pQueryInfo->info.onlyTagQuery) {
int32_t num = (int32_t) taosArrayGetSize(pExprs);
SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info);
if (pQueryInfo->info.distinct) {
pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL);
}
return pNode;
}
SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info);
if (pQueryInfo->info.projectionQuery) {
int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL);
} else {
STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0);
// table source column projection, generate the projection expr
int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols);
SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumn* pCol = taosArrayGetP(tableCols, i);
SSourceParam param = {0};
addIntoSourceParam(&param, NULL, pCol);
SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name);
SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", &param, &s, 0);
pExpr[i] = p;
}
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL);
tfree(pExpr);
}
return pNode;
}
static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) {
// group by column not by tag
size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo);
// check for aggregation
int32_t level = getExprFunctionLevel(pQueryInfo);
for(int32_t i = level - 1; i >= 0; --i) {
SArray* p = pQueryInfo->exprList[i];
size_t num = taosArrayGetSize(p);
bool aggregateFunc = false;
for(int32_t j = 0; j < num; ++j) {
SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0);
if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) {
continue;
}
aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName);
if (aggregateFunc) {
break;
}
}
if (aggregateFunc) {
if (pQueryInfo->interval.interval > 0) {
pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval);
} else if (pQueryInfo->sessionWindow.gap > 0) {
pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow);
} else if (pQueryInfo->stateWindow.col.info.colId > 0) {
pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow);
} else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) {
pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr);
} else {
pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL);
}
} else {
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL);
}
}
if (pQueryInfo->havingFieldNum > 0) {
// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1);
// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL);
}
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo));
pInfo->fillType = pQueryInfo->fillType;
pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t));
memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr);
SArray* p = pQueryInfo->exprList[0]; // top expression in select clause
pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo);
}
if (pQueryInfo->order != NULL) {
SArray* pList = pQueryInfo->exprList[0];
pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order);
}
if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) {
pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit);
}
return pNode;
}
static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs,
SArray* tableCols) {
char name[TSDB_TABLE_FNAME_LEN] = {0};
tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN);
SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
// handle the only tag query
SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols);
if (pQueryInfo->info.onlyTagQuery) {
tfree(info.tableName);
return pNode;
}
SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
tfree(info.tableName);
return pNode1;
}
static bool isAllAggExpr(SArray* pList) {
assert(pList != NULL);
for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) {
SExprInfo* p = taosArrayGetP(pList, k);
if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) {
return false;
}
}
return true;
}
SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) {
SArray* upstream = NULL;
if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause
upstream = taosArrayInit(4, POINTER_BYTES);
size_t size = taosArrayGetSize(pQueryInfo->pUpstream);
for(int32_t i = 0; i < size; ++i) {
SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i);
SArray* p = createQueryPlanImpl(pq);
taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p));
}
}
if (pQueryInfo->numOfTables > 1) { // it is a join query
// 1. separate the select clause according to table
taosArrayDestroy(upstream);
upstream = taosArrayInit(5, POINTER_BYTES);
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i];
uint64_t uid = pTableMetaInfo->pTableMeta->uid;
SArray* exprList = taosArrayInit(4, POINTER_BYTES);
if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
// dropAllExprInfo(exprList);
exit(-1);
}
// 2. create the query execution node
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
// 3. get the required table column list
SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn));
columnListCopy(tableColumnList, pQueryInfo->colList, uid);
// 4. add the projection query node
SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList);
columnListDestroy(tableColumnList);
// dropAllExprInfo(exprList);
taosArrayPush(upstream, &pNode);
}
// 3. add the join node here
SQueryTableInfo info = {0};
int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]);
SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables,
pQueryInfo->exprList[0]->pData, num, NULL);
// 4. add the aggregation or projection execution node
pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
} else { // only one table, normal query process
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
}
return upstream;
}
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
tfree(pQueryNode->pExtInfo);
tfree(pQueryNode->pSchema);
tfree(pQueryNode->info.name);
// dropAllExprInfo(pQueryNode->pExpr);
if (pQueryNode->pChildren != NULL) {
int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren);
for(int32_t i = 0; i < size; ++i) {
SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i);
doDestroyQueryNode(p);
}
taosArrayDestroy(pQueryNode->pChildren);
}
tfree(pQueryNode);
}
static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
if (level > 0) {
sprintf(buf + totalLen, "%*c", level, ' ');
totalLen += level;
}
int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name);
int32_t len = len1 + totalLen;
switch(pQueryNode->info.type) {
case QNODE_TABLESCAN: {
SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid,
pInfo->window.skey, pInfo->window.ekey);
assert(len1 > 0);
len += len1;
for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i);
len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId);
assert(len1 > 0);
len += len1;
}
len1 = sprintf(buf + len, "\n");
assert(len1 > 0);
len += len1;
break;
}
case QNODE_PROJECT: {
len1 = sprintf(buf + len, "cols:");
assert(len1 > 0);
len += len1;
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ")");
len += len1;
// todo print filter info
len1 = sprintf(buf + len, " filters:(nil)\n");
len += len1;
break;
}
case QNODE_AGGREGATE: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_TIMEWINDOW: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ") ");
len += len1;
SInterval* pInterval = pQueryNode->pExtInfo;
// todo dynamic return the time precision
len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n",
pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding,
TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR);
len += len1;
break;
}
case QNODE_STATEWINDOW: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ") ");
len += len1;
SColumn* pCol = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId);
len += len1;
break;
}
case QNODE_SESSIONWINDOW: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ") ");
len += len1;
struct SSessionWindow* ps = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap);
len += len1;
break;
}
case QNODE_GROUPBY: {
len = printExprInfo(buf, pQueryNode, len);
SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, ") groupby_col: ");
len += len1;
for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) {
SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i);
len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId);
len += len1;
}
len += sprintf(buf + len, "\n");
break;
}
case QNODE_FILL: {
SFillEssInfo* pEssInfo = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "%d", pEssInfo->fillType);
len += len1;
if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) {
len1 = sprintf(buf + len, ", val:");
len += len1;
// todo get the correct fill data type
for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]);
len += len1;
if (i < pQueryNode->numOfExpr - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
}
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_LIMIT: {
SLimit* pVal = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset);
len += len1;
break;
}
case QNODE_DISTINCT:
case QNODE_TAGSCAN: {
len1 = sprintf(buf + len, "cols: ");
len += len1;
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_SORT: {
len1 = sprintf(buf + len, "cols:");
len += len1;
SArray* pSort = pQueryNode->pExtInfo;
for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) {
SOrder* p = taosArrayGet(pSort, i);
len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC");
len += len1;
}
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_JOIN: {
// print join condition
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
}
return len;
}
int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len) {
int32_t len1 = 0;
for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
SSqlExpr* pExpr = &pExprInfo->base;
len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId);
assert(len1 > 0);
len += len1;
if (i < pQueryNode->numOfExpr - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
return len;
}
int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen);
for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) {
SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i);
int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len);
len = len1;
}
return len;
}
int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) {
assert(pQueryNode);
*str = calloc(1, 4096);
int32_t len = sprintf(*str, "===== logic plan =====\n");
queryPlanToStringImpl(*str, pQueryNode, 0, len);
return TSDB_CODE_SUCCESS;
}
SQueryPlanNode* queryPlanFromString() {
return NULL;
}

207
source/libs/planner/src/physicalPlan.c
View File

@ -14,23 +14,208 @@
*/
#include "plannerInt.h"
#include "parser.h"
SPhyNode* createScanNode(SQueryPlanNode* pPlanNode) {
return NULL;
#define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan
#define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _
static const char* gOpName[] = {
"Unknown",
#define INCLUDE_AS_NAME
#include "plannerOp.h"
#undef INCLUDE_AS_NAME
};
typedef struct SPlanContext {
struct SCatalog* pCatalog;
struct SQueryDag* pDag;
SSubplan* pCurrentSubplan;
SSubplanId nextId;
} SPlanContext;
static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) {
SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*);
dataBlockSchema->numOfCols = pPlanNode->numOfCols;
}
SPhyNode* createPhyNode(SQueryPlanNode* node) {
switch (node->info.type) {
case LP_SCAN:
return createScanNode(node);
static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) {
SPhyNode* node = (SPhyNode*)calloc(1, size);
node->info.type = type;
node->info.name = gOpName[type];
SWAP(node->pTargets, pPlanNode->pExpr, SArray*);
toDataBlockSchema(pPlanNode, &(node->targetSchema));
}
static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) {
SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size);
node->uid = pTable->pMeta->pTableMeta->uid;
node->tableType = pTable->pMeta->pTableMeta->tableType;
return (SPhyNode*)node;
}
static SPhyNode* createPseudoScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) {
return initScanNode(pPlanNode, pTable, op, sizeof(SScanPhyNode));
}
static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) {
SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo;
return createPseudoScanNode(pPlanNode, pTable, OP_TagScan);
}
static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
// todo
return MASTER_SCAN;
}
static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) {
STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode));
node->scanFlag = getScanFlag(pPlanNode, pTable);
node->window = pTable->window;
// todo tag cond
return (SPhyNode*)node;
}
static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
return createUserTableScanNode(pPlanNode, pTable, OP_TableScan);
}
static bool isSystemTable(SQueryTableInfo* pTable) {
// todo
return false;
}
static bool needSeqScan(SQueryPlanNode* pPlanNode) {
// todo
return false;
}
static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
if (isSystemTable(pTable)) {
return createPseudoScanNode(pPlanNode, pTable, OP_SystemTableScan);
} else if (needSeqScan(pPlanNode)) {
return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan);
}
return NULL;
return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan);
}
SPhyNode* createSubplan(SQueryPlanNode* pSubquery) {
return NULL;
static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
SSubplan* subplan = calloc(1, sizeof(SSubplan));
subplan->id = pCxt->nextId;
++(pCxt->nextId.subplanId);
subplan->type = type;
subplan->level = 0;
if (NULL != pCxt->pCurrentSubplan) {
subplan->level = pCxt->pCurrentSubplan->level + 1;
if (NULL == pCxt->pCurrentSubplan->pChildern) {
pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
}
taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan);
subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan);
}
SArray* currentLevel;
if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) {
currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
taosArrayPush(pCxt->pDag->pSubplans, &currentLevel);
} else {
currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level);
}
taosArrayPush(currentLevel, &subplan);
pCxt->pCurrentSubplan = subplan;
return subplan;
}
int32_t createDag(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) {
return 0;
static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) {
epSet->inUse = 0; // todo
epSet->numOfEps = vg->numOfEps;
for (int8_t i = 0; i < vg->numOfEps; ++i) {
epSet->port[i] = vg->epAddr[i].port;
strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn);
}
return;
}
static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) {
SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList;
for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) {
STORE_CURRENT_SUBPLAN(pCxt);
SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN);
vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet);
subplan->pNode = createMultiTableScanNode(pPlanNode, pTable);
RECOVERY_CURRENT_SUBPLAN(pCxt);
}
return pCxt->nextId.templateId++;
}
static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) {
SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode));
node->srcTemplateId = srcTemplateId;
return (SPhyNode*)node;
}
static bool needMultiNodeScan(SQueryTableInfo* pTable) {
// todo system table, for instance, user_tables
return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType);
}
static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo;
if (needMultiNodeScan(pTable)) {
return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable));
}
return createSingleTableScanNode(pPlanNode, pTable);
}
static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) {
SPhyNode* node = NULL;
switch (pPlanNode->info.type) {
case QNODE_TAGSCAN:
node = createTagScanNode(pPlanNode);
break;
case QNODE_TABLESCAN:
node = createTableScanNode(pCxt, pPlanNode);
break;
default:
assert(false);
}
if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) {
node->pChildren = taosArrayInit(4, POINTER_BYTES);
size_t size = taosArrayGetSize(pPlanNode->pChildren);
for(int32_t i = 0; i < size; ++i) {
SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i));
child->pParent = node;
taosArrayPush(node->pChildren, &child);
}
}
return node;
}
static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) {
SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE);
++(pCxt->nextId.templateId);
subplan->pNode = createPhyNode(pCxt, pRoot);
SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
taosArrayPush(l0, &subplan);
taosArrayPush(pCxt->pDag->pSubplans, &l0);
// todo deal subquery
}
int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) {
SPlanContext context = {
.pCatalog = pCatalog,
.pDag = calloc(1, sizeof(SQueryDag)),
.pCurrentSubplan = NULL,
.nextId = {0} // todo queryid
};
if (NULL == context.pDag) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
context.pDag->pSubplans = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
createSubplanByLevel(&context, pQueryNode);
*pDag = context.pDag;
return TSDB_CODE_SUCCESS;
}
int32_t subPlanToString(struct SSubplan *pPhyNode, char** str) {
return TSDB_CODE_SUCCESS;
}

631
source/libs/planner/src/planner.c
View File

@ -13,620 +13,31 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "function.h"
#include "os.h"
#include "parser.h"
#include "plannerInt.h"
#define QNODE_TAGSCAN 1
#define QNODE_TABLESCAN 2
#define QNODE_PROJECT 3
#define QNODE_AGGREGATE 4
#define QNODE_GROUPBY 5
#define QNODE_LIMIT 6
#define QNODE_JOIN 7
#define QNODE_DISTINCT 8
#define QNODE_SORT 9
#define QNODE_UNION 10
#define QNODE_TIMEWINDOW 11
#define QNODE_SESSIONWINDOW 12
#define QNODE_STATEWINDOW 13
#define QNODE_FILL 14
typedef struct SFillEssInfo {
int32_t fillType; // fill type
int64_t *val; // fill value
} SFillEssInfo;
typedef struct SJoinCond {
bool tagExists; // denote if tag condition exists or not
SColumn *tagCond[2];
SColumn *colCond[2];
} SJoinCond;
static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo);
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode);
int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len);
int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) {
return 0;
void qDestroyQueryDag(struct SQueryDag* pDag) {
// todo
}
int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) {
SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo);
assert(taosArrayGetSize(upstream) == 1);
*pQueryNode = taosArrayGetP(upstream, 0);
taosArrayDestroy(upstream);
int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) {
SQueryPlanNode* logicPlan;
int32_t code = createQueryPlan(pQueryInfo, &logicPlan);
if (TSDB_CODE_SUCCESS != code) {
destroyQueryPlan(logicPlan);
return code;
}
code = optimizeQueryPlan(logicPlan);
if (TSDB_CODE_SUCCESS != code) {
destroyQueryPlan(logicPlan);
return code;
}
code = createDag(logicPlan, NULL, pDag);
if (TSDB_CODE_SUCCESS != code) {
destroyQueryPlan(logicPlan);
qDestroyQueryDag(*pDag);
return code;
}
destroyQueryPlan(logicPlan);
return TSDB_CODE_SUCCESS;
}
int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) {
return 0;
}
int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) {
return 0;
}
int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str) {
return 0;
}
void* destroyQueryPlan(SQueryPlanNode* pQueryNode) {
if (pQueryNode == NULL) {
return NULL;
}
doDestroyQueryNode(pQueryNode);
return NULL;
}
void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode) {
return NULL;
}
//======================================================================================================================
static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev,
SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) {
SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode));
pNode->info.type = type;
pNode->info.name = strdup(name);
pNode->numOfExpr = numOfOutput;
pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES);
for(int32_t i = 0; i < numOfOutput; ++i) {
taosArrayPush(pNode->pExpr, &pExpr[i]);
}
pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES);
for(int32_t i = 0; i < numOfPrev; ++i) {
taosArrayPush(pNode->pPrevNodes, &prev[i]);
}
switch(type) {
case QNODE_TABLESCAN: {
SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo));
memcpy(info, pExtInfo, sizeof(SQueryTableInfo));
info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName);
pNode->pExtInfo = info;
break;
}
case QNODE_TIMEWINDOW: {
SInterval* pInterval = calloc(1, sizeof(SInterval));
pNode->pExtInfo = pInterval;
memcpy(pInterval, pExtInfo, sizeof(SInterval));
break;
}
case QNODE_STATEWINDOW: {
SColumn* psw = calloc(1, sizeof(SColumn));
pNode->pExtInfo = psw;
memcpy(psw, pExtInfo, sizeof(SColumn));
break;
}
case QNODE_SESSIONWINDOW: {
SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow));
pNode->pExtInfo = pSessionWindow;
memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow));
break;
}
case QNODE_GROUPBY: {
SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo;
SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr));
pGroupbyExpr->groupbyTag = p->groupbyTag;
pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo);
pNode->pExtInfo = pGroupbyExpr;
break;
}
case QNODE_FILL: { // todo !!
pNode->pExtInfo = pExtInfo;
break;
}
case QNODE_LIMIT: {
pNode->pExtInfo = calloc(1, sizeof(SLimit));
memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit));
break;
}
case QNODE_SORT: {
pNode->pExtInfo = taosArrayDup(pExtInfo);
break;
}
default:
break;
}
return pNode;
}
static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info,
SArray* pExprs, SArray* tableCols) {
if (pQueryInfo->info.onlyTagQuery) {
int32_t num = (int32_t) taosArrayGetSize(pExprs);
SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, NULL);
if (pQueryInfo->info.distinct) {
pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL);
}
return pNode;
}
SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info);
if (pQueryInfo->info.projectionQuery) {
int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL);
} else {
STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0);
// table source column projection, generate the projection expr
int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols);
SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumn* pCol = taosArrayGetP(tableCols, i);
SSourceParam param = {0};
addIntoSourceParam(&param, NULL, pCol);
SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name);
SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", &param, &s, 0);
pExpr[i] = p;
}
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL);
tfree(pExpr);
}
return pNode;
}
static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) {
// group by column not by tag
size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo);
// check for aggregation
int32_t level = getExprFunctionLevel(pQueryInfo);
for(int32_t i = level - 1; i >= 0; --i) {
SArray* p = pQueryInfo->exprList[i];
size_t num = taosArrayGetSize(p);
bool aggregateFunc = false;
for(int32_t j = 0; j < num; ++j) {
SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0);
if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) {
continue;
}
aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName);
if (aggregateFunc) {
break;
}
}
if (aggregateFunc) {
if (pQueryInfo->interval.interval > 0) {
pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval);
} else if (pQueryInfo->sessionWindow.gap > 0) {
pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow);
} else if (pQueryInfo->stateWindow.col.info.colId > 0) {
pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow);
} else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) {
pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr);
} else {
pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL);
}
} else {
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL);
}
}
if (pQueryInfo->havingFieldNum > 0) {
// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1);
// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL);
}
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo));
pInfo->fillType = pQueryInfo->fillType;
pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t));
memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr);
SArray* p = pQueryInfo->exprList[0]; // top expression in select clause
pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo);
}
if (pQueryInfo->order != NULL) {
SArray* pList = pQueryInfo->exprList[0];
pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order);
}
if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) {
pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit);
}
return pNode;
}
static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs,
SArray* tableCols) {
char name[TSDB_TABLE_FNAME_LEN] = {0};
tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN);
SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
// handle the only tag query
SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols);
if (pQueryInfo->info.onlyTagQuery) {
tfree(info.tableName);
return pNode;
}
SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
tfree(info.tableName);
return pNode1;
}
static bool isAllAggExpr(SArray* pList) {
assert(pList != NULL);
for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) {
SExprInfo* p = taosArrayGetP(pList, k);
if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) {
return false;
}
}
return true;
}
SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) {
SArray* upstream = NULL;
if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause
upstream = taosArrayInit(4, POINTER_BYTES);
size_t size = taosArrayGetSize(pQueryInfo->pUpstream);
for(int32_t i = 0; i < size; ++i) {
SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i);
SArray* p = createQueryPlanImpl(pq);
taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p));
}
}
if (pQueryInfo->numOfTables > 1) { // it is a join query
// 1. separate the select clause according to table
taosArrayDestroy(upstream);
upstream = taosArrayInit(5, POINTER_BYTES);
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i];
uint64_t uid = pTableMetaInfo->pTableMeta->uid;
SArray* exprList = taosArrayInit(4, POINTER_BYTES);
if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
// dropAllExprInfo(exprList);
exit(-1);
}
// 2. create the query execution node
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
// 3. get the required table column list
SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn));
columnListCopy(tableColumnList, pQueryInfo->colList, uid);
// 4. add the projection query node
SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList);
columnListDestroy(tableColumnList);
// dropAllExprInfo(exprList);
taosArrayPush(upstream, &pNode);
}
// 3. add the join node here
SQueryTableInfo info = {0};
int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]);
SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables,
pQueryInfo->exprList[0]->pData, num, NULL);
// 4. add the aggregation or projection execution node
pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
} else { // only one table, normal query process
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
}
return upstream;
}
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
tfree(pQueryNode->pExtInfo);
tfree(pQueryNode->pSchema);
tfree(pQueryNode->info.name);
// dropAllExprInfo(pQueryNode->pExpr);
if (pQueryNode->pPrevNodes != NULL) {
int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes);
for(int32_t i = 0; i < size; ++i) {
SQueryPlanNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i);
doDestroyQueryNode(p);
}
taosArrayDestroy(pQueryNode->pPrevNodes);
}
tfree(pQueryNode);
}
static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
if (level > 0) {
sprintf(buf + totalLen, "%*c", level, ' ');
totalLen += level;
}
int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name);
int32_t len = len1 + totalLen;
switch(pQueryNode->info.type) {
case QNODE_TABLESCAN: {
SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid,
pInfo->window.skey, pInfo->window.ekey);
assert(len1 > 0);
len += len1;
for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i);
len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId);
assert(len1 > 0);
len += len1;
}
len1 = sprintf(buf + len, "\n");
assert(len1 > 0);
len += len1;
break;
}
case QNODE_PROJECT: {
len1 = sprintf(buf + len, "cols:");
assert(len1 > 0);
len += len1;
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ")");
len += len1;
// todo print filter info
len1 = sprintf(buf + len, " filters:(nil)\n");
len += len1;
break;
}
case QNODE_AGGREGATE: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_TIMEWINDOW: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ") ");
len += len1;
SInterval* pInterval = pQueryNode->pExtInfo;
// todo dynamic return the time precision
len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n",
pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding,
TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR);
len += len1;
break;
}
case QNODE_STATEWINDOW: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ") ");
len += len1;
SColumn* pCol = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId);
len += len1;
break;
}
case QNODE_SESSIONWINDOW: {
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ") ");
len += len1;
struct SSessionWindow* ps = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap);
len += len1;
break;
}
case QNODE_GROUPBY: {
len = printExprInfo(buf, pQueryNode, len);
SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, ") groupby_col: ");
len += len1;
for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) {
SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i);
len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId);
len += len1;
}
len += sprintf(buf + len, "\n");
break;
}
case QNODE_FILL: {
SFillEssInfo* pEssInfo = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "%d", pEssInfo->fillType);
len += len1;
if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) {
len1 = sprintf(buf + len, ", val:");
len += len1;
// todo get the correct fill data type
for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]);
len += len1;
if (i < pQueryNode->numOfExpr - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
}
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_LIMIT: {
SLimit* pVal = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset);
len += len1;
break;
}
case QNODE_DISTINCT:
case QNODE_TAGSCAN: {
len1 = sprintf(buf + len, "cols: ");
len += len1;
len = printExprInfo(buf, pQueryNode, len);
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_SORT: {
len1 = sprintf(buf + len, "cols:");
len += len1;
SArray* pSort = pQueryNode->pExtInfo;
for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) {
SOrder* p = taosArrayGet(pSort, i);
len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC");
len += len1;
}
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_JOIN: {
// print join condition
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
}
return len;
}
int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len) {
int32_t len1 = 0;
for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
SSqlExpr* pExpr = &pExprInfo->base;
len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId);
assert(len1 > 0);
len += len1;
if (i < pQueryNode->numOfExpr - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
return len;
}
int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen);
for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) {
SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i);
int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len);
len = len1;
}
return len;
}
int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) {
assert(pQueryNode);
*str = calloc(1, 4096);
int32_t len = sprintf(*str, "===== logic plan =====\n");
queryPlanToStringImpl(*str, pQueryNode, 0, len);
return TSDB_CODE_SUCCESS;
}
SQueryPlanNode* queryPlanFromString() {
return NULL;
}

11
source/libs/query/inc/queryInt.h
View File

@ -21,17 +21,6 @@ extern "C" {
#endif
#include "tlog.h"
extern int32_t qDebugFlag;
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
#ifdef __cplusplus
}

178
source/libs/query/src/querymsg.c
View File

@ -21,17 +21,6 @@ int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msg
int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0};
int32_t queryBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) {
if (NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
*msgLen = 0;
return TSDB_CODE_SUCCESS;
}
int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) {
if (NULL == input || NULL == msg || NULL == msgLen) {
return TSDB_CODE_TSC_INVALID_INPUT;
@ -81,8 +70,7 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms
strncpy(bMsg->db, bInput->db, sizeof(bMsg->db));
bMsg->db[sizeof(bMsg->db) - 1] = 0;
bMsg->vgroupVersion = bInput->vgroupVersion;
bMsg->dbGroupVersion = bInput->dbGroupVersion;
bMsg->vgVersion = bInput->vgVersion;
*msgLen = (int32_t)sizeof(*bMsg);
@ -90,58 +78,12 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms
}
int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) {
if (NULL == output || NULL == msg || msgSize <= 0) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
SVgroupListRspMsg *pRsp = (SVgroupListRspMsg *)msg;
pRsp->vgroupNum = htonl(pRsp->vgroupNum);
pRsp->vgroupVersion = htonl(pRsp->vgroupVersion);
if (pRsp->vgroupNum < 0) {
qError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
if (pRsp->vgroupVersion < 0) {
qError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) {
qError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
// keep SVgroupListInfo/SVgroupListRspMsg the same
*(SVgroupListInfo **)output = (SVgroupListInfo *)msg;
if (pRsp->vgroupNum == 0) {
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < pRsp->vgroupNum; ++i) {
pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId);
for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) {
pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port);
}
}
return TSDB_CODE_SUCCESS;
}
int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) {
if (NULL == output || NULL == msg || msgSize <= 0) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
SUseDbRspMsg *pRsp = (SUseDbRspMsg *)msg;
SUseDbRsp *pRsp = (SUseDbRsp *)msg;
SUseDbOutput *pOut = (SUseDbOutput *)output;
int32_t code = 0;
@ -150,104 +92,52 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) {
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
pRsp->vgroupVersion = htonl(pRsp->vgroupVersion);
pRsp->dbVgroupVersion = htonl(pRsp->dbVgroupVersion);
pRsp->vgVersion = htonl(pRsp->vgVersion);
pRsp->vgNum = htonl(pRsp->vgNum);
pRsp->vgroupNum = htonl(pRsp->vgroupNum);
pRsp->dbVgroupNum = htonl(pRsp->dbVgroupNum);
if (pRsp->vgroupNum < 0) {
qError("invalid vgroup number[%d]", pRsp->vgroupNum);
if (pRsp->vgNum < 0) {
qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum);
return TSDB_CODE_TSC_INVALID_VALUE;
}
if (pRsp->dbVgroupNum < 0) {
qError("invalid db vgroup number[%d]", pRsp->dbVgroupNum);
return TSDB_CODE_TSC_INVALID_VALUE;
}
int32_t expectSize = pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + pRsp->dbVgroupNum * sizeof(int32_t) + sizeof(*pRsp);
int32_t expectSize = pRsp->vgNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp);
if (msgSize != expectSize) {
qError("vgroup list msg size mis-match, msgSize:%d, expected:%d, vgroup number:%d, db vgroup number:%d", msgSize, expectSize, pRsp->vgroupNum, pRsp->dbVgroupNum);
qError("use db rsp size mis-match, msgSize:%d, expected:%d, vgnumber:%d", msgSize, expectSize, pRsp->vgNum);
return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE;
}
if (pRsp->vgroupVersion < 0) {
qInfo("no new vgroup list info");
if (pRsp->vgroupNum != 0) {
qError("invalid vgroup number[%d] for no new vgroup list case", pRsp->vgroupNum);
return TSDB_CODE_TSC_INVALID_VALUE;
}
} else {
int32_t s = sizeof(*pOut->vgroupList) + sizeof(pOut->vgroupList->vgroupInfo[0]) * pRsp->vgroupNum;
pOut->vgroupList = calloc(1, s);
if (NULL == pOut->vgroupList) {
qError("calloc size[%d] failed", s);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
pOut->vgroupList->vgroupNum = pRsp->vgroupNum;
pOut->vgroupList->vgroupVersion = pRsp->vgroupVersion;
for (int32_t i = 0; i < pRsp->vgroupNum; ++i) {
pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId);
for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) {
pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port);
}
memcpy(&pOut->vgroupList->vgroupInfo[i], &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]));
}
pOut->dbVgroup.vgVersion = pRsp->vgVersion;
pOut->dbVgroup.hashMethod = pRsp->hashMethod;
pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
if (NULL == pOut->dbVgroup.vgInfo) {
qError("hash init[%d] failed", pRsp->vgNum);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
int32_t *vgIdList = (int32_t *)((char *)pRsp->vgroupInfo + sizeof(pRsp->vgroupInfo[0]) * pRsp->vgroupNum);
for (int32_t i = 0; i < pRsp->vgNum; ++i) {
pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId);
pRsp->vgroupInfo[i].hashBegin = htonl(pRsp->vgroupInfo[i].hashBegin);
pRsp->vgroupInfo[i].hashEnd = htonl(pRsp->vgroupInfo[i].hashEnd);
for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) {
pRsp->vgroupInfo[i].epAddr[n].port = htons(pRsp->vgroupInfo[i].epAddr[n].port);
}
if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) {
qError("hash push failed");
goto _return;
}
}
memcpy(pOut->db, pRsp->db, sizeof(pOut->db));
if (pRsp->dbVgroupVersion < 0) {
qInfo("no new vgroup info for db[%s]", pRsp->db);
} else {
pOut->dbVgroup = calloc(1, sizeof(*pOut->dbVgroup));
if (NULL == pOut->dbVgroup) {
qError("calloc size[%d] failed", (int32_t)sizeof(*pOut->dbVgroup));
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _exit;
}
pOut->dbVgroup->vgId = taosArrayInit(pRsp->dbVgroupNum, sizeof(int32_t));
if (NULL == pOut->dbVgroup->vgId) {
qError("taosArrayInit size[%d] failed", pRsp->dbVgroupNum);
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _exit;
}
pOut->dbVgroup->vgroupVersion = pRsp->dbVgroupVersion;
pOut->dbVgroup->hashRange = htonl(pRsp->dbHashRange);
pOut->dbVgroup->hashType = htonl(pRsp->dbHashType);
if (pOut->dbVgroup->hashRange < 0) {
qError("invalid hashRange[%d] for db[%s]", pOut->dbVgroup->hashRange, pRsp->db);
code = TSDB_CODE_TSC_INVALID_INPUT;
goto _exit;
}
for (int32_t i = 0; i < pRsp->dbVgroupNum; ++i) {
*(vgIdList + i) = htonl(*(vgIdList + i));
taosArrayPush(pOut->dbVgroup->vgId, vgIdList + i) ;
}
}
return code;
_exit:
if (pOut->dbVgroup && pOut->dbVgroup->vgId) {
taosArrayDestroy(pOut->dbVgroup->vgId);
pOut->dbVgroup->vgId = NULL;
_return:
if (pOut) {
tfree(pOut->dbVgroup.vgInfo);
}
tfree(pOut->dbVgroup);
tfree(pOut->vgroupList);
return code;
}
@ -375,11 +265,9 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
void msgInit() {
queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg;
queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg;
queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg;
queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = queryProcessTableMetaRsp;
queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp;
queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp;
/*
@ -417,7 +305,7 @@ void msgInit() {
tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg;
tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg;
tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg;
tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg;
tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg;
tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg;
tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg;
@ -435,7 +323,7 @@ void msgInit() {
tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp;
tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp;
tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function.
tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function.
tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp;
tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp;

2
source/libs/scheduler/CMakeLists.txt
View File

@ -9,5 +9,5 @@ target_include_directories(
target_link_libraries(
scheduler
PRIVATE os util planner
PRIVATE os util planner common query
)

54
source/libs/scheduler/inc/schedulerInt.h
View File

@ -24,15 +24,57 @@ extern "C" {
#include "tarray.h"
#include "planner.h"
#include "scheduler.h"
#include "thash.h"
#define SCHEDULE_DEFAULT_JOB_NUMBER 1000
enum {
SCH_STATUS_NOT_START = 1,
SCH_STATUS_EXECUTING,
SCH_STATUS_SUCCEED,
SCH_STATUS_FAILED,
SCH_STATUS_CANCELLING,
SCH_STATUS_CANCELLED
};
typedef struct SSchedulerMgmt {
SHashObj *Jobs; // key: queryId, value: SQueryJob*
} SSchedulerMgmt;
typedef struct SQueryTask {
uint64_t taskId; // task id
char *pSubplan; // operator tree
int8_t status; // task status
SQueryProfileSummary summary; // task execution summary
} SQueryTask;
typedef struct SQueryLevel {
int8_t status;
int32_t taskNum;
SArray *subTasks; // Element is SQueryTask
SArray *subPlans; // Element is SSubplan
} SQueryLevel;
typedef struct SQueryJob {
uint64_t queryId;
int32_t levelNum;
int32_t levelIdx;
int8_t status;
SQueryProfileSummary summary;
SArray *levels; // Element is SQueryLevel, starting from 0.
SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0.
} SQueryJob;
#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0)
#define SCH_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0)
#define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0)
typedef struct SQuery {
SArray **pSubquery;
int32_t numOfLevels;
int32_t currentLevel;
} SQuery;
#ifdef __cplusplus
}
#endif
#endif /*_TD_SCHEDULER_INT_H_*/
#endif /*_TD_SCHEDULER_INT_H_*/

130
source/libs/scheduler/src/scheduler.c
View File

@ -13,4 +13,132 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "schedulerInt.h"
#include "schedulerInt.h"
#include "taosmsg.h"
#include "query.h"
SSchedulerMgmt schMgmt = {0};
int32_t schBuildAndSendRequest(void *pRpc, const SEpSet* pMgmtEps, __taos_async_fn_t fp) {
/*
SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT);
if (pRequest == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SRequestMsgBody body = {0};
buildConnectMsg(pRequest, &body);
int64_t transporterId = 0;
sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
tsem_wait(&pRequest->body.rspSem);
destroyConnectMsg(&body);
if (pRequest->code != TSDB_CODE_SUCCESS) {
const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno);
printf("failed to connect to server, reason: %s\n\n", errorMsg);
destroyRequest(pRequest);
taos_close(pTscObj);
pTscObj = NULL;
} else {
tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter);
destroyRequest(pRequest);
}
*/
}
int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) {
int32_t levelNum = (int32_t)taosArrayGetSize(dag->pSubplans);
if (levelNum <= 0) {
qError("invalid level num:%d", levelNum);
return TSDB_CODE_QRY_INVALID_INPUT;
}
job->levels = taosArrayInit(levelNum, sizeof(SQueryLevel));
if (NULL == job->levels) {
qError("taosArrayInit %d failed", levelNum);
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
job->levelNum = levelNum;
job->levelIdx = levelNum - 1;
job->status = SCH_STATUS_NOT_START;
job->subPlans = dag->pSubplans;
SQueryLevel level = {0};
SArray *levelPlans = NULL;
int32_t levelPlanNum = 0;
for (int32_t i = 0; i < levelNum; ++i) {
levelPlans = taosArrayGetP(dag->pSubplans, i);
if (NULL == levelPlans) {
qError("no level plans for level %d", i);
return TSDB_CODE_QRY_INVALID_INPUT;
}
levelPlanNum = (int32_t)taosArrayGetSize(levelPlans);
if (levelPlanNum <= 0) {
qError("invalid level plans number:%d, level:%d", levelPlanNum, i);
return TSDB_CODE_QRY_INVALID_INPUT;
}
for (int32_t n = 0; n < levelPlanNum; ++n) {
}
}
return TSDB_CODE_SUCCESS;
}
int32_t schedulerInit(SSchedulerCfg *cfg) {
schMgmt.Jobs = taosHashInit(SCHEDULE_DEFAULT_JOB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK);
if (NULL == schMgmt.Jobs) {
SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", SCHEDULE_DEFAULT_JOB_NUMBER);
}
return TSDB_CODE_SUCCESS;
}
int32_t scheduleQueryJob(SQueryDag* pDag, void** pJob) {
if (NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
SQueryJob *job = calloc(1, sizeof(SQueryJob));
if (NULL == job) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
schValidateAndBuildJob(pDag, job);
*(SQueryJob **)pJob = job;
}
int32_t scheduleFetchRows(void *pJob, void *data);
int32_t scheduleCancelJob(void *pJob);
void schedulerDestroy(void) {
if (schMgmt.Jobs) {
taosHashCleanup(schMgmt.Jobs); //TBD
schMgmt.Jobs = NULL;
}
}

56
source/libs/wal/inc/walInt.h
View File

@ -16,72 +16,70 @@
#ifndef _TD_WAL_INT_H_
#define _TD_WAL_INT_H_
#include "wal.h"
#include "compare.h"
#include "tchecksum.h"
#include "wal.h"
#ifdef __cplusplus
extern "C" {
#endif
//meta section begin
// meta section begin
typedef struct WalFileInfo {
int64_t firstVer;
int64_t lastVer;
int64_t createTs;
int64_t closeTs;
int64_t fileSize;
} WalFileInfo;
} SWalFileInfo;
typedef struct WalIdxEntry {
int64_t ver;
int64_t offset;
} WalIdxEntry;
} SWalIdxEntry;
static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) {
WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft;
WalFileInfo* pInfoRight = (WalFileInfo*)pRight;
SWalFileInfo* pInfoLeft = (SWalFileInfo*)pLeft;
SWalFileInfo* pInfoRight = (SWalFileInfo*)pRight;
return compareInt64Val(&pInfoLeft->firstVer, &pInfoRight->firstVer);
}
static inline int64_t walGetLastFileSize(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
return pInfo->fileSize;
}
static inline int64_t walGetLastFileFirstVer(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
return pInfo->firstVer;
}
static inline int64_t walGetCurFileFirstVer(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
return pInfo->firstVer;
}
static inline int64_t walGetCurFileLastVer(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
return pInfo->firstVer;
}
static inline int64_t walGetCurFileOffset(SWal* pWal) {
WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
return pInfo->fileSize;
}
static inline bool walCurFileClosed(SWal* pWal) {
return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur;
static inline bool walCurFileClosed(SWal* pWal) { return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; }
static inline SWalFileInfo* walGetCurFileInfo(SWal* pWal) {
return (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
}
static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) {
return (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur);
}
static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) {
static inline int walBuildLogName(SWal* pWal, int64_t fileFirstVer, char* buf) {
return sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer);
}
static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) {
static inline int walBuildIdxName(SWal* pWal, int64_t fileFirstVer, char* buf) {
return sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer);
}
@ -93,11 +91,11 @@ static inline int walValidBodyCksum(SWalHead* pHead) {
return taosCheckChecksum((uint8_t*)pHead->head.body, pHead->head.len, pHead->cksumBody);
}
static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) {
static inline int walValidCksum(SWalHead* pHead, void* body, int64_t bodyLen) {
return walValidHeadCksum(pHead) && walValidBodyCksum(pHead);
}
static inline uint32_t walCalcHeadCksum(SWalHead *pHead) {
static inline uint32_t walCalcHeadCksum(SWalHead* pHead) {
return taosCalcChecksum(0, (uint8_t*)&pHead->head, sizeof(SWalReadHead));
}
@ -106,7 +104,7 @@ static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) {
}
static inline int64_t walGetVerIdxOffset(SWal* pWal, int64_t ver) {
return (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry);
return (ver - walGetCurFileFirstVer(pWal)) * sizeof(SWalIdxEntry);
}
static inline void walResetVer(SWalVer* pVer) {
@ -126,16 +124,16 @@ int walCheckAndRepairMeta(SWal* pWal);
int walCheckAndRepairIdx(SWal* pWal);
char* walMetaSerialize(SWal* pWal);
int walMetaDeserialize(SWal* pWal, const char* bytes);
//meta section end
int walMetaDeserialize(SWal* pWal, const char* bytes);
// meta section end
//seek section
int walChangeFile(SWal *pWal, int64_t ver);
//seek section end
// seek section
int walChangeFile(SWal* pWal, int64_t ver);
// seek section end
int64_t walGetSeq();
int walSeekVer(SWal *pWal, int64_t ver);
int walRoll(SWal *pWal);
int walSeekVer(SWal* pWal, int64_t ver);
int walRoll(SWal* pWal);
#ifdef __cplusplus
}

129
source/libs/wal/src/walMeta.c
View File

@ -14,27 +14,21 @@
*/
#define _DEFAULT_SOURCE
#include "cJSON.h"
#include "os.h"
#include "taoserror.h"
#include "tref.h"
#include "tfile.h"
#include "cJSON.h"
#include "tref.h"
#include "walInt.h"
#include <libgen.h>
#include <regex.h>
int64_t inline walGetFirstVer(SWal *pWal) {
return pWal->vers.firstVer;
}
int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; }
int64_t inline walGetSnaphostVer(SWal *pWal) {
return pWal->vers.snapshotVer;
}
int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; }
int64_t inline walGetLastVer(SWal *pWal) {
return pWal->vers.lastVer;
}
int64_t inline walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; }
static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) {
return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer);
@ -44,31 +38,30 @@ int walCheckAndRepairMeta(SWal* pWal) {
// load log files, get first/snapshot/last version info
const char* logPattern = "^[0-9]+.log$";
const char* idxPattern = "^[0-9]+.idx$";
regex_t logRegPattern;
regex_t idxRegPattern;
SArray* pLogArray = taosArrayInit(8, sizeof(int64_t));
regex_t logRegPattern;
regex_t idxRegPattern;
SArray* pLogArray = taosArrayInit(8, sizeof(int64_t));
regcomp(&logRegPattern, logPattern, REG_EXTENDED);
regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
DIR *dir = opendir(pWal->path);
if(dir == NULL) {
DIR* dir = opendir(pWal->path);
if (dir == NULL) {
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1;
}
struct dirent* ent;
while((ent = readdir(dir)) != NULL) {
char *name = basename(ent->d_name);
int code = regexec(&logRegPattern, name, 0, NULL, 0);
if(code == 0) {
while ((ent = readdir(dir)) != NULL) {
char* name = basename(ent->d_name);
int code = regexec(&logRegPattern, name, 0, NULL, 0);
if (code == 0) {
int64_t firstVer;
sscanf(name, "%" PRId64 ".log", &firstVer);
taosArrayPush(pLogArray, &firstVer);
}
}
// load meta
// if not match, or meta missing
// rebuild meta
@ -85,15 +78,15 @@ int walRollFileInfo(SWal* pWal) {
int64_t ts = taosGetTimestampSec();
SArray* pArray = pWal->fileInfoSet;
if(taosArrayGetSize(pArray) != 0) {
WalFileInfo *pInfo = taosArrayGetLast(pArray);
if (taosArrayGetSize(pArray) != 0) {
SWalFileInfo* pInfo = taosArrayGetLast(pArray);
pInfo->lastVer = pWal->vers.lastVer;
pInfo->closeTs = ts;
}
//TODO: change to emplace back
WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo));
if(pNewInfo == NULL) {
// TODO: change to emplace back
SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo));
if (pNewInfo == NULL) {
return -1;
}
pNewInfo->firstVer = pWal->vers.lastVer + 1;
@ -109,13 +102,13 @@ int walRollFileInfo(SWal* pWal) {
char* walMetaSerialize(SWal* pWal) {
char buf[30];
ASSERT(pWal->fileInfoSet);
int sz = pWal->fileInfoSet->size;
int sz = pWal->fileInfoSet->size;
cJSON* pRoot = cJSON_CreateObject();
cJSON* pMeta = cJSON_CreateObject();
cJSON* pFiles = cJSON_CreateArray();
cJSON* pField;
if(pRoot == NULL || pMeta == NULL || pFiles == NULL) {
//TODO
if (pRoot == NULL || pMeta == NULL || pFiles == NULL) {
// TODO
return NULL;
}
cJSON_AddItemToObject(pRoot, "meta", pMeta);
@ -129,16 +122,16 @@ char* walMetaSerialize(SWal* pWal) {
cJSON_AddStringToObject(pMeta, "lastVer", buf);
cJSON_AddItemToObject(pRoot, "files", pFiles);
WalFileInfo* pData = pWal->fileInfoSet->pData;
for(int i = 0; i < sz; i++) {
WalFileInfo* pInfo = &pData[i];
SWalFileInfo* pData = pWal->fileInfoSet->pData;
for (int i = 0; i < sz; i++) {
SWalFileInfo* pInfo = &pData[i];
cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject());
if(pField == NULL) {
if (pField == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
//cjson only support int32_t or double
//string are used to prohibit the loss of precision
// cjson only support int32_t or double
// string are used to prohibit the loss of precision
sprintf(buf, "%" PRId64, pInfo->firstVer);
cJSON_AddStringToObject(pField, "firstVer", buf);
sprintf(buf, "%" PRId64, pInfo->lastVer);
@ -171,13 +164,13 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
pFiles = cJSON_GetObjectItem(pRoot, "files");
int sz = cJSON_GetArraySize(pFiles);
//deserialize
// deserialize
SArray* pArray = pWal->fileInfoSet;
taosArrayEnsureCap(pArray, sz);
WalFileInfo *pData = pArray->pData;
for(int i = 0; i < sz; i++) {
cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
WalFileInfo* pInfo = &pData[i];
SWalFileInfo* pData = pArray->pData;
for (int i = 0; i < sz; i++) {
cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i);
SWalFileInfo* pInfo = &pData[i];
pField = cJSON_GetObjectItem(pInfoJson, "firstVer");
pInfo->firstVer = atoll(cJSON_GetStringValue(pField));
pField = cJSON_GetObjectItem(pInfoJson, "lastVer");
@ -196,24 +189,24 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) {
}
static int walFindCurMetaVer(SWal* pWal) {
const char * pattern = "^meta-ver[0-9]+$";
regex_t walMetaRegexPattern;
const char* pattern = "^meta-ver[0-9]+$";
regex_t walMetaRegexPattern;
regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED);
DIR *dir = opendir(pWal->path);
if(dir == NULL) {
DIR* dir = opendir(pWal->path);
if (dir == NULL) {
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1;
}
struct dirent* ent;
//find existing meta-ver[x].json
// find existing meta-ver[x].json
int metaVer = -1;
while((ent = readdir(dir)) != NULL) {
char *name = basename(ent->d_name);
int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0);
if(code == 0) {
while ((ent = readdir(dir)) != NULL) {
char* name = basename(ent->d_name);
int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0);
if (code == 0) {
sscanf(name, "meta-ver%d", &metaVer);
break;
}
@ -224,23 +217,23 @@ static int walFindCurMetaVer(SWal* pWal) {
}
int walSaveMeta(SWal* pWal) {
int metaVer = walFindCurMetaVer(pWal);
int metaVer = walFindCurMetaVer(pWal);
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer+1, fnameStr);
walBuildMetaName(pWal, metaVer + 1, fnameStr);
int metaTfd = tfOpenCreateWrite(fnameStr);
if(metaTfd < 0) {
if (metaTfd < 0) {
return -1;
}
char* serialized = walMetaSerialize(pWal);
int len = strlen(serialized);
if(len != tfWrite(metaTfd, serialized, len)) {
//TODO:clean file
int len = strlen(serialized);
if (len != tfWrite(metaTfd, serialized, len)) {
// TODO:clean file
return -1;
}
tfClose(metaTfd);
//delete old file
if(metaVer > -1) {
// delete old file
if (metaVer > -1) {
walBuildMetaName(pWal, metaVer, fnameStr);
remove(fnameStr);
}
@ -250,30 +243,30 @@ int walSaveMeta(SWal* pWal) {
int walLoadMeta(SWal* pWal) {
ASSERT(pWal->fileInfoSet->size == 0);
//find existing meta file
// find existing meta file
int metaVer = walFindCurMetaVer(pWal);
if(metaVer == -1) {
if (metaVer == -1) {
return 0;
}
char fnameStr[WAL_FILE_LEN];
walBuildMetaName(pWal, metaVer, fnameStr);
//read metafile
// read metafile
struct stat statbuf;
stat(fnameStr, &statbuf);
int size = statbuf.st_size;
int size = statbuf.st_size;
char* buf = malloc(size + 5);
if(buf == NULL) {
if (buf == NULL) {
return -1;
}
memset(buf, 0, size+5);
memset(buf, 0, size + 5);
int tfd = tfOpenRead(fnameStr);
if(tfRead(tfd, buf, size) != size) {
if (tfRead(tfd, buf, size) != size) {
free(buf);
return -1;
}
//load into fileInfoSet
// load into fileInfoSet
int code = walMetaDeserialize(pWal, buf);
if(code != 0) {
if (code != 0) {
free(buf);
return -1;
}

48
source/libs/wal/src/walMgmt.c
View File

@ -14,11 +14,11 @@
*/
#define _DEFAULT_SOURCE
#include "compare.h"
#include "os.h"
#include "taoserror.h"
#include "tref.h"
#include "tfile.h"
#include "compare.h"
#include "tref.h"
#include "walInt.h"
typedef struct {
@ -34,16 +34,14 @@ static int32_t walCreateThread();
static void walStopThread();
static void walFreeObj(void *pWal);
int64_t walGetSeq() {
return (int64_t)atomic_load_32(&tsWal.seq);
}
int64_t walGetSeq() { return (int64_t)atomic_load_32(&tsWal.seq); }
int32_t walInit() {
int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1);
if(old == 1) return 0;
if (old == 1) return 0;
int code = tfInit();
if(code != 0) {
if (code != 0) {
wError("failed to init tfile since %s", tstrerror(code));
atomic_store_8(&tsWal.inited, 0);
return code;
@ -63,7 +61,7 @@ int32_t walInit() {
void walCleanUp() {
int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0);
if(old == 0) {
if (old == 0) {
return;
}
walStopThread();
@ -78,52 +76,52 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
return NULL;
}
//set config
// set config
memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg));
pWal->fsyncSeq = pCfg->fsyncPeriod / 1000;
if(pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1;
tstrncpy(pWal->path, path, sizeof(pWal->path));
if(taosMkDir(pWal->path) != 0) {
if (taosMkDir(pWal->path) != 0) {
wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return NULL;
}
//open meta
// open meta
walResetVer(&pWal->vers);
pWal->writeLogTfd = -1;
pWal->writeIdxTfd = -1;
pWal->writeCur = -1;
pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo));
if(pWal->fileInfoSet == NULL) {
pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo));
if (pWal->fileInfoSet == NULL) {
wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno));
free(pWal);
return NULL;
}
//init status
// init status
pWal->totSize = 0;
pWal->lastRollSeq = -1;
//init write buffer
// init write buffer
memset(&pWal->writeHead, 0, sizeof(SWalHead));
pWal->writeHead.head.headVer = WAL_HEAD_VER;
if(pthread_mutex_init(&pWal->mutex, NULL) < 0) {
if (pthread_mutex_init(&pWal->mutex, NULL) < 0) {
taosArrayDestroy(pWal->fileInfoSet);
free(pWal);
return NULL;
}
pWal->refId = taosAddRef(tsWal.refSetId, pWal);
if(pWal->refId < 0) {
if (pWal->refId < 0) {
pthread_mutex_destroy(&pWal->mutex);
taosArrayDestroy(pWal->fileInfoSet);
free(pWal);
return NULL;
}
if(walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) {
if (walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) {
taosRemoveRef(tsWal.refSetId, pWal->refId);
pthread_mutex_destroy(&pWal->mutex);
taosArrayDestroy(pWal->fileInfoSet);
@ -131,11 +129,11 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) {
return NULL;
}
if(walCheckAndRepairIdx(pWal) < 0) {
if (walCheckAndRepairIdx(pWal) < 0) {
}
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod);
wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level,
pWal->cfg.fsyncPeriod);
return pWal;
}
@ -203,10 +201,12 @@ static void walFsyncAll() {
SWal *pWal = taosIterateRef(tsWal.refSetId, 0);
while (pWal) {
if (walNeedFsync(pWal)) {
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq));
wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq,
atomic_load_32(&tsWal.seq));
int32_t code = tfFsync(pWal->writeLogTfd);
if (code != 0) {
wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code));
wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(code));
}
}
pWal = taosIterateRef(tsWal.refSetId, pWal->refId);

109
source/libs/wal/src/walRead.c
View File

@ -13,12 +13,12 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "walInt.h"
#include "tfile.h"
#include "walInt.h"
SWalReadHandle* walOpenReadHandle(SWal* pWal) {
SWalReadHandle *walOpenReadHandle(SWal *pWal) {
SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle));
if(pRead == NULL) {
if (pRead == NULL) {
return NULL;
}
pRead->pWal = pWal;
@ -29,7 +29,7 @@ SWalReadHandle* walOpenReadHandle(SWal* pWal) {
pRead->capacity = 0;
pRead->status = 0;
pRead->pHead = malloc(sizeof(SWalHead));
if(pRead->pHead == NULL) {
if (pRead->pHead == NULL) {
free(pRead);
return NULL;
}
@ -43,27 +43,25 @@ void walCloseReadHandle(SWalReadHandle *pRead) {
free(pRead);
}
int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) {
return 0;
}
int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; }
static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) {
int code = 0;
int64_t idxTfd = pRead->readIdxTfd;
int64_t logTfd = pRead->readLogTfd;
//seek position
int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry);
// seek position
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
code = tfLseek(idxTfd, offset, SEEK_SET);
if(code < 0) {
if (code < 0) {
return -1;
}
WalIdxEntry entry;
if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
return -1;
}
//TODO:deserialize
// TODO:deserialize
ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET);
if (code < 0) {
@ -80,13 +78,13 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
int64_t logTfd = tfOpenRead(fnameStr);
if(logTfd < 0) {
if (logTfd < 0) {
return -1;
}
walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr);
int64_t idxTfd = tfOpenRead(fnameStr);
if(idxTfd < 0) {
if (idxTfd < 0) {
return -1;
}
@ -96,76 +94,75 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
}
static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
int code;
int code;
SWal *pWal = pRead->pWal;
if(ver == pRead->curVersion) {
if (ver == pRead->curVersion) {
return 0;
}
if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
return -1;
}
if(ver < pWal->vers.snapshotVer) {
if (ver < pWal->vers.snapshotVer) {
}
WalFileInfo tmpInfo;
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
//bsearch in fileSet
WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
// bsearch in fileSet
SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL);
if(pRead->curFileFirstVer != pRet->firstVer) {
if (pRead->curFileFirstVer != pRet->firstVer) {
code = walReadChangeFile(pRead, pRet->firstVer);
if(code < 0) {
//TODO: set error flag
if (code < 0) {
// TODO: set error flag
return -1;
}
}
code = walReadSeekFilePos(pRead, pRet->firstVer, ver);
if(code < 0) {
if (code < 0) {
return -1;
}
pRead->curVersion = ver;
return 0;
}
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
int code;
//TODO: check wal life
if(pRead->curVersion != ver) {
code = walReadSeekVer(pRead, ver);
if(code != 0) {
// TODO: check wal life
if (pRead->curVersion != ver) {
code = walReadSeekVer(pRead, ver);
if (code != 0) {
return -1;
}
}
if(!tfValid(pRead->readLogTfd)) return -1;
if (!tfValid(pRead->readLogTfd)) return -1;
code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead));
if(code != sizeof(SWalHead)) {
if (code != sizeof(SWalHead)) {
return -1;
}
code = walValidHeadCksum(pRead->pHead);
if(code != 0) {
if (code != 0) {
return -1;
}
if(pRead->capacity < pRead->pHead->head.len) {
void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len);
if(ptr == NULL) {
if (pRead->capacity < pRead->pHead->head.len) {
void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len);
if (ptr == NULL) {
return -1;
}
pRead->pHead = ptr;
pRead->capacity = pRead->pHead->head.len;
}
if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) {
return -1;
}
/*code = walValidBodyCksum(pRead->pHead);*/
ASSERT(pRead->pHead->head.version == ver);
if(code != 0) {
code = walValidBodyCksum(pRead->pHead);
if (code != 0) {
return -1;
}
pRead->curVersion++;
@ -176,40 +173,40 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) {
int code;
code = walSeekVer(pWal, ver);
if(code != 0) {
if (code != 0) {
return code;
}
if(*ppHead == NULL) {
void* ptr = realloc(*ppHead, sizeof(SWalHead));
if(ptr == NULL) {
if (*ppHead == NULL) {
void *ptr = realloc(*ppHead, sizeof(SWalHead));
if (ptr == NULL) {
return -1;
}
*ppHead = ptr;
}
if(tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
return -1;
}
//TODO: endian compatibility processing after read
if(walValidHeadCksum(*ppHead) != 0) {
// TODO: endian compatibility processing after read
if (walValidHeadCksum(*ppHead) != 0) {
return -1;
}
void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
if(ptr == NULL) {
void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len);
if (ptr == NULL) {
free(*ppHead);
*ppHead = NULL;
return -1;
}
if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) {
return -1;
}
//TODO: endian compatibility processing after read
if(walValidBodyCksum(*ppHead) != 0) {
// TODO: endian compatibility processing after read
if (walValidBodyCksum(*ppHead) != 0) {
return -1;
}
return 0;
}
/*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/
/*return 0;*/
/*return 0;*/
/*}*/

69
source/libs/wal/src/walSeek.c
View File

@ -16,8 +16,8 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tref.h"
#include "tfile.h"
#include "tref.h"
#include "walInt.h"
static int walSeekFilePos(SWal* pWal, int64_t ver) {
@ -25,17 +25,17 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
int64_t idxTfd = pWal->writeIdxTfd;
int64_t logTfd = pWal->writeLogTfd;
//seek position
// seek position
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
if(code != 0) {
if (code != 0) {
return -1;
}
WalIdxEntry entry;
//TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry));
if(code != 0) {
SWalIdxEntry entry;
// TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry));
if (code != 0) {
return -1;
}
ASSERT(entry.ver == ver);
@ -46,51 +46,51 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
return code;
}
int walChangeFileToLast(SWal *pWal) {
int64_t idxTfd, logTfd;
WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
int walChangeFileToLast(SWal* pWal) {
int64_t idxTfd, logTfd;
SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet);
ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer;
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenReadWrite(fnameStr);
if(idxTfd < 0) {
if (idxTfd < 0) {
return -1;
}
walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenReadWrite(fnameStr);
if(logTfd < 0) {
if (logTfd < 0) {
return -1;
}
//switch file
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
return 0;
}
int walChangeFile(SWal *pWal, int64_t ver) {
int code = 0;
int walChangeFile(SWal* pWal, int64_t ver) {
int code = 0;
int64_t idxTfd, logTfd;
char fnameStr[WAL_FILE_LEN];
char fnameStr[WAL_FILE_LEN];
code = tfClose(pWal->writeLogTfd);
if(code != 0) {
//TODO
if (code != 0) {
// TODO
return -1;
}
code = tfClose(pWal->writeIdxTfd);
if(code != 0) {
//TODO
if (code != 0) {
// TODO
return -1;
}
WalFileInfo tmpInfo;
SWalFileInfo tmpInfo;
tmpInfo.firstVer = ver;
//bsearch in fileSet
WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
// bsearch in fileSet
SWalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
ASSERT(pRet != NULL);
int64_t fileFirstVer = pRet->firstVer;
//closed
if(taosArrayGetLast(pWal->fileInfoSet) != pRet) {
// closed
if (taosArrayGetLast(pWal->fileInfoSet) != pRet) {
walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenRead(fnameStr);
walBuildLogName(pWal, fileFirstVer, fnameStr);
@ -107,27 +107,26 @@ int walChangeFile(SWal *pWal, int64_t ver) {
return fileFirstVer;
}
int walSeekVer(SWal *pWal, int64_t ver) {
int walSeekVer(SWal* pWal, int64_t ver) {
int code;
if(ver == pWal->vers.lastVer) {
if (ver == pWal->vers.lastVer) {
return 0;
}
if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) {
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
return -1;
}
if(ver < pWal->vers.snapshotVer) {
if (ver < pWal->vers.snapshotVer) {
}
if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) {
code = walChangeFile(pWal, ver);
if(code != 0) {
if (code != 0) {
return -1;
}
}
code = walSeekFilePos(pWal, ver);
if(code != 0) {
if (code != 0) {
return -1;
}
return 0;
}

225
source/libs/wal/src/walWrite.c
View File

@ -24,7 +24,7 @@
int32_t walCommit(SWal *pWal, int64_t ver) {
ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer);
ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer);
if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) {
if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) {
return -1;
}
pWal->vers.commitVer = ver;
@ -32,57 +32,57 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
}
int32_t walRollback(SWal *pWal, int64_t ver) {
int code;
int code;
char fnameStr[WAL_FILE_LEN];
if(ver == pWal->vers.lastVer) {
if (ver == pWal->vers.lastVer) {
return 0;
}
if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
return -1;
}
pthread_mutex_lock(&pWal->mutex);
//find correct file
if(ver < walGetLastFileFirstVer(pWal)) {
//close current files
// find correct file
if (ver < walGetLastFileFirstVer(pWal)) {
// close current files
tfClose(pWal->writeIdxTfd);
tfClose(pWal->writeLogTfd);
//open old files
// open old files
code = walChangeFile(pWal, ver);
if(code != 0) {
if (code != 0) {
return -1;
}
//delete files
// delete files
int fileSetSize = taosArrayGetSize(pWal->fileInfoSet);
for(int i = pWal->writeCur; i < fileSetSize; i++) {
walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
for (int i = pWal->writeCur; i < fileSetSize; i++) {
walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
remove(fnameStr);
walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr);
remove(fnameStr);
}
//pop from fileInfoSet
// pop from fileInfoSet
taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1);
}
walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t idxTfd = tfOpenReadWrite(fnameStr);
//TODO:change to deserialize function
if(idxTfd < 0) {
// TODO:change to deserialize function
if (idxTfd < 0) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET);
if(code < 0) {
if (code < 0) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
//read idx file and get log file pos
//TODO:change to deserialize function
WalIdxEntry entry;
if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) {
// read idx file and get log file pos
// TODO:change to deserialize function
SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
@ -90,56 +90,56 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr);
int64_t logTfd = tfOpenReadWrite(fnameStr);
if(logTfd < 0) {
//TODO
if (logTfd < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
code = tfLseek(logTfd, entry.offset, SEEK_SET);
if(code < 0) {
//TODO
if (code < 0) {
// TODO
pthread_mutex_unlock(&pWal->mutex);
return -1;
}
//validate offset
// validate offset
SWalHead head;
ASSERT(tfValid(logTfd));
int size = tfRead(logTfd, &head, sizeof(SWalHead));
if(size != sizeof(SWalHead)) {
if (size != sizeof(SWalHead)) {
return -1;
}
code = walValidHeadCksum(&head);
ASSERT(code == 0);
if(code != 0) {
if (code != 0) {
return -1;
}
if(head.head.version != ver) {
//TODO
if (head.head.version != ver) {
// TODO
return -1;
}
//truncate old files
// truncate old files
code = tfFtruncate(logTfd, entry.offset);
if(code < 0) {
if (code < 0) {
return -1;
}
code = tfFtruncate(idxTfd, idxOff);
if(code < 0) {
if (code < 0) {
return -1;
}
pWal->vers.lastVer = ver - 1;
((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1;
((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset;
//unlock
// unlock
pthread_mutex_unlock(&pWal->mutex);
return 0;
}
int32_t walBeginSnapshot(SWal* pWal, int64_t ver) {
int32_t walBeginSnapshot(SWal *pWal, int64_t ver) {
pWal->vers.verInSnapshotting = ver;
//check file rolling
if(pWal->cfg.retentionPeriod == 0) {
// check file rolling
if (pWal->cfg.retentionPeriod == 0) {
walRoll(pWal);
}
@ -148,54 +148,54 @@ int32_t walBeginSnapshot(SWal* pWal, int64_t ver) {
int32_t walEndSnapshot(SWal *pWal) {
int64_t ver = pWal->vers.verInSnapshotting;
if(ver == -1) return -1;
if (ver == -1) return -1;
pWal->vers.snapshotVer = ver;
int ts = taosGetTimestampSec();
int deleteCnt = 0;
int64_t newTotSize = pWal->totSize;
WalFileInfo tmp;
int deleteCnt = 0;
int64_t newTotSize = pWal->totSize;
SWalFileInfo tmp;
tmp.firstVer = ver;
//find files safe to delete
WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
if(ver >= pInfo->lastVer) {
// find files safe to delete
SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE);
if (ver >= pInfo->lastVer) {
pInfo++;
}
//iterate files, until the searched result
for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
if(pWal->totSize > pWal->cfg.retentionSize ||
iter->closeTs + pWal->cfg.retentionPeriod > ts) {
//delete according to file size or close time
// iterate files, until the searched result
for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) {
if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) {
// delete according to file size or close time
deleteCnt++;
newTotSize -= iter->fileSize;
}
}
char fnameStr[WAL_FILE_LEN];
//remove file
for(int i = 0; i < deleteCnt; i++) {
WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i);
walBuildLogName(pWal, pInfo->firstVer, fnameStr);
// remove file
for (int i = 0; i < deleteCnt; i++) {
SWalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i);
walBuildLogName(pWal, pInfo->firstVer, fnameStr);
remove(fnameStr);
walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
walBuildIdxName(pWal, pInfo->firstVer, fnameStr);
remove(fnameStr);
}
//make new array, remove files
taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt);
if(taosArrayGetSize(pWal->fileInfoSet) == 0) {
// make new array, remove files
taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt);
if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
pWal->writeCur = -1;
pWal->vers.firstVer = -1;
} else {
pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
}
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
;
pWal->totSize = newTotSize;
pWal->vers.verInSnapshotting = -1;
//save snapshot ver, commit ver
// save snapshot ver, commit ver
int code = walSaveMeta(pWal);
if(code != 0) {
if (code != 0) {
return -1;
}
@ -204,41 +204,41 @@ int32_t walEndSnapshot(SWal *pWal) {
int walRoll(SWal *pWal) {
int code = 0;
if(pWal->writeIdxTfd != -1) {
if (pWal->writeIdxTfd != -1) {
code = tfClose(pWal->writeIdxTfd);
if(code != 0) {
if (code != 0) {
return -1;
}
}
if(pWal->writeLogTfd != -1) {
if (pWal->writeLogTfd != -1) {
code = tfClose(pWal->writeLogTfd);
if(code != 0) {
if (code != 0) {
return -1;
}
}
int64_t idxTfd, logTfd;
//create new file
// create new file
int64_t newFileFirstVersion = pWal->vers.lastVer + 1;
char fnameStr[WAL_FILE_LEN];
char fnameStr[WAL_FILE_LEN];
walBuildIdxName(pWal, newFileFirstVersion, fnameStr);
idxTfd = tfOpenCreateWriteAppend(fnameStr);
if(idxTfd < 0) {
if (idxTfd < 0) {
ASSERT(0);
return -1;
}
walBuildLogName(pWal, newFileFirstVersion, fnameStr);
logTfd = tfOpenCreateWriteAppend(fnameStr);
if(logTfd < 0) {
if (logTfd < 0) {
ASSERT(0);
return -1;
}
code = walRollFileInfo(pWal);
if(code != 0) {
if (code != 0) {
ASSERT(0);
return -1;
}
//switch file
// switch file
pWal->writeIdxTfd = idxTfd;
pWal->writeLogTfd = logTfd;
pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;
@ -248,10 +248,10 @@ int walRoll(SWal *pWal) {
}
static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) {
WalIdxEntry entry = { .ver = ver, .offset = offset };
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry));
if(size != sizeof(WalIdxEntry)) {
//TODO truncate
SWalIdxEntry entry = {.ver = ver, .offset = offset};
int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry));
if (size != sizeof(SWalIdxEntry)) {
// TODO truncate
return -1;
}
return 0;
@ -265,21 +265,21 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0;
if (index == pWal->vers.lastVer + 1) {
if(taosArrayGetSize(pWal->fileInfoSet) == 0) {
if (taosArrayGetSize(pWal->fileInfoSet) == 0) {
pWal->vers.firstVer = index;
code = walRoll(pWal);
ASSERT(code == 0);
} else {
int64_t passed = walGetSeq() - pWal->lastRollSeq;
if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) {
walRoll(pWal);
} else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
} else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) {
walRoll(pWal);
}
}
} else {
//reject skip log or rewrite log
//must truncate explicitly first
// reject skip log or rewrite log
// must truncate explicitly first
return -1;
}
/*if (!tfValid(pWal->writeLogTfd)) return -1;*/
@ -294,28 +294,30 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
//ftruncate
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno));
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
}
if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) {
//ftruncate
if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) {
// ftruncate
code = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno));
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
}
code = walWriteIndex(pWal, index, offset);
if(code != 0) {
//TODO
if (code != 0) {
// TODO
return -1;
}
//set status
// set status
pWal->vers.lastVer = index;
pWal->totSize += sizeof(SWalHead) + bodyLen;
walGetCurFileInfo(pWal)->lastVer = index;
walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen;
pthread_mutex_unlock(&pWal->mutex);
return code;
@ -323,33 +325,34 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i
void walFsync(SWal *pWal, bool forceFsync) {
if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) {
wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal));
if (tfFsync(pWal->writeLogTfd) < 0) {
wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno));
wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal),
strerror(errno));
}
}
}
/*static int walValidateOffset(SWal* pWal, int64_t ver) {*/
/*int code = 0;*/
/*SWalHead *pHead = NULL;*/
/*code = (int)walRead(pWal, &pHead, ver);*/
/*if(pHead->head.version != ver) {*/
/*return -1;*/
/*}*/
/*return 0;*/
/*int code = 0;*/
/*SWalHead *pHead = NULL;*/
/*code = (int)walRead(pWal, &pHead, ver);*/
/*if(pHead->head.version != ver) {*/
/*return -1;*/
/*}*/
/*return 0;*/
/*}*/
/*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/
/*int code = walSeekVer(pWal, ver);*/
/*if(code != 0) {*/
/*return -1;*/
/*}*/
/*code = walValidateOffset(pWal, ver);*/
/*if(code != 0) {*/
/*return -1;*/
/*}*/
/*return 0;*/
/*int code = walSeekVer(pWal, ver);*/
/*if(code != 0) {*/
/*return -1;*/
/*}*/
/*code = walValidateOffset(pWal, ver);*/
/*if(code != 0) {*/
/*return -1;*/
/*}*/
/*return 0;*/
/*}*/

212
source/libs/wal/test/walMetaTest.cpp
View File

@ -6,111 +6,105 @@
#include "walInt.h"
const char* ranStr = "tvapq02tcp";
const int ranStrLen = strlen(ranStr);
const int ranStrLen = strlen(ranStr);
class WalCleanEnv : public ::testing::Test {
protected:
static void SetUpTestCase() {
int code = walInit();
ASSERT(code == 0);
}
protected:
static void SetUpTestCase() {
int code = walInit();
ASSERT(code == 0);
}
static void TearDownTestCase() {
walCleanUp();
}
static void TearDownTestCase() { walCleanUp(); }
void SetUp() override {
taosRemoveDir(pathName);
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg));
pCfg->rollPeriod = -1;
pCfg->segSize = -1;
pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg);
free(pCfg);
ASSERT(pWal != NULL);
}
void SetUp() override {
taosRemoveDir(pathName);
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg));
pCfg->rollPeriod = -1;
pCfg->segSize = -1;
pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg);
free(pCfg);
ASSERT(pWal != NULL);
}
void TearDown() override {
walClose(pWal);
pWal = NULL;
}
void TearDown() override {
walClose(pWal);
pWal = NULL;
}
SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test";
SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test";
};
class WalCleanDeleteEnv : public ::testing::Test {
protected:
static void SetUpTestCase() {
int code = walInit();
ASSERT(code == 0);
}
protected:
static void SetUpTestCase() {
int code = walInit();
ASSERT(code == 0);
}
static void TearDownTestCase() {
walCleanUp();
}
static void TearDownTestCase() { walCleanUp(); }
void SetUp() override {
taosRemoveDir(pathName);
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg));
pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg);
free(pCfg);
ASSERT(pWal != NULL);
}
void SetUp() override {
taosRemoveDir(pathName);
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg));
pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg);
free(pCfg);
ASSERT(pWal != NULL);
}
void TearDown() override {
walClose(pWal);
pWal = NULL;
}
void TearDown() override {
walClose(pWal);
pWal = NULL;
}
SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test";
SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test";
};
class WalKeepEnv : public ::testing::Test {
protected:
static void SetUpTestCase() {
int code = walInit();
ASSERT(code == 0);
}
protected:
static void SetUpTestCase() {
int code = walInit();
ASSERT(code == 0);
}
static void TearDownTestCase() {
walCleanUp();
}
static void TearDownTestCase() { walCleanUp(); }
void walResetEnv() {
TearDown();
taosRemoveDir(pathName);
SetUp();
}
void walResetEnv() {
TearDown();
taosRemoveDir(pathName);
SetUp();
}
void SetUp() override {
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg));
pCfg->rollPeriod = -1;
pCfg->segSize = -1;
pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg);
free(pCfg);
ASSERT(pWal != NULL);
}
void SetUp() override {
SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg));
memset(pCfg, 0, sizeof(SWalCfg));
pCfg->rollPeriod = -1;
pCfg->segSize = -1;
pCfg->retentionPeriod = 0;
pCfg->retentionSize = 0;
pCfg->level = TAOS_WAL_FSYNC;
pWal = walOpen(pathName, pCfg);
free(pCfg);
ASSERT(pWal != NULL);
}
void TearDown() override {
walClose(pWal);
pWal = NULL;
}
void TearDown() override {
walClose(pWal);
pWal = NULL;
}
SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test";
SWal* pWal = NULL;
const char* pathName = "/tmp/wal_test";
};
TEST_F(WalCleanEnv, createNew) {
@ -139,7 +133,7 @@ TEST_F(WalCleanEnv, serialize) {
ASSERT(code == 0);
code = walRollFileInfo(pWal);
ASSERT(code == 0);
char*ss = walMetaSerialize(pWal);
char* ss = walMetaSerialize(pWal);
printf("%s\n", ss);
free(ss);
code = walSaveMeta(pWal);
@ -162,11 +156,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
walResetEnv();
int code;
for(int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen);
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen);
code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1);
ASSERT_EQ(pWal->vers.lastVer, i);
}
@ -182,7 +176,7 @@ TEST_F(WalKeepEnv, readOldMeta) {
int len = strlen(oldss);
ASSERT_EQ(len, strlen(newss));
for(int i = 0; i < len; i++) {
for (int i = 0; i < len; i++) {
EXPECT_EQ(oldss[i], newss[i]);
}
free(oldss);
@ -191,11 +185,11 @@ TEST_F(WalKeepEnv, readOldMeta) {
TEST_F(WalCleanEnv, write) {
int code;
for(int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen);
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen);
code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, -1);
ASSERT_EQ(pWal->vers.lastVer, i);
}
@ -205,8 +199,8 @@ TEST_F(WalCleanEnv, write) {
TEST_F(WalCleanEnv, rollback) {
int code;
for(int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen);
for (int i = 0; i < 10; i++) {
code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
}
@ -223,7 +217,7 @@ TEST_F(WalCleanEnv, rollback) {
TEST_F(WalCleanDeleteEnv, roll) {
int code;
int i;
for(i = 0; i < 100; i++) {
for (i = 0; i < 100; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
ASSERT_EQ(pWal->vers.lastVer, i);
@ -231,16 +225,16 @@ TEST_F(WalCleanDeleteEnv, roll) {
ASSERT_EQ(pWal->vers.commitVer, i);
}
walBeginSnapshot(pWal, i-1);
ASSERT_EQ(pWal->vers.verInSnapshotting, i-1);
walBeginSnapshot(pWal, i - 1);
ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1);
walEndSnapshot(pWal);
ASSERT_EQ(pWal->vers.snapshotVer, i-1);
ASSERT_EQ(pWal->vers.snapshotVer, i - 1);
ASSERT_EQ(pWal->vers.verInSnapshotting, -1);
code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen);
ASSERT_NE(code, 0);
for(; i < 200; i++) {
for (; i < 200; i++) {
code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen);
ASSERT_EQ(code, 0);
code = walCommit(pWal, i);
@ -255,36 +249,36 @@ TEST_F(WalCleanDeleteEnv, roll) {
TEST_F(WalKeepEnv, readHandleRead) {
walResetEnv();
int code;
int code;
SWalReadHandle* pRead = walOpenReadHandle(pWal);
ASSERT(pRead != NULL);
int i ;
for(i = 0; i < 100; i++) {
int i;
for (i = 0; i < 100; i++) {
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, i);
int len = strlen(newStr);
code = walWrite(pWal, i, 0, newStr, len);
ASSERT_EQ(code, 0);
}
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
int ver = rand() % 100;
code = walReadWithHandle(pRead, ver);
ASSERT_EQ(code, 0);
//printf("rrbody: \n");
//for(int i = 0; i < pRead->pHead->head.len; i++) {
//printf("%d ", pRead->pHead->head.body[i]);
// printf("rrbody: \n");
// for(int i = 0; i < pRead->pHead->head.len; i++) {
// printf("%d ", pRead->pHead->head.body[i]);
//}
//printf("\n");
// printf("\n");
ASSERT_EQ(pRead->pHead->head.version, ver);
ASSERT_EQ(pRead->curVersion, ver+1);
ASSERT_EQ(pRead->curVersion, ver + 1);
char newStr[100];
sprintf(newStr, "%s-%d", ranStr, ver);
int len = strlen(newStr);
ASSERT_EQ(pRead->pHead->head.len, len);
for(int j = 0; j < len; j++) {
for (int j = 0; j < len; j++) {
EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
}
}

10
source/os/src/osSysinfo.c
View File

@ -1120,11 +1120,11 @@ SysNameInfo taosGetSysNameInfo() {
struct utsname uts;
if (!uname(&uts)) {
info.sysname = strdup(uts.sysname);
info.nodename = strdup(uts.nodename);
info.release = strdup(uts.release);
info.version = strdup(uts.version);
info.machine = strdup(uts.machine);
tstrncpy(info.sysname, uts.sysname, sizeof(info.sysname));
tstrncpy(info.nodename, uts.nodename, sizeof(info.nodename));
tstrncpy(info.release, uts.release, sizeof(info.release));
tstrncpy(info.version, uts.version, sizeof(info.version));
tstrncpy(info.machine, uts.machine, sizeof(info.machine));
}
return info;

2
source/util/src/tarray.c
View File

@ -17,7 +17,7 @@
#include "tarray.h"
#include "talgo.h"
void* taosArrayInit(size_t size, size_t elemSize) {
SArray* taosArrayInit(size_t size, size_t elemSize) {
assert(elemSize > 0);
if (size < TARRAY_MIN_SIZE) {

1
source/util/src/terror.c
View File

@ -322,6 +322,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit ha
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION, "One valid time range condition expected")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_INPUT, "invalid input")
// grant

6
src/client/src/tscAsync.c
View File

@ -183,7 +183,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo
pSql->fp = fp;
if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH;
}
if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
@ -265,7 +265,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) {
}
return;
} else if (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) {
} else if (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) {
// in case of show command, return no data
(*pSql->fetchFp)(param, pSql, 0);
} else {
@ -273,7 +273,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) {
}
} else { // current query is not completed, continue retrieve from node
if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH;
}
SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd);

Some files were not shown because too many files have changed in this diff Show More