diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..fc9a51af65 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile + +# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT="bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} + +# [Optional] Uncomment this section to install additional packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9b752d091d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp +{ + "name": "C++", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 + // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon + "args": { "VARIANT": "ubuntu-21.04" } + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "austin.code-gnu-global", + "visualstudioexptteam.vscodeintel" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "gcc -v", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/include/client/taos.h b/include/client/taos.h index 19d191b84e..7357478555 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -39,7 +39,7 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes #define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes #define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_BINARY 8 // string, alias for varchar #define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes #define TSDB_DATA_TYPE_NCHAR 10 // unicode string #define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte @@ -47,10 +47,10 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_UINT 13 // 4 bytes #define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes #define TSDB_DATA_TYPE_VARCHAR 15 // string -#define TSDB_DATA_TYPE_JSON 16 // json -#define TSDB_DATA_TYPE_DECIMAL 17 // decimal -#define TSDB_DATA_TYPE_BLOB 18 // binary string -#define TSDB_DATA_TYPE_LONGBLOB 19 // long binary string +#define TSDB_DATA_TYPE_VARBINARY 16 // binary +#define TSDB_DATA_TYPE_JSON 17 // json +#define TSDB_DATA_TYPE_DECIMAL 18 // decimal +#define TSDB_DATA_TYPE_BLOB 19 // binary typedef enum { TSDB_OPTION_LOCALE, @@ -61,10 +61,27 @@ typedef enum { TSDB_MAX_OPTIONS } TSDB_OPTION; +typedef enum { + TSDB_SML_UNKNOWN_PROTOCOL = 0, + TSDB_SML_LINE_PROTOCOL = 1, + TSDB_SML_TELNET_PROTOCOL = 2, + TSDB_SML_JSON_PROTOCOL = 3, +} TSDB_SML_PROTOCOL_TYPE; + +typedef enum { + TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0, + TSDB_SML_TIMESTAMP_HOURS, + TSDB_SML_TIMESTAMP_MINUTES, + TSDB_SML_TIMESTAMP_SECONDS, + TSDB_SML_TIMESTAMP_MILLI_SECONDS, + TSDB_SML_TIMESTAMP_MICRO_SECONDS, + TSDB_SML_TIMESTAMP_NANO_SECONDS, +} TSDB_SML_TIMESTAMP_TYPE; + typedef struct taosField { char name[65]; - uint8_t type; - int16_t bytes; + int8_t type; + int32_t bytes; } TAOS_FIELD; #ifdef _TD_GO_DLL_ @@ -136,37 +153,36 @@ DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); + DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); -DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); -DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); +DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); -// TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); -// TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); - -// TODO: the return value should be `const` DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); -DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); -DLL_EXPORT int taos_errno(TAOS_RES *tres); +DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); +DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); -DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); +typedef void (*__taos_sub_fn_t)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); +DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, __taos_sub_fn_t fp, void *param, int interval); DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); @@ -175,8 +191,7 @@ DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp) DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); - -DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines); +DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); #ifdef __cplusplus } diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 1fe930ba7a..f8652bec9a 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -50,10 +50,9 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_ACK, "mq-ack" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_RESET, "mq-reset" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" ) // message from client to mnode -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_ACCT, "alter-acct" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_ACCT, "drop-acct" ) @@ -215,17 +214,15 @@ typedef enum _mgmt_table { extern char *taosMsg[]; typedef struct SBuildTableMetaInput { - int32_t vgId; - char *tableFullName; + int32_t vgId; + char *tableFullName; } SBuildTableMetaInput; typedef struct SBuildUseDBInput { char db[TSDB_TABLE_FNAME_LEN]; - int32_t vgroupVersion; - int32_t dbGroupVersion; + int32_t vgVersion; } SBuildUseDBInput; - #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -235,7 +232,7 @@ typedef struct { } SEpAddrMsg; typedef struct { - char* fqdn; + char *fqdn; uint16_t port; } SEpAddr1; @@ -262,10 +259,10 @@ typedef struct SSubmitBlk { // Submit message for this TSDB typedef struct SSubmitMsg { - SMsgHead header; - int32_t length; - int32_t numOfBlocks; - char blocks[]; + SMsgHead header; + int32_t length; + int32_t numOfBlocks; + char blocks[]; } SSubmitMsg; typedef struct { @@ -344,7 +341,7 @@ typedef struct { typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; char db[TSDB_FULL_DB_NAME_LEN]; - int16_t type; /* operation type */ + int16_t type; /* operation type */ int16_t numOfCols; /* number of schema */ int32_t tagValLen; SSchema schema[]; @@ -353,17 +350,17 @@ typedef struct { } SAlterTableMsg; typedef struct { - SMsgHead head; - int64_t uid; - int32_t tid; - int16_t tversion; - int16_t colId; - int8_t type; - int16_t bytes; - int32_t tagValLen; - int16_t numOfTags; - int32_t schemaLen; - char data[]; + SMsgHead head; + int64_t uid; + int32_t tid; + int16_t tversion; + int16_t colId; + int8_t type; + int16_t bytes; + int32_t tagValLen; + int16_t numOfTags; + int32_t schemaLen; + char data[]; } SUpdateTableTagValMsg; typedef struct { @@ -407,8 +404,10 @@ typedef struct { } SDropUserMsg, SDropAcctMsg; typedef struct { + int8_t type; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; + int8_t superUser; // denote if it is a super user or not int32_t reserve[8]; } SCreateUserMsg, SAlterUserMsg; @@ -424,7 +423,7 @@ typedef struct { int32_t contLen; int32_t vgId; uint64_t uid; - char tableFname[TSDB_TABLE_FNAME_LEN]; + char tableFname[TSDB_TABLE_FNAME_LEN]; } SDropSTableMsg; typedef struct SColIndex { @@ -437,7 +436,7 @@ typedef struct SColIndex { typedef struct SColumnFilterInfo { int16_t lowerRelOptr; int16_t upperRelOptr; - int16_t filterstr; // denote if current column is char(binary/nchar) + int16_t filterstr; // denote if current column is char(binary/nchar) union { struct { @@ -456,9 +455,9 @@ typedef struct SColumnFilterInfo { } SColumnFilterInfo; typedef struct SColumnFilterList { - int16_t numOfFilters; - union{ - int64_t placeholder; + int16_t numOfFilters; + union { + int64_t placeholder; SColumnFilterInfo *filterInfo; }; } SColumnFilterList; @@ -467,10 +466,10 @@ typedef struct SColumnFilterList { * But for data in vnode side, we need all the following information. */ typedef struct SColumnInfo { - int16_t colId; - int16_t type; - int16_t bytes; - SColumnFilterList flist; + int16_t colId; + int16_t type; + int16_t bytes; + SColumnFilterList flist; } SColumnInfo; typedef struct STableIdInfo { @@ -484,14 +483,14 @@ typedef struct STimeWindow { } STimeWindow; typedef struct { - int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed - int32_t tsLen; // total length of ts comp block - int32_t tsNumOfBlocks; // ts comp block numbers - int32_t tsOrder; // ts comp block order + int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed + int32_t tsLen; // total length of ts comp block + int32_t tsNumOfBlocks; // ts comp block numbers + int32_t tsOrder; // ts comp block order } STsBufInfo; typedef struct SInterval { - int32_t tz; // query client timezone + int32_t tz; // query client timezone char intervalUnit; char slidingUnit; char offsetUnit; @@ -501,51 +500,51 @@ typedef struct SInterval { } SInterval; typedef struct { - SMsgHead head; - char version[TSDB_VERSION_LEN]; + SMsgHead head; + char version[TSDB_VERSION_LEN]; - bool stableQuery; // super table query or not - bool topBotQuery; // TODO used bitwise flag - bool interpQuery; // interp query or not - bool groupbyColumn; // denote if this is a groupby normal column query - bool hasTagResults; // if there are tag values in final result or not - bool timeWindowInterpo;// if the time window start/end required interpolation - bool queryBlockDist; // if query data block distribution - bool stabledev; // super table stddev query - bool tsCompQuery; // is tscomp query - bool simpleAgg; - bool pointInterpQuery; // point interpolation query - bool needReverseScan; // need reverse scan - bool stateWindow; // state window flag + bool stableQuery; // super table query or not + bool topBotQuery; // TODO used bitwise flag + bool interpQuery; // interp query or not + bool groupbyColumn; // denote if this is a groupby normal column query + bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo; // if the time window start/end required interpolation + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query + bool tsCompQuery; // is tscomp query + bool simpleAgg; + bool pointInterpQuery; // point interpolation query + bool needReverseScan; // need reverse scan + bool stateWindow; // state window flag STimeWindow window; int32_t numOfTables; int16_t order; int16_t orderColId; - int16_t numOfCols; // the number of columns will be load from vnode + int16_t numOfCols; // the number of columns will be load from vnode SInterval interval; -// SSessionWindow sw; // session window - int16_t tagCondLen; // tag length in current query - int16_t colCondLen; // column length in current query - int16_t numOfGroupCols; // num of group by columns - int16_t orderByIdx; - int16_t orderType; // used in group by xx order by xxx - int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. - int16_t prjOrder; // global order in super table projection query. - int64_t limit; - int64_t offset; - int32_t queryType; // denote another query process - int16_t numOfOutput; // final output columns numbers - int16_t fillType; // interpolate type - int64_t fillVal; // default value array list - int32_t secondStageOutput; - STsBufInfo tsBuf; // tsBuf info - int32_t numOfTags; // number of tags columns involved - int32_t sqlstrLen; // sql query string - int32_t prevResultLen; // previous result length + // SSessionWindow sw; // session window + int16_t tagCondLen; // tag length in current query + int16_t colCondLen; // column length in current query + int16_t numOfGroupCols; // num of group by columns + int16_t orderByIdx; + int16_t orderType; // used in group by xx order by xxx + int64_t vgroupLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. + int16_t prjOrder; // global order in super table projection query. + int64_t limit; + int64_t offset; + int32_t queryType; // denote another query process + int16_t numOfOutput; // final output columns numbers + int16_t fillType; // interpolate type + int64_t fillVal; // default value array list + int32_t secondStageOutput; + STsBufInfo tsBuf; // tsBuf info + int32_t numOfTags; // number of tags columns involved + int32_t sqlstrLen; // sql query string + int32_t prevResultLen; // previous result length int32_t numOfOperator; - int32_t tableScanOperator;// table scan operator. -1 means no scan operator - int32_t udfNum; // number of udf function + int32_t tableScanOperator; // table scan operator. -1 means no scan operator + int32_t udfNum; // number of udf function int32_t udfContentOffset; int32_t udfContentLen; SColumnInfo tableCols[]; @@ -571,19 +570,19 @@ typedef struct { } SRetrieveTableMsg; typedef struct SRetrieveTableRsp { - int32_t numOfRows; - int64_t offset; // updated offset value for multi-vnode projection query int64_t useconds; int8_t completed; // all results are returned to client int8_t precision; int8_t compressed; - int8_t reserved; int32_t compLen; + + int32_t numOfRows; char data[]; } SRetrieveTableRsp; typedef struct { char db[TSDB_FULL_DB_NAME_LEN]; + int32_t numOfVgroups; int32_t cacheBlockSize; // MB int32_t totalBlocks; int32_t daysPerFile; @@ -626,9 +625,7 @@ typedef struct { typedef struct { char db[TSDB_TABLE_FNAME_LEN]; - int8_t ignoreNotExists; - int32_t vgroupVersion; - int32_t dbGroupVersion; + int32_t vgVersion; int32_t reserve[8]; } SUseDbMsg; @@ -790,8 +787,8 @@ typedef struct { } SStbInfoMsg; typedef struct { - SMsgHead msgHead; - char tableFname[TSDB_TABLE_FNAME_LEN]; + SMsgHead msgHead; + char tableFname[TSDB_TABLE_FNAME_LEN]; } STableInfoMsg; typedef struct { @@ -808,6 +805,9 @@ typedef struct SSTableVgroupMsg { typedef struct SVgroupInfo { int32_t vgId; + uint32_t hashBegin; + uint32_t hashEnd; + int8_t inUse; int8_t numOfEps; SEpAddrMsg epAddr[TSDB_MAX_REPLICA]; } SVgroupInfo; @@ -830,19 +830,19 @@ typedef struct { } SVgroupsMsg, SVgroupsInfo; typedef struct { - char tbFname[TSDB_TABLE_FNAME_LEN]; // table id - char stbFname[TSDB_TABLE_FNAME_LEN]; - int32_t numOfTags; - int32_t numOfColumns; - int8_t precision; - int8_t tableType; - int8_t update; - int32_t sversion; - int32_t tversion; - uint64_t tuid; - uint64_t suid; - int32_t vgId; - SSchema pSchema[]; + char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name + char stbFname[TSDB_TABLE_FNAME_LEN]; + int32_t numOfTags; + int32_t numOfColumns; + int8_t precision; + int8_t tableType; + int8_t update; + int32_t sversion; + int32_t tversion; + uint64_t tuid; + uint64_t suid; + int32_t vgId; + SSchema pSchema[]; } STableMetaMsg; typedef struct SMultiTableMeta { @@ -863,18 +863,12 @@ typedef struct { } STagData; typedef struct { - int32_t vgroupNum; - int32_t vgroupVersion; - char db[TSDB_TABLE_FNAME_LEN]; - int32_t dbVgroupVersion; - int32_t dbVgroupNum; - int32_t dbHashRange; - int32_t dbHashType; + char db[TSDB_FULL_DB_NAME_LEN]; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; SVgroupInfo vgroupInfo[]; -//int32_t vgIdList[]; -} SUseDbRspMsg; - - +} SUseDbRsp; /* * sql: show tables like '%a_%' @@ -889,7 +883,7 @@ typedef struct { } SShowMsg; typedef struct { - char db[TSDB_FULL_DB_NAME_LEN]; + char db[TSDB_FULL_DB_NAME_LEN]; int32_t numOfVgroup; int32_t vgid[]; } SCompactMsg; @@ -1010,45 +1004,37 @@ typedef struct { } SAuthMsg, SAuthRsp; typedef struct { - int8_t finished; - int8_t reserved1[7]; - char name[TSDB_STEP_NAME_LEN]; - char desc[TSDB_STEP_DESC_LEN]; + int8_t finished; + int8_t reserved1[7]; + char name[TSDB_STEP_NAME_LEN]; + char desc[TSDB_STEP_DESC_LEN]; } SStartupMsg; // mq related typedef struct { - } SMqConnectReq; typedef struct { - } SMqConnectRsp; typedef struct { - } SMqDisconnectReq; typedef struct { - } SMqDisconnectRsp; typedef struct { - } SMqAckReq; typedef struct { - } SMqAckRsp; typedef struct { - } SMqResetReq; typedef struct { - } SMqResetRsp; -//mq related end +// mq related end typedef struct { /* data */ @@ -1102,7 +1088,6 @@ typedef struct { /* data */ } SUpdateTagValRsp; - #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index d357ca1f47..1fb10ae15b 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -62,7 +62,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_DNODE, "cfg-dnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CFG_MNODE, "cfg-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SHOW, "show" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE, "retrieve" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_MNODE, "retrieve" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_QUERY, "kill-query" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_STREAM, "kill-stream" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_KILL_CONNECTION, "kill-connection" ) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 074678f248..7993a8f1ab 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -16,9 +16,9 @@ #ifndef _TD_TQ_H_ #define _TD_TQ_H_ +#include "mallocator.h" #include "os.h" #include "tutil.h" -#include "mallocator.h" #ifdef __cplusplus extern "C" { @@ -62,10 +62,10 @@ typedef struct TmqDisconnectRsp { int8_t status; } TmqDisconnectRsp; -typedef struct TmqConsumeReq { +typedef struct STqConsumeReq { TmqMsgHead head; TmqAcks acks; -} TmqConsumeReq; +} STqConsumeReq; typedef struct TmqMsgContent { int64_t topicId; @@ -73,11 +73,11 @@ typedef struct TmqMsgContent { char msg[]; } TmqMsgContent; -typedef struct TmqConsumeRsp { +typedef struct STqConsumeRsp { TmqMsgHead head; int64_t bodySize; TmqMsgContent msgs[]; -} TmqConsumeRsp; +} STqConsumeRsp; typedef struct TmqSubscribeReq { TmqMsgHead head; @@ -97,128 +97,125 @@ typedef struct TmqHeartbeatReq { typedef struct TmqHeartbeatRsp { } TmqHeartbeatRsp; -typedef struct TqTopicVhandle { +typedef struct STqTopicVhandle { int64_t topicId; // executor for filter void* filterExec; // callback for mnode // trigger when vnode list associated topic change void* (*mCallback)(void*, void*); -} TqTopicVhandle; - +} STqTopicVhandle; #define TQ_BUFFER_SIZE 8 -typedef struct TqBufferItem { +typedef struct STqBufferItem { int64_t offset; // executors are identical but not concurrent // so there must be a copy in each item void* executor; int64_t size; void* content; -} TqBufferItem; +} STqBufferItem; -typedef struct TqBufferHandle { +typedef struct STqBufferHandle { // char* topic; //c style, end with '\0' // int64_t cgId; // void* ahandle; - int64_t nextConsumeOffset; - int64_t topicId; - int32_t head; - int32_t tail; - TqBufferItem buffer[TQ_BUFFER_SIZE]; -} TqBufferHandle; + int64_t nextConsumeOffset; + int64_t floatingCursor; + int64_t topicId; + int32_t head; + int32_t tail; + STqBufferItem buffer[TQ_BUFFER_SIZE]; +} STqBufferHandle; -typedef struct TqListHandle { - TqBufferHandle bufHandle; - struct TqListHandle* next; -} TqListHandle; +typedef struct STqListHandle { + STqBufferHandle bufHandle; + struct STqListHandle* next; +} STqListHandle; -typedef struct TqGroupHandle { - int64_t cId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - TqListHandle* head; -} TqGroupHandle; +typedef struct STqGroupHandle { + int64_t cId; + int64_t cgId; + void* ahandle; + int32_t topicNum; + STqListHandle* head; +} STqGroupHandle; -typedef struct TqQueryExec { - void* src; - TqBufferItem* dest; - void* executor; -} TqQueryExec; +typedef struct STqQueryExec { + void* src; + STqBufferItem* dest; + void* executor; +} STqQueryExec; -typedef struct TqQueryMsg { - TqQueryExec* exec; - struct TqQueryMsg* next; -} TqQueryMsg; +typedef struct STqQueryMsg { + STqQueryExec* exec; + struct STqQueryMsg* next; +} STqQueryMsg; -typedef struct TqLogReader { +typedef struct STqLogReader { void* logHandle; int32_t (*logRead)(void* logHandle, void** data, int64_t ver); int64_t (*logGetFirstVer)(void* logHandle); int64_t (*logGetSnapshotVer)(void* logHandle); int64_t (*logGetLastVer)(void* logHandle); -} TqLogReader; +} STqLogReader; typedef struct STqCfg { // TODO } STqCfg; -typedef struct TqMemRef { - SMemAllocatorFactory *pAlloctorFactory; - SMemAllocator *pAllocator; -} TqMemRef; +typedef struct STqMemRef { + SMemAllocatorFactory* pAlloctorFactory; + SMemAllocator* pAllocator; +} STqMemRef; -typedef struct TqSerializedHead { +typedef struct STqSerializedHead { int16_t ver; int16_t action; int32_t checksum; int64_t ssize; char content[]; -} TqSerializedHead; +} STqSerializedHead; -typedef int (*TqSerializeFun)(const void* pObj, TqSerializedHead** ppHead); -typedef const void* (*TqDeserializeFun)(const TqSerializedHead* pHead, void** ppObj); -typedef void (*TqDeleteFun)(void*); +typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); +typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj); +typedef void (*FTqDelete)(void*); #define TQ_BUCKET_MASK 0xFF #define TQ_BUCKET_SIZE 256 #define TQ_PAGE_SIZE 4096 -//key + offset + size +// key + offset + size #define TQ_IDX_SIZE 24 -//4096 / 24 +// 4096 / 24 #define TQ_MAX_IDX_ONE_PAGE 170 -//24 * 170 +// 24 * 170 #define TQ_IDX_PAGE_BODY_SIZE 4080 -//4096 - 4080 +// 4096 - 4080 #define TQ_IDX_PAGE_HEAD_SIZE 16 -#define TQ_ACTION_CONST 0 -#define TQ_ACTION_INUSE 1 +#define TQ_ACTION_CONST 0 +#define TQ_ACTION_INUSE 1 #define TQ_ACTION_INUSE_CONT 2 -#define TQ_ACTION_INTXN 3 +#define TQ_ACTION_INTXN 3 -#define TQ_SVER 0 +#define TQ_SVER 0 -//TODO: inplace mode is not implemented -#define TQ_UPDATE_INPLACE 0 -#define TQ_UPDATE_APPEND 1 +// TODO: inplace mode is not implemented +#define TQ_UPDATE_INPLACE 0 +#define TQ_UPDATE_APPEND 1 #define TQ_DUP_INTXN_REWRITE 0 -#define TQ_DUP_INTXN_REJECT 2 +#define TQ_DUP_INTXN_REJECT 2 -static inline bool TqUpdateAppend(int32_t tqConfigFlag) { - return tqConfigFlag & TQ_UPDATE_APPEND; -} +static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } -static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { - return tqConfigFlag & TQ_DUP_INTXN_REJECT; -} +static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; -#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE + +#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE typedef struct TqMetaHandle { int64_t key; @@ -229,62 +226,63 @@ typedef struct TqMetaHandle { } STqMetaHandle; typedef struct TqMetaList { - STqMetaHandle handle; + STqMetaHandle handle; struct TqMetaList* next; - //struct TqMetaList* inTxnPrev; - //struct TqMetaList* inTxnNext; + // struct TqMetaList* inTxnPrev; + // struct TqMetaList* inTxnNext; struct TqMetaList* unpersistPrev; struct TqMetaList* unpersistNext; } STqMetaList; typedef struct TqMetaStore { - STqMetaList* bucket[TQ_BUCKET_SIZE]; - //a table head - STqMetaList* unpersistHead; - //TODO:temporaral use, to be replaced by unified tfile - int fileFd; - //TODO:temporaral use, to be replaced by unified tfile - int idxFd; - char* dirPath; - int32_t tqConfigFlag; - TqSerializeFun pSerializer; - TqDeserializeFun pDeserializer; - TqDeleteFun pDeleter; + STqMetaList* bucket[TQ_BUCKET_SIZE]; + // a table head + STqMetaList* unpersistHead; + // TODO:temporaral use, to be replaced by unified tfile + int fileFd; + // TODO:temporaral use, to be replaced by unified tfile + int idxFd; + char* dirPath; + int32_t tqConfigFlag; + FTqSerialize pSerializer; + FTqDeserialize pDeserializer; + FTqDelete pDeleter; } STqMetaStore; typedef struct STQ { // the collection of group handle // the handle of kvstore - char* path; - STqCfg* tqConfig; - TqLogReader* tqLogReader; - TqMemRef tqMemRef; + char* path; + STqCfg* tqConfig; + STqLogReader* tqLogReader; + STqMemRef tqMemRef; STqMetaStore* tqMeta; } STQ; // open in each vnode -STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac); +STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac); void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); +int tqSetCursor(STQ*, void* msg); -int tqConsume(STQ*, TmqConsumeReq*); +int tqConsume(STQ*, STqConsumeReq*); -TqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); +STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); -TqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqMoveOffsetToNext(TqGroupHandle*); -int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); -int tqRegisterContext(TqGroupHandle*, void* ahandle); -int tqLaunchQuery(TqGroupHandle*); -int tqSendLaunchQuery(TqGroupHandle*); +STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqMoveOffsetToNext(STqGroupHandle*); +int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); +int tqRegisterContext(STqGroupHandle*, void* ahandle); +int tqLaunchQuery(STqGroupHandle*); +int tqSendLaunchQuery(STqGroupHandle*); -int tqSerializeGroupHandle(const TqGroupHandle* gHandle, TqSerializedHead** ppHead); +int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead); -const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle** gHandle); +const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle); #ifdef __cplusplus } diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6bbc4f9109..ee626865fb 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -32,8 +32,7 @@ extern "C" { struct SCatalog; typedef struct SCatalogReq { - char dbName[TSDB_DB_NAME_LEN]; - SArray *pTableName; // table full name + SArray *pTableName; // element is SNAME SArray *pUdf; // udf name bool qNodeRequired; // valid qnode } SCatalogReq; @@ -54,54 +53,86 @@ typedef struct SCatalogCfg { int32_t catalogInit(SCatalogCfg *cfg); /** - * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. - * There is ONLY one SCatalog object for one process space, and this function returns a singleton. - * @param clusterId - * @return + * Get a cluster's catalog handle for all later operations. + * @param clusterId (input, end with \0) + * @param catalogHandle (output, NO need to free it) + * @return error code */ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle); - - -int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version); - -/** - * get cluster vgroup list. - * @pVgroupList - hash of vgroup list, key:vgId, value:SVgroupInfo - * @return - */ -int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash); -int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup); - - - int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version); -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo); +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo); int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); - +/** + * Get a table's meta data. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param pTableMeta(output, table meta data, NEED to free it by calller) + * @return error code + */ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); + +/** + * Force renew a table's local cached meta data. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @return error code + */ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName); + +/** + * Force renew a table's local cached meta data and get the new one. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param pTableMeta(output, table meta data, NEED to free it by calller) + * @return error code + */ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta); /** - * get table's vgroup list. - * @param clusterId - * @pVgroupList - array of SVgroupInfo - * @return + * Get a table's actual vgroup, for stable it's all possible vgroup list. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param pVgroupList (output, vgroup info list, element is SVgroupInfo, NEED to simply free the array by caller) + * @return error code */ -int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList); +int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList); + +/** + * Get a table's vgroup from its name's hash value. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pDBName (input, full db name) + * @param pTableName (input, table name, NOT including db name) + * @param vgInfo (output, vgroup info) + * @return error code + */ +int32_t catalogGetTableHashVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo); /** - * Get the required meta data from mnode. - * Note that this is a synchronized API and is also thread-safety. - * @param pCatalog - * @param pMgmtEps - * @param pMetaReq - * @param pMetaData - * @return + * Get all meta data required in pReq. + * @param pCatalog (input, got with catalogGetHandle) + * @param pRpc (input, rpc object) + * @param pMgmtEps (input, mnode EPs) + * @param pReq (input, reqest info) + * @param pRsp (output, response data) + * @return error code */ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp); @@ -112,7 +143,6 @@ int32_t catalogGetQnodeList(struct SCatalog* pCatalog, const SEpSet* pMgmtEps, S /** * Destroy catalog and relase all resources - * @param pCatalog */ void catalogDestroy(void); diff --git a/include/libs/index/index.h b/include/libs/index/index.h index 1b74928568..0885ce151e 100644 --- a/include/libs/index/index.h +++ b/include/libs/index/index.h @@ -28,6 +28,15 @@ typedef struct SIndexOpts SIndexOpts; typedef struct SIndexMultiTermQuery SIndexMultiTermQuery; typedef struct SArray SIndexMultiTerm; +typedef enum { + ADD_VALUE, // add index colume value + DEL_VALUE, // delete index column value + UPDATE_VALUE, // update index column value + ADD_INDEX, // add index on specify column + DROP_INDEX, // drop existed index + DROP_SATBLE // drop stable +} SIndexColumnType; + typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType; typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType; /* diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 2f152c3e2b..49adaecfdd 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -132,13 +132,15 @@ struct SInsertStmtInfo; bool qIsInsertSql(const char* pStr, size_t length); typedef struct SParseContext { + const char* pAcctId; + const char* pDbname; + void *pRpc; + const char* pClusterId; + const SEpSet* pEpSet; + int64_t id; // query id, generated by uuid generator + int8_t schemaAttached; // denote if submit block is built with table schema or not const char* pSql; // sql string size_t sqlLen; // length of the sql string - int64_t id; // operator id, generated by uuid generator - const char* pDbname; - const SEpSet* pEpSet; - int8_t schemaAttached; // denote if submit block is built with table schema or not - char* pMsg; // extended error message if exists to help avoid the problem in sql statement. int32_t msgLen; // max length of the msg } SParseContext; @@ -151,7 +153,7 @@ typedef struct SParseContext { * @param msg extended error message if exists. * @return error code */ -int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen); +int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); typedef enum { PAYLOAD_TYPE_KV = 0, diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 8f217a0deb..a7d418d45e 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -20,52 +20,92 @@ extern "C" { #endif +#include "taosmsg.h" + #define QUERY_TYPE_MERGE 1 #define QUERY_TYPE_PARTIAL 2 #define QUERY_TYPE_SCAN 3 enum OPERATOR_TYPE_E { - OP_TableScan = 1, - OP_DataBlocksOptScan = 2, - OP_TableSeqScan = 3, - OP_TagScan = 4, - OP_TableBlockInfoScan= 5, - OP_Aggregate = 6, - OP_Project = 7, - OP_Groupby = 8, - OP_Limit = 9, - OP_SLimit = 10, - OP_TimeWindow = 11, - OP_SessionWindow = 12, - OP_StateWindow = 22, - OP_Fill = 13, - OP_MultiTableAggregate = 14, - OP_MultiTableTimeInterval = 15, -// OP_DummyInput = 16, //TODO remove it after fully refactor. -// OP_MultiwayMergeSort = 17, // multi-way data merge into one input stream. -// OP_GlobalAggregate = 18, // global merge for the multi-way data sources. - OP_Filter = 19, - OP_Distinct = 20, - OP_Join = 21, - OP_AllTimeWindow = 23, - OP_AllMultiTableTimeInterval = 24, - OP_Order = 25, - OP_Exchange = 26, + OP_Unknown, +#define INCLUDE_AS_ENUM +#include "plannerOp.h" +#undef INCLUDE_AS_ENUM + OP_TotalNum }; struct SEpSet; -struct SQueryPlanNode; -struct SPhyNode; struct SQueryStmtInfo; +typedef SSchema SSlotSchema; + +typedef struct SDataBlockSchema { + SSlotSchema *pSchema; + int32_t numOfCols; // number of columns +} SDataBlockSchema; + +typedef struct SQueryNodeBasicInfo { + int32_t type; // operator type + const char *name; // operator name +} SQueryNodeBasicInfo; + +typedef struct SPhyNode { + SQueryNodeBasicInfo info; + SArray *pTargets; // target list to be computed or scanned at this node + SArray *pConditions; // implicitly-ANDed qual conditions + SDataBlockSchema targetSchema; + // children plan to generated result for current node to process + // in case of join, multiple plan nodes exist. + SArray *pChildren; + struct SPhyNode *pParent; +} SPhyNode; + +typedef struct SScanPhyNode { + SPhyNode node; + uint64_t uid; // unique id of the table + int8_t tableType; +} SScanPhyNode; + +typedef SScanPhyNode SSystemTableScanPhyNode; +typedef SScanPhyNode STagScanPhyNode; + +typedef struct STableScanPhyNode { + SScanPhyNode scan; + uint8_t scanFlag; // denotes reversed scan of data or not + STimeWindow window; + SArray *pTagsConditions; // implicitly-ANDed tag qual conditions +} STableScanPhyNode; + +typedef STableScanPhyNode STableSeqScanPhyNode; + +typedef struct SProjectPhyNode { + SPhyNode node; +} SProjectPhyNode; + +typedef struct SExchangePhyNode { + SPhyNode node; + uint64_t srcTemplateId; // template id of datasource suplans + SArray *pSourceEpSet; // SEpSet, scheduler fill by calling qSetSuplanExecutionNode +} SExchangePhyNode; + +typedef struct SSubplanId { + uint64_t queryId; + uint64_t templateId; + uint64_t subplanId; +} SSubplanId; + typedef struct SSubplan { - int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN - SArray *pDatasource; // the datasource subplan,from which to fetch the result - struct SPhyNode *pNode; // physical plan of current subplan + SSubplanId id; // unique id of the subplan + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL|QUERY_TYPE_SCAN + int32_t level; // the execution level of current subplan, starting from 0. + SEpSet execEpSet; // for the scan sub plan, the optional execution node + SArray *pChildern; // the datasource subplan,from which to fetch the result + SArray *pParents; // the data destination subplan, get data from current subplan + SPhyNode *pNode; // physical plan of current subplan } SSubplan; typedef struct SQueryDag { - SArray **pSubplans; + SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. } SQueryDag; /** @@ -73,8 +113,11 @@ typedef struct SQueryDag { */ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); +int32_t qSetSuplanExecutionNode(SArray* subplans, SArray* nodes); + int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); + /** * Convert to subplan to string for the scheduler to send to the executor */ @@ -85,7 +128,7 @@ int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str); * @param pQueryPhyNode * @return */ -void* qDestroyQueryDag(struct SQueryDag* pDag); +void qDestroyQueryDag(struct SQueryDag* pDag); #ifdef __cplusplus } diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h new file mode 100644 index 0000000000..9f51969dc1 --- /dev/null +++ b/include/libs/planner/plannerOp.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#if defined(INCLUDE_AS_ENUM) // enum define mode + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) OP_##op, +#elif defined(INCLUDE_AS_NAME) // comment define mode + #undef OP_ENUM_MACRO + #define OP_ENUM_MACRO(op) #op, +#else + #error To use this include file, first define either INCLUDE_AS_ENUM or INCLUDE_AS_NAME +#endif + +OP_ENUM_MACRO(TableScan) +OP_ENUM_MACRO(DataBlocksOptScan) +OP_ENUM_MACRO(TableSeqScan) +OP_ENUM_MACRO(TagScan) +OP_ENUM_MACRO(SystemTableScan) +OP_ENUM_MACRO(Aggregate) +OP_ENUM_MACRO(Project) +OP_ENUM_MACRO(Groupby) +OP_ENUM_MACRO(Limit) +OP_ENUM_MACRO(SLimit) +OP_ENUM_MACRO(TimeWindow) +OP_ENUM_MACRO(SessionWindow) +OP_ENUM_MACRO(StateWindow) +OP_ENUM_MACRO(Fill) +OP_ENUM_MACRO(MultiTableAggregate) +OP_ENUM_MACRO(MultiTableTimeInterval) +OP_ENUM_MACRO(Filter) +OP_ENUM_MACRO(Distinct) +OP_ENUM_MACRO(Join) +OP_ENUM_MACRO(AllTimeWindow) +OP_ENUM_MACRO(AllMultiTableTimeInterval) +OP_ENUM_MACRO(Order) +OP_ENUM_MACRO(Exchange) diff --git a/include/libs/query/query.h b/include/libs/query/query.h index bfe2db6a61..d92f7d4497 100644 --- a/include/libs/query/query.h +++ b/include/libs/query/query.h @@ -21,6 +21,8 @@ extern "C" { #endif #include "tarray.h" +#include "thash.h" +#include "tlog.h" typedef SVgroupListRspMsg SVgroupListInfo; @@ -63,16 +65,14 @@ typedef struct STableMeta { typedef struct SDBVgroupInfo { - int32_t vgroupVersion; - SArray *vgId; - int32_t hashRange; - int32_t hashType; + int32_t vgVersion; + int8_t hashMethod; + SHashObj *vgInfo; //key:vgId, value:SVgroupInfo } SDBVgroupInfo; typedef struct SUseDbOutput { - SVgroupListInfo *vgroupList; - char db[TSDB_TABLE_FNAME_LEN]; - SDBVgroupInfo *dbVgroup; + char db[TSDB_FULL_DB_NAME_LEN]; + SDBVgroupInfo dbVgroup; } SUseDbOutput; typedef struct STableMetaOutput { @@ -86,6 +86,19 @@ typedef struct STableMetaOutput { extern int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen); extern int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize); +extern void msgInit(); + + +extern int32_t qDebugFlag; + +#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) +#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) + #ifdef __cplusplus } diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 6b3c9ed021..d73e388c20 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,6 +20,12 @@ extern "C" { #endif +#include "planner.h" + +typedef struct SSchedulerCfg { + +} SSchedulerCfg; + typedef struct SQueryProfileSummary { int64_t startTs; // Object created and added into the message queue int64_t endTs; // the timestamp when the task is completed @@ -43,43 +49,23 @@ typedef struct SQueryProfileSummary { uint64_t resultSize; // generated result size in Kb. } SQueryProfileSummary; -typedef struct SQueryTask { - uint64_t queryId; // query id - uint64_t taskId; // task id - char *pSubplan; // operator tree - uint64_t status; // task status - SQueryProfileSummary summary; // task execution summary - void *pOutputHandle; // result buffer handle, to temporarily keep the output result for next stage -} SQueryTask; - -typedef struct SQueryJob { - SArray **pSubtasks; - // todo -} SQueryJob; - /** * Process the query job, generated according to the query physical plan. * This is a synchronized API, and is also thread-safety. * @param pJob * @return */ -int32_t qProcessQueryJob(struct SQueryJob* pJob); +int32_t scheduleQueryJob(SQueryDag* pDag, void** pJob); + +int32_t scheduleFetchRows(void *pJob, void *data); -/** - * The SSqlObj should not be here???? - * @param pSql - * @param pVgroupId - * @param pRetVgroupId - * @return - */ -//SArray* qGetInvolvedVgroupIdList(struct SSqlObj* pSql, SArray* pVgroupId, SArray* pRetVgroupId); /** * Cancel query job * @param pJob * @return */ -int32_t qKillQueryJob(struct SQueryJob* pJob); +int32_t scheduleCancelJob(void *pJob); #ifdef __cplusplus } diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index bafdc6c082..89f24cf3a4 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -16,49 +16,75 @@ #define _TD_WAL_H_ #include "os.h" +#include "tarray.h" #include "tdef.h" #include "tlog.h" -#include "tarray.h" #ifdef __cplusplus extern "C" { #endif extern int32_t wDebugFlag; -#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} -#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} -#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} -#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} -#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} -#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define wFatal(...) \ + { \ + if (wDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \ + } \ + } +#define wError(...) \ + { \ + if (wDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \ + } \ + } +#define wWarn(...) \ + { \ + if (wDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \ + } \ + } +#define wInfo(...) \ + { \ + if (wDebugFlag & DEBUG_INFO) { \ + taosPrintLog("WAL ", 255, __VA_ARGS__); \ + } \ + } +#define wDebug(...) \ + { \ + if (wDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ + } \ + } +#define wTrace(...) \ + { \ + if (wDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ + } \ + } -#define WAL_HEAD_VER 0 +#define WAL_HEAD_VER 0 #define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) -#define WAL_LOG_SUFFIX "log" +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) +#define WAL_LOG_SUFFIX "log" #define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_CUR_FAILED 1 +#define WAL_CUR_FAILED 1 #pragma pack(push, 1) -typedef enum { - TAOS_WAL_NOLOG = 0, - TAOS_WAL_WRITE = 1, - TAOS_WAL_FSYNC = 2 -} EWalType; +typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType; typedef struct SWalReadHead { - int8_t headVer; - uint8_t msgType; - int8_t reserved[2]; - int32_t len; - int64_t ingestTs; //not implemented - int64_t version; - char body[]; + int8_t headVer; + uint8_t msgType; + int8_t reserved[2]; + int32_t len; + int64_t ingestTs; // not implemented + int64_t version; + char body[]; } SWalReadHead; typedef struct { @@ -68,12 +94,12 @@ typedef struct { int32_t rollPeriod; // secs int64_t retentionSize; int64_t segSize; - EWalType level; // wal level + EWalType level; // wal level } SWalCfg; typedef struct { - uint32_t cksumHead; - uint32_t cksumBody; + uint32_t cksumHead; + uint32_t cksumBody; SWalReadHead head; } SWalHead; @@ -89,37 +115,37 @@ typedef struct SWal { // cfg SWalCfg cfg; int32_t fsyncSeq; - //meta + // meta SWalVer vers; int64_t writeLogTfd; int64_t writeIdxTfd; int32_t writeCur; - SArray* fileInfoSet; - //status + SArray *fileInfoSet; + // status int64_t totSize; int64_t lastRollSeq; - //ctl - int64_t refId; + // ctl + int64_t refId; pthread_mutex_t mutex; - //path + // path char path[WAL_PATH_LEN]; - //reusable write head + // reusable write head SWalHead writeHead; } SWal; // WAL HANDLE typedef struct SWalReadHandle { - SWal* pWal; - int64_t readLogTfd; - int64_t readIdxTfd; - int64_t curFileFirstVer; - int64_t curVersion; - int64_t capacity; - int64_t status; //if cursor valid - SWalHead* pHead; + SWal *pWal; + int64_t readLogTfd; + int64_t readIdxTfd; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + int64_t status; // if cursor valid + SWalHead *pHead; } SWalReadHandle; #pragma pack(pop) -//typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); +// typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // module initialization int32_t walInit(); @@ -141,15 +167,15 @@ int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely int32_t walBeginSnapshot(SWal *, int64_t ver); int32_t walEndSnapshot(SWal *); -//int32_t walDataCorrupted(SWal*); +// int32_t walDataCorrupted(SWal*); // read -SWalReadHandle* walOpenReadHandle(SWal *); -void walCloseReadHandle(SWalReadHandle *); -int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); +SWalReadHandle *walOpenReadHandle(SWal *); +void walCloseReadHandle(SWalReadHandle *); +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); int32_t walRead(SWal *, SWalHead **, int64_t ver); -//int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); +// int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); // lifecycle check int64_t walGetFirstVer(SWal *); diff --git a/include/os/os.h b/include/os/os.h index 53a6cef96a..de2a8182db 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -46,6 +46,7 @@ extern "C" { #include #include #include +#include #include #include diff --git a/include/os/osSysinfo.h b/include/os/osSysinfo.h index 56f6b3e0da..6952b91742 100644 --- a/include/os/osSysinfo.h +++ b/include/os/osSysinfo.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include "os.h" + #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 96 @@ -57,11 +59,11 @@ char * taosGetCmdlineByPID(int pid); void taosSetCoreDump(bool enable); typedef struct { - const char *sysname; - const char *nodename; - const char *release; - const char *version; - const char *machine; + char sysname[_UTSNAME_MACHINE_LENGTH]; + char nodename[_UTSNAME_MACHINE_LENGTH]; + char release[_UTSNAME_MACHINE_LENGTH]; + char version[_UTSNAME_MACHINE_LENGTH]; + char machine[_UTSNAME_MACHINE_LENGTH]; } SysNameInfo; SysNameInfo taosGetSysNameInfo(); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 4f1ef7da7b..689d2676d1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -324,6 +324,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica") #define TSDB_CODE_QRY_INVALID_TIME_CONDITION TAOS_DEF_ERROR_CODE(0, 0x070D) //"invalid time condition") #define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070E) //"System error") +#define TSDB_CODE_QRY_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x070F) //"invalid input") // grant diff --git a/include/util/tarray.h b/include/util/tarray.h index e0f14dcd25..f7c72add01 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -41,7 +41,7 @@ typedef struct SArray { * @param elemSize * @return */ -void* taosArrayInit(size_t size, size_t elemSize); +SArray* taosArrayInit(size_t size, size_t elemSize); /** * diff --git a/include/util/tdef.h b/include/util/tdef.h index 85b51b6eff..dfb53de58f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -196,6 +196,7 @@ do { \ #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 +#define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 @@ -232,8 +233,8 @@ do { \ #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MIN_VNODES 64 #define TSDB_MAX_VNODES 512 -#define TSDB_MIN_VNODES_PER_DB 2 -#define TSDB_MAX_VNODES_PER_DB 64 +#define TSDB_MIN_VNODES_PER_DB 1 +#define TSDB_MAX_VNODES_PER_DB 4096 #define TSDB_DNODE_ROLE_ANY 0 #define TSDB_DNODE_ROLE_MGMT 1 @@ -246,7 +247,7 @@ do { \ #define TSDB_RES_COL_ID (-5000) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta - + #define TSDB_MIN_CACHE_BLOCK_SIZE 1 #define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode #define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 diff --git a/include/util/tmacro.h b/include/util/tmacro.h deleted file mode 100644 index 5cca8a1062..0000000000 --- a/include/util/tmacro.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_UTIL_MACRO_H_ -#define _TD_UTIL_MACRO_H_ - -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Module init/clear MACRO definitions -#define TD_MOD_UNINITIALIZED 0 -#define TD_MOD_INITIALIZED 1 - -#define TD_MOD_UNCLEARD 0 -#define TD_MOD_CLEARD 1 - -typedef int8_t td_mode_flag_t; - -#define TD_CHECK_AND_SET_MODE_INIT(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNINITIALIZED, TD_MOD_INITIALIZED) - -#define TD_CHECK_AND_SET_MOD_CLEAR(FLAG) atomic_val_compare_exchange_8((FLAG), TD_MOD_UNCLEARD, TD_MOD_CLEARD) - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_UTIL_MACRO_H_*/ \ No newline at end of file diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index b46afa31b9..61b60e5ba6 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries( taos PRIVATE common INTERFACE api - PRIVATE os util common transport parser + PRIVATE os util common transport parser catalog function query ) ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 3180923aff..9ef1d67e74 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -21,13 +21,14 @@ extern "C" { #endif #include "taos.h" +#include "common.h" #include "taosmsg.h" +#include "tdef.h" +#include "tep.h" #include "thash.h" #include "tlist.h" -#include "trpc.h" -#include "tdef.h" #include "tmsgtype.h" -#include "tep.h" +#include "trpc.h" typedef struct SQueryExecMetric { int64_t start; // start timestamp @@ -86,12 +87,29 @@ typedef struct STscObj { SAppInstInfo *pAppInfo; } STscObj; +typedef struct SClientResultInfo { + const char *pMsg; + const char *pData; + TAOS_FIELD *fields; + int32_t numOfCols; + int32_t numOfRows; + int32_t current; + int32_t *length; + TAOS_ROW row; + char **pCol; +} SClientResultInfo; + typedef struct SReqBody { tsem_t rspSem; // not used now void* fp; void* param; + int32_t paramLen; + int64_t execId; // showId/queryId + SClientResultInfo* pResInfo; } SRequestBody; +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 + typedef struct SRequestObj { uint64_t requestId; int32_t type; // request type @@ -118,7 +136,7 @@ extern int32_t tscReqRef; extern void *tscQhandle; extern int32_t tscConnRef; -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg); +extern int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody); extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); int taos_init(); @@ -129,8 +147,6 @@ void destroyTscObj(void*pObj); void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); - void taos_init_imp(void); int taos_options_imp(TSDB_OPTION option, const char *str); @@ -139,6 +155,12 @@ void* openTransporter(const char *user, const char *auth); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + +void* doFetchRow(SRequestObj* pRequest); +void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); + #ifdef __cplusplus } #endif diff --git a/source/client/inc/tscLog.h b/source/client/inc/clientLog.h similarity index 96% rename from source/client/inc/tscLog.h rename to source/client/inc/clientLog.h index f205a50227..bfa2c0319b 100644 --- a/source/client/inc/tscLog.h +++ b/source/client/inc/clientLog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCLOG_H -#define TDENGINE_TSCLOG_H +#ifndef TDENGINE_CLIENTLOG_H +#define TDENGINE_CLIENTLOG_H #ifdef __cplusplus extern "C" { diff --git a/source/client/src/client.c b/source/client/src/client.c index 863e96aa74..e69de29bb2 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tdef.h" -#include "tglobal.h" -#include "clientInt.h" -#include "tscLog.h" - -TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0)? port:tsServerPort; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); - if (user == NULL) { - user = TSDB_DEFAULT_USER; - } - - if (pass == NULL) { - pass = TSDB_DEFAULT_PASS; - } - - return taos_connect_internal(ip, user, pass, NULL, db, p); -} - -TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { - tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); - if (user == NULL) { - user = TSDB_DEFAULT_USER; - } - - if (auth == NULL) { - tscError("No auth info is given, failed to connect to server"); - return NULL; - } - - return taos_connect_internal(ip, user, NULL, auth, db, port); -} - -TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { - char ipStr[TSDB_EP_LEN] = {0}; - char dbStr[TSDB_DB_NAME_LEN] = {0}; - char userStr[TSDB_USER_LEN] = {0}; - char passStr[TSDB_PASSWORD_LEN] = {0}; - - strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); - strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipStr, userStr, passStr, dbStr, port); -} - diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2ae79f1947..4815e862e8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,15 +1,18 @@ -#include + #include "clientInt.h" +#include "clientLog.h" #include "tdef.h" #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" +#include "tnote.h" +#include "tpagedfile.h" #include "tref.h" -#include "tscLog.h" +#include "parser.h" static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); -static void destroyConnectMsg(SRequestMsgBody* pMsgBody); +static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody); static int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId); @@ -96,17 +99,84 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, char* key = getClusterKey(user, secretEncrypt, ip, port); - SAppInstInfo* pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL) { - pInst = calloc(1, sizeof(struct SAppInstInfo)); + SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); - pInst->mgmtEp = epSet; - pInst->pTransporter = openTransporter(user, secretEncrypt); + p->mgmtEp = epSet; + p->pTransporter = openTransporter(user, secretEncrypt); + taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); - taosHashPut(appInfo.pInstMap, key, strlen(key), &pInst, POINTER_BYTES); + pInst = &p; } - return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst); + tfree(key); + return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, *pInst); +} + +TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { + STscObj *pTscObj = (STscObj *)taos; + if (sqlLen > (size_t) tsMaxSQLStringLen) { + tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + return NULL; + } + + nPrintTsc("%s", sql) + + SRequestObj* pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT); + if (pRequest == NULL) { + tscError("failed to malloc sqlObj"); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + pRequest->sqlstr = malloc(sqlLen + 1); + if (pRequest->sqlstr == NULL) { + tscError("0x%"PRIx64" failed to prepare sql string buffer", pRequest->self); + + pRequest->msgBuf = strdup("failed to prepare sql string buffer"); + terrno = pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRequest; + } + + strntolower(pRequest->sqlstr, sql, (int32_t)sqlLen); + pRequest->sqlstr[sqlLen] = 0; + + tscDebugL("0x%"PRIx64" SQL: %s", pRequest->requestId, pRequest->sqlstr); + + int32_t code = 0; + if (qIsInsertSql(pRequest->sqlstr, sqlLen)) { + // todo add + } else { + int32_t type = 0; + void* output = NULL; + int32_t outputLen = 0; + code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW) { + pRequest->type = type; + pRequest->body.param = output; + pRequest->body.paramLen = outputLen; + + SRequestMsgBody body = {0}; + buildRequestMsgFp[type](pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyRequestMsgBody(&body); + } else { + assert(0); + } + } + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + return pRequest; + } + + return pRequest; } int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { @@ -166,7 +236,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); tsem_wait(&pRequest->body.rspSem); - destroyConnectMsg(&body); + destroyRequestMsgBody(&body); if (pRequest->code != TSDB_CODE_SUCCESS) { const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); @@ -213,7 +283,7 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) return 0; } -static void destroyConnectMsg(SRequestMsgBody* pMsgBody) { +static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody) { assert(pMsgBody != NULL); tfree(pMsgBody->pData); } @@ -269,11 +339,18 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { tscDebug("0x%" PRIx64 " message:%s, code:%s rspLen:%d, elapsed:%"PRId64 " ms", pRequest->requestId, taosMsg[pMsg->msgType], tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); if (handleRequestRspFp[pRequest->type]) { - pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); + char *p = malloc(pMsg->contLen); + if (p == NULL) { + pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + terrno = pRequest->code; + } else { + memcpy(p, pMsg->pCont, pMsg->contLen); + pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, p, pMsg->contLen); + } } } else { - tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType], - tstrerror(pMsg->code), pMsg->contLen); + tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%"PRId64" ms", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); } taosReleaseRef(tscReqRef, requestRefId); @@ -281,3 +358,80 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { sem_post(&pRequest->body.rspSem); } + +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (auth == NULL) { + tscError("No auth info is given, failed to connect to server"); + return NULL; + } + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { + char ipStr[TSDB_EP_LEN] = {0}; + char dbStr[TSDB_DB_NAME_LEN] = {0}; + char userStr[TSDB_USER_LEN] = {0}; + char passStr[TSDB_PASSWORD_LEN] = {0}; + + strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipStr, userStr, passStr, dbStr, port); +} + +void* doFetchRow(SRequestObj* pRequest) { + assert(pRequest != NULL); + SClientResultInfo* pResultInfo = pRequest->body.pResInfo; + + if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { + pRequest->type = TSDB_SQL_RETRIEVE_MNODE; + + SRequestMsgBody body = {0}; + buildRequestMsgFp[pRequest->type](pRequest, &body); + + int64_t transporterId = 0; + STscObj* pTscObj = pRequest->pTscObj; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyRequestMsgBody(&body); + + pResultInfo->current = 0; + if (pResultInfo->numOfRows <= pResultInfo->current) { + return NULL; + } + } + + for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) { + pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; + if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { + pResultInfo->length[i] = varDataLen(pResultInfo->row[i]); + pResultInfo->row[i] = varDataVal(pResultInfo->row[i]); + } + } + + pResultInfo->current += 1; + return pResultInfo->row; +} + +void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { + assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); + if (numOfRows == 0) { + return; + } + + int32_t offset = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + pResultInfo->length[i] = pResultInfo->fields[i].bytes; + pResultInfo->row[i] = pResultInfo->pData + offset * pResultInfo->numOfRows; + pResultInfo->pCol[i] = pResultInfo->row[i]; + offset += pResultInfo->fields[i].bytes; + } +} \ No newline at end of file diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c new file mode 100644 index 0000000000..21e632db8d --- /dev/null +++ b/source/client/src/clientMain.c @@ -0,0 +1,269 @@ +#include "clientInt.h" +#include "clientLog.h" +#include "os.h" +#include "taosmsg.h" +#include "tcache.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "tref.h" +#include "trpc.h" +#include "tsched.h" +#include "ttime.h" +#include "ttimezone.h" + +#define TSC_VAR_NOT_RELEASE 1 +#define TSC_VAR_RELEASED 0 + +static int32_t sentinel = TSC_VAR_NOT_RELEASE; + +int taos_options(TSDB_OPTION option, const void *arg, ...) { + static int32_t lock = 0; + + for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { + if (i % 1000 == 0) { + tscInfo("haven't acquire lock after spin %d times.", i); + sched_yield(); + } + } + + int ret = taos_options_imp(option, (const char*)arg); + atomic_store_32(&lock, 0); + return ret; +} + +// this function may be called by user or system, or by both simultaneously. +void taos_cleanup(void) { + tscDebug("start to cleanup client environment"); + + if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { + return; + } + + int32_t id = tscReqRef; + tscReqRef = -1; + taosCloseRef(id); + + void* p = tscQhandle; + tscQhandle = NULL; + taosCleanUpScheduler(p); + + id = tscConnRef; + tscConnRef = -1; + taosCloseRef(id); + + rpcCleanup(); + taosCloseLog(); +} + +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + int32_t p = (port != 0) ? port : tsServerPort; + + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } + + return taos_connect_internal(ip, user, pass, NULL, db, p); +} + +void taos_close(TAOS* taos) { + if (taos == NULL) { + return; + } + + STscObj *pTscObj = (STscObj *)taos; + tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); + + taosRemoveRef(tscConnRef, pTscObj->id); +} + +int taos_errno(TAOS_RES *tres) { + if (tres == NULL) { + return terrno; + } + + return ((SRequestObj*) tres)->code; +} + +const char *taos_errstr(TAOS_RES *res) { + SRequestObj *pRequest = (SRequestObj *) res; + + if (pRequest == NULL) { + return (const char*) tstrerror(terrno); + } + + if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { + return pRequest->msgBuf; + } else { + return (const char*)tstrerror(pRequest->code); + } +} + +void taos_free_result(TAOS_RES *res) { + SRequestObj* pRequest = (SRequestObj*) res; + destroyRequest(pRequest); +} + +int taos_field_count(TAOS_RES *res) { + if (res == NULL) { + return 0; + } + + SRequestObj* pRequest = (SRequestObj*) res; + + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + if (pResInfo == NULL) { + return 0; + } + + return pResInfo->numOfCols; +} + +int taos_num_fields(TAOS_RES *res) { + return taos_field_count(res); +} + +TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { + if (taos_num_fields(res) == 0) { + return NULL; + } + + SClientResultInfo* pResInfo = ((SRequestObj*) res)->body.pResInfo; + return pResInfo->fields; +} + +TAOS_RES *taos_query(TAOS *taos, const char *sql) { + if (taos == NULL || sql == NULL) { + return NULL; + } + + return taos_query_l(taos, sql, strlen(sql)); +} + +TAOS_ROW taos_fetch_row(TAOS_RES *pRes) { + if (pRes == NULL) { + return NULL; + } + + SRequestObj *pRequest = (SRequestObj *) pRes; + if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + pRequest->type == TSDB_SQL_INSERT) { + return NULL; + } + + return doFetchRow(pRequest); +} + +int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) { + int32_t len = 0; + for (int i = 0; i < num_fields; ++i) { + if (i > 0) { + str[len++] = ' '; + } + + if (row[i] == NULL) { + len += sprintf(str + len, "%s", TSDB_DATA_NULL_STR); + continue; + } + + switch (fields[i].type) { + case TSDB_DATA_TYPE_TINYINT: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UTINYINT: + len += sprintf(str + len, "%u", *((uint8_t *)row[i])); + break; + + case TSDB_DATA_TYPE_SMALLINT: + len += sprintf(str + len, "%d", *((int16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_USMALLINT: + len += sprintf(str + len, "%u", *((uint16_t *)row[i])); + break; + + case TSDB_DATA_TYPE_INT: + len += sprintf(str + len, "%d", *((int32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UINT: + len += sprintf(str + len, "%u", *((uint32_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BIGINT: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_UBIGINT: + len += sprintf(str + len, "%" PRIu64, *((uint64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_FLOAT: { + float fv = 0; + fv = GET_FLOAT_VAL(row[i]); + len += sprintf(str + len, "%f", fv); + } break; + + case TSDB_DATA_TYPE_DOUBLE: { + double dv = 0; + dv = GET_DOUBLE_VAL(row[i]); + len += sprintf(str + len, "%lf", dv); + } break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + int32_t charLen = varDataLen((char*)row[i] - VARSTR_HEADER_SIZE); + if (fields[i].type == TSDB_DATA_TYPE_BINARY) { + assert(charLen <= fields[i].bytes && charLen >= 0); + } else { + assert(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0); + } + + memcpy(str + len, row[i], charLen); + len += charLen; + } break; + + case TSDB_DATA_TYPE_TIMESTAMP: + len += sprintf(str + len, "%" PRId64, *((int64_t *)row[i])); + break; + + case TSDB_DATA_TYPE_BOOL: + len += sprintf(str + len, "%d", *((int8_t *)row[i])); + default: + break; + } + } + + return len; +} + +int* taos_fetch_lengths(TAOS_RES *res) { + if (res == NULL) { + return NULL; + } + + return ((SRequestObj*) res)->body.pResInfo->length; +} + +const char *taos_data_type(int type) { + switch (type) { + case TSDB_DATA_TYPE_NULL: return "TSDB_DATA_TYPE_NULL"; + case TSDB_DATA_TYPE_BOOL: return "TSDB_DATA_TYPE_BOOL"; + case TSDB_DATA_TYPE_TINYINT: return "TSDB_DATA_TYPE_TINYINT"; + case TSDB_DATA_TYPE_SMALLINT: return "TSDB_DATA_TYPE_SMALLINT"; + case TSDB_DATA_TYPE_INT: return "TSDB_DATA_TYPE_INT"; + case TSDB_DATA_TYPE_BIGINT: return "TSDB_DATA_TYPE_BIGINT"; + case TSDB_DATA_TYPE_FLOAT: return "TSDB_DATA_TYPE_FLOAT"; + case TSDB_DATA_TYPE_DOUBLE: return "TSDB_DATA_TYPE_DOUBLE"; + case TSDB_DATA_TYPE_BINARY: return "TSDB_DATA_TYPE_BINARY"; + case TSDB_DATA_TYPE_TIMESTAMP: return "TSDB_DATA_TYPE_TIMESTAMP"; + case TSDB_DATA_TYPE_NCHAR: return "TSDB_DATA_TYPE_NCHAR"; + default: return "UNKNOWN"; + } +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 62466a096d..646964e319 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ -#include "os.h" +#include #include "clientInt.h" +#include "clientLog.h" +#include "os.h" #include "tmsgtype.h" #include "trpc.h" -#include "tscLog.h" int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); @@ -528,7 +529,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) { if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || @@ -2699,7 +2700,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { } STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE) || + if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && @@ -3091,6 +3092,36 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { #endif +int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; + pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); + if (pConnect == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + // TODO refactor full_name + char *db; // ugly code to move the space + + STscObj *pObj = pRequest->pTscObj; + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + pConnect->pid = htonl(appInfo.pid); + pConnect->startTime = htobe64(appInfo.startTime); + tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); + + pMsgBody->pData = pConnect; + return 0; +} + int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { STscObj *pTscObj = pRequest->pTscObj; @@ -3099,6 +3130,11 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { pConnect->connId = htonl(pConnect->connId); pConnect->clusterId = htonl(pConnect->clusterId); + assert(pConnect->epSet.numOfEps > 0); + for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) { + pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]); + } + // TODO refactor pthread_mutex_lock(&pTscObj->mutex); char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; @@ -3108,13 +3144,12 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); pthread_mutex_unlock(&pTscObj->mutex); - assert(pConnect->epSet.numOfEps > 0); if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); } for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id); + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id); } pTscObj->connId = pConnect->connId; @@ -3123,14 +3158,132 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); - // createHbObj(pTscObj); + pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); + pRequest->body.pResInfo->pMsg = pMsg; - // launch a timer to send heartbeat to maintain the connection and send status to mnode - // taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer); + tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); return 0; } +int32_t buildCreateUserMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; + pMsgBody->msgLen = sizeof(SCreateUserMsg); + pMsgBody->requestObjRefId = pRequest->self; + pMsgBody->pData = pRequest->body.param; + return 0; +} + +int32_t buildShowMsg(SRequestObj* pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_SHOW; + pMsgBody->msgLen = pRequest->body.paramLen; + pMsgBody->requestObjRefId = pRequest->self; + pMsgBody->pData = pRequest->body.param; +} + +STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); + + size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); + + pTableMeta->tableType = pTableMetaMsg->tableType; + pTableMeta->vgId = pTableMetaMsg->vgId; + pTableMeta->suid = pTableMetaMsg->suid; + pTableMeta->uid = pTableMetaMsg->tuid; + + pTableMeta->tableInfo = (STableComInfo) { + .numOfTags = pTableMetaMsg->numOfTags, + .precision = pTableMetaMsg->precision, + .numOfColumns = pTableMetaMsg->numOfColumns, + }; + + pTableMeta->sversion = pTableMetaMsg->sversion; + pTableMeta->tversion = pTableMetaMsg->tversion; + + memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize); + + int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < numOfTotalCols; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + SShowRsp* pShow = (SShowRsp *)pMsg; + pShow->showId = htonl(pShow->showId); + + STableMetaMsg *pMetaMsg = &(pShow->tableMeta); + pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); + + SSchema* pSchema = pMetaMsg->pSchema; + pMetaMsg->tuid = htobe64(pMetaMsg->tuid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema->bytes = htonl(pSchema->bytes); + pSchema++; + } + + pSchema = pMetaMsg->pSchema; + TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { + tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); + pFields[i].type = pSchema[i].type; + pFields[i].bytes = pSchema[i].bytes; + } + + if (pRequest->body.pResInfo == NULL) { + pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); + } + + pRequest->body.pResInfo->pMsg = pMsg; + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + + pResInfo->fields = pFields; + pResInfo->numOfCols = pMetaMsg->numOfColumns; + pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); + + pRequest->body.execId = pShow->showId; + return 0; +} + +int buildRetrieveMnodeMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE; + pMsgBody->msgLen = sizeof(SRetrieveTableMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg)); + pRetrieveMsg->showId = htonl(pRequest->body.execId); + + pMsgBody->pData = pRetrieveMsg; + return TSDB_CODE_SUCCESS; +} + +int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + assert(msgLen >= sizeof(SRetrieveTableRsp)); + + tfree(pRequest->body.pResInfo->pMsg); + pRequest->body.pResInfo->pMsg = pMsg; + + SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg; + pRetrieve->numOfRows = htonl(pRetrieve->numOfRows); + pRetrieve->precision = htons(pRetrieve->precision); + + SClientResultInfo* pResInfo = pRequest->body.pResInfo; + pResInfo->numOfRows = pRetrieve->numOfRows; + pResInfo->pData = pRetrieve->data; // todo fix this in async model + + pResInfo->current = 0; + setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); + + tscDebug("0x%"PRIx64" numOfRows:%d, complete:%d, qId:0x%"PRIx64, pRequest->self, pRetrieve->numOfRows, + pRetrieve->completed, pRequest->body.execId); + return 0; +} + + void initMsgHandleFp() { #if 0 tscBuildMsg[TSDB_SQL_SELECT] = tscBuildQueryMsg; @@ -3167,7 +3320,7 @@ void initMsgHandleFp() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -3185,7 +3338,7 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; @@ -3207,6 +3360,14 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; #endif -// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + buildRequestMsgFp[TSDB_SQL_CONNECT] = buildConnectMsg; handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; + + buildRequestMsgFp[TSDB_SQL_CREATE_USER] = buildCreateUserMsg; + + buildRequestMsgFp[TSDB_SQL_SHOW] = buildShowMsg; + handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; + + buildRequestMsgFp[TSDB_SQL_RETRIEVE_MNODE] = buildRetrieveMnodeMsg; + handleRequestRspFp[TSDB_SQL_RETRIEVE_MNODE]= processRetrieveMnodeRsp; } \ No newline at end of file diff --git a/source/client/src/clientmain.c b/source/client/src/clientmain.c deleted file mode 100644 index ba80135850..0000000000 --- a/source/client/src/clientmain.c +++ /dev/null @@ -1,85 +0,0 @@ -#include "clientInt.h" -#include "trpc.h" -#include "os.h" -#include "taosmsg.h" -#include "tcache.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "tref.h" -#include "tscLog.h" -#include "tsched.h" -#include "ttime.h" -#include "ttimezone.h" - -#define TSC_VAR_NOT_RELEASE 1 -#define TSC_VAR_RELEASED 0 - -static int32_t sentinel = TSC_VAR_NOT_RELEASE; -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; - -extern int32_t tscInitRes; - -int taos_options(TSDB_OPTION option, const void *arg, ...) { - static int32_t lock = 0; - - for (int i = 1; atomic_val_compare_exchange_32(&lock, 0, 1) != 0; ++i) { - if (i % 1000 == 0) { - tscInfo("haven't acquire lock after spin %d times.", i); - sched_yield(); - } - } - - int ret = taos_options_imp(option, (const char*)arg); - - atomic_store_32(&lock, 0); - return ret; -} - -int taos_init() { - pthread_once(&tscinit, taos_init_imp); - return tscInitRes; -} - -// this function may be called by user or system, or by both simultaneously. -void taos_cleanup(void) { - tscDebug("start to cleanup client environment"); - - if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) { - return; - } - - int32_t id = tscReqRef; - tscReqRef = -1; - taosCloseRef(id); - - void* p = tscQhandle; - tscQhandle = NULL; - taosCleanUpScheduler(p); - - id = tscConnRef; - tscConnRef = -1; - taosCloseRef(id); - - rpcCleanup(); - taosCloseLog(); -} - -void taos_close(TAOS* taos) { - if (taos == NULL) { - return; - } - - STscObj *pTscObj = (STscObj *)taos; - tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); - - taosRemoveRef(tscConnRef, pTscObj->id); -} - -const char *taos_errstr(TAOS_RES *res) { - -} - -void taos_free_result(TAOS_RES *res) { - -} \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 43d73bf3db..023bd6ebe9 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include "clientInt.h" +#include "clientLog.h" #include "os.h" #include "taosmsg.h" #include "tcache.h" @@ -20,12 +22,10 @@ #include "tglobal.h" #include "tnote.h" #include "tref.h" -#include "tscLog.h" +#include "trpc.h" #include "tsched.h" #include "ttime.h" -#include "trpc.h" #include "ttimezone.h" -#include "clientInt.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -35,6 +35,7 @@ int32_t tscReqRef = -1; int32_t tscConnRef = -1; void *tscQhandle = NULL; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; int32_t tsNumOfThreads = 1; volatile int32_t tscInitRes = 0; @@ -64,7 +65,7 @@ static void deregisterRequest(SRequestObj* pRequest) { SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); - int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); taosReleaseRef(tscConnRef, pTscObj->id); @@ -126,8 +127,6 @@ void destroyTscObj(void *pObj) { atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); - - closeTransporter(pTscObj); pthread_mutex_destroy(&pTscObj->mutex); tfree(pTscObj); } @@ -172,6 +171,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty pRequest->pTscObj = pObj; pRequest->body.fp = fp; pRequest->body.param = param; + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); @@ -188,6 +188,12 @@ static void doDestroyRequest(void* p) { tfree(pRequest->sqlstr); tfree(pRequest->pInfo); + if (pRequest->body.pResInfo != NULL) { + tfree(pRequest->body.pResInfo->pData); + tfree(pRequest->body.pResInfo->pMsg); + tfree(pRequest->body.pResInfo); + } + deregisterRequest(pRequest); tfree(pRequest); } @@ -250,6 +256,11 @@ void taos_init_imp(void) { tscDebug("client is initialized successfully"); } +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + int taos_options_imp(TSDB_OPTION option, const char *str) { SGlobalCfg *cfg = NULL; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 8da4caf7eb..46fd76234e 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -14,6 +14,7 @@ */ #include +#include #include #include "tglobal.h" #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -22,6 +23,7 @@ #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" +#include "../inc/clientInt.h" #include "taos.h" namespace { @@ -33,8 +35,62 @@ int main(int argc, char** argv) { } TEST(testCase, driverInit_Test) { + taos_init(); +} + +TEST(testCase, connect_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + taos_close(pConn); +} + +TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); taos_close(pConn); -} \ No newline at end of file +} + +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 6dc46cefcd..1db92644ae 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -44,8 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_ACK] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_RESET] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg; // msg from client to mnode pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mgmt/impl/test/acct/acct.cpp b/source/dnode/mgmt/impl/test/acct/acct.cpp index e1a71c5c9d..6de8c70ba9 100644 --- a/source/dnode/mgmt/impl/test/acct/acct.cpp +++ b/source/dnode/mgmt/impl/test/acct/acct.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file vnodeApiTests.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module acct-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -49,7 +45,7 @@ SServer* DndTestAcct::pServer; SClient* DndTestAcct::pClient; int32_t DndTestAcct::connId; -TEST_F(DndTestAcct, CreateAcct) { +TEST_F(DndTestAcct, 01_CreateAcct) { ASSERT_NE(pClient, nullptr); SCreateAcctMsg* pReq = (SCreateAcctMsg*)rpcMallocCont(sizeof(SCreateAcctMsg)); @@ -65,7 +61,7 @@ TEST_F(DndTestAcct, CreateAcct) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } -TEST_F(DndTestAcct, AlterAcct) { +TEST_F(DndTestAcct, 02_AlterAcct) { ASSERT_NE(pClient, nullptr); SAlterAcctMsg* pReq = (SAlterAcctMsg*)rpcMallocCont(sizeof(SAlterAcctMsg)); @@ -81,7 +77,7 @@ TEST_F(DndTestAcct, AlterAcct) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } -TEST_F(DndTestAcct, DropAcct) { +TEST_F(DndTestAcct, 03_DropAcct) { ASSERT_NE(pClient, nullptr); SDropAcctMsg* pReq = (SDropAcctMsg*)rpcMallocCont(sizeof(SDropAcctMsg)); @@ -97,7 +93,7 @@ TEST_F(DndTestAcct, DropAcct) { ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); } -TEST_F(DndTestAcct, ShowAcct) { +TEST_F(DndTestAcct, 04_ShowAcct) { ASSERT_NE(pClient, nullptr); SShowMsg* pReq = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); diff --git a/source/dnode/mgmt/impl/test/cluster/cluster.cpp b/source/dnode/mgmt/impl/test/cluster/cluster.cpp index d47e63a85f..4e1dbf8911 100644 --- a/source/dnode/mgmt/impl/test/cluster/cluster.cpp +++ b/source/dnode/mgmt/impl/test/cluster/cluster.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file vnodeApiTests.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module cluster-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index b0ee00cb64..6821dec0a6 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file vnodeApiTests.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module db-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -30,7 +26,7 @@ class DndTestDb : public ::testing::Test { const char* firstEp = "localhost:9040"; pServer = CreateServer("/tmp/dnode_test_db", fqdn, 9040, firstEp); pClient = createClient("root", "taosdata", fqdn, 9040); - taosMsleep(300); + taosMsleep(1100); } static void TearDownTestSuite() { @@ -48,11 +44,12 @@ class DndTestDb : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns) { + void SendTheCheckShowMetaMsg(int8_t showType, const char* showName, int32_t columns, const char* db) { SShowMsg* pShow = (SShowMsg*)rpcMallocCont(sizeof(SShowMsg)); pShow->type = showType; - strcpy(pShow->db, ""); - + if (db != NULL) { + strcpy(pShow->db, db); + } SRpcMsg showRpcMsg = {0}; showRpcMsg.pCont = pShow; showRpcMsg.contLen = sizeof(SShowMsg); @@ -67,10 +64,10 @@ class DndTestDb : public ::testing::Test { ASSERT_NE(pShowRsp, nullptr); pShowRsp->showId = htonl(pShowRsp->showId); pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); pMeta->tuid = htobe64(pMeta->tuid); pMeta->suid = htobe64(pMeta->suid); @@ -183,31 +180,33 @@ SClient* DndTestDb::pClient; int32_t DndTestDb::connId; TEST_F(DndTestDb, 01_ShowDb) { - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); CheckSchema(0, TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN - 1 + VARSTR_HEADER_SIZE, "name"); CheckSchema(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create time"); - CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); - CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); - CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "days"); - CheckSchema(5, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); - CheckSchema(6, TSDB_DATA_TYPE_INT, 4, "cache(MB)"); - CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "blocks"); - CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "minrows"); - CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "maxrows"); - CheckSchema(10, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); - CheckSchema(11, TSDB_DATA_TYPE_INT, 4, "fsync"); - CheckSchema(12, TSDB_DATA_TYPE_TINYINT, 1, "comp"); - CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); - CheckSchema(14, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); - CheckSchema(15, TSDB_DATA_TYPE_TINYINT, 1, "update"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "vgroups"); + CheckSchema(3, TSDB_DATA_TYPE_SMALLINT, 2, "replica"); + CheckSchema(4, TSDB_DATA_TYPE_SMALLINT, 2, "quorum"); + CheckSchema(5, TSDB_DATA_TYPE_SMALLINT, 2, "days"); + CheckSchema(6, TSDB_DATA_TYPE_BINARY, 24 + VARSTR_HEADER_SIZE, "keep0,keep1,keep2"); + CheckSchema(7, TSDB_DATA_TYPE_INT, 4, "cache(MB)"); + CheckSchema(8, TSDB_DATA_TYPE_INT, 4, "blocks"); + CheckSchema(9, TSDB_DATA_TYPE_INT, 4, "minrows"); + CheckSchema(10, TSDB_DATA_TYPE_INT, 4, "maxrows"); + CheckSchema(11, TSDB_DATA_TYPE_TINYINT, 1, "wallevel"); + CheckSchema(12, TSDB_DATA_TYPE_INT, 4, "fsync"); + CheckSchema(13, TSDB_DATA_TYPE_TINYINT, 1, "comp"); + CheckSchema(14, TSDB_DATA_TYPE_TINYINT, 1, "cachelast"); + CheckSchema(15, TSDB_DATA_TYPE_BINARY, 3 + VARSTR_HEADER_SIZE, "precision"); + CheckSchema(16, TSDB_DATA_TYPE_TINYINT, 1, "update"); SendThenCheckShowRetrieveMsg(0); } -TEST_F(DndTestDb, 02_CreateDb) { +TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { { SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); strcpy(pReq->db, "1.d1"); + pReq->numOfVgroups = htonl(2); pReq->cacheBlockSize = htonl(16); pReq->totalBlocks = htonl(10); pReq->daysPerFile = htonl(10); @@ -238,10 +237,11 @@ TEST_F(DndTestDb, 02_CreateDb) { ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); SendThenCheckShowRetrieveMsg(1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); + CheckInt16(2); // vgroups CheckInt16(1); // replica CheckInt16(1); // quorum CheckInt16(10); // days @@ -256,9 +256,22 @@ TEST_F(DndTestDb, 02_CreateDb) { CheckInt8(0); // cachelast CheckBinary("ms", 3); // precision CheckInt8(0); // update -} -TEST_F(DndTestDb, 03_AlterDb) { + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_VGROUP, "show vgroups", 4, "1.d1"); + CheckSchema(0, TSDB_DATA_TYPE_INT, 4, "vgId"); + CheckSchema(1, TSDB_DATA_TYPE_INT, 4, "tables"); + CheckSchema(2, TSDB_DATA_TYPE_SMALLINT, 2, "v1_dnode"); + CheckSchema(3, TSDB_DATA_TYPE_BINARY, 9 + VARSTR_HEADER_SIZE, "v1_status"); + SendThenCheckShowRetrieveMsg(2); + CheckInt32(1); + CheckInt32(2); + CheckInt32(0); + CheckInt32(0); + CheckInt16(1); + CheckInt16(1); + CheckBinary("master", 9); + CheckBinary("master", 9); + { SAlterDbMsg* pReq = (SAlterDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); strcpy(pReq->db, "1.d1"); @@ -282,10 +295,11 @@ TEST_F(DndTestDb, 03_AlterDb) { ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); SendThenCheckShowRetrieveMsg(1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); + CheckInt16(2); // vgroups CheckInt16(1); // replica CheckInt16(2); // quorum CheckInt16(10); // days @@ -300,9 +314,8 @@ TEST_F(DndTestDb, 03_AlterDb) { CheckInt8(1); // cachelast CheckBinary("ms", 3); // precision CheckInt8(0); // update -} -TEST_F(DndTestDb, 04_RestartDnode) { + // restart stopServer(pServer); pServer = NULL; @@ -314,10 +327,11 @@ TEST_F(DndTestDb, 04_RestartDnode) { uInfo("all server is running"); - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); SendThenCheckShowRetrieveMsg(1); CheckBinary("d1", TSDB_DB_NAME_LEN - 1); CheckTimestamp(); + CheckInt16(2); // vgroups CheckInt16(1); // replica CheckInt16(2); // quorum CheckInt16(10); // days @@ -332,11 +346,9 @@ TEST_F(DndTestDb, 04_RestartDnode) { CheckInt8(1); // cachelast CheckBinary("ms", 3); // precision CheckInt8(0); // update -} -TEST_F(DndTestDb, 05_DropDb) { { - SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SAlterDbMsg)); + SDropDbMsg* pReq = (SDropDbMsg*)rpcMallocCont(sizeof(SDropDbMsg)); strcpy(pReq->db, "1.d1"); SRpcMsg rpcMsg = {0}; @@ -350,6 +362,103 @@ TEST_F(DndTestDb, 05_DropDb) { ASSERT_EQ(pMsg->code, 0); } - SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 16); + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); SendThenCheckShowRetrieveMsg(0); -} \ No newline at end of file +} + +TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { + { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.d2"); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + + SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DB, "show databases", 17, NULL); + SendThenCheckShowRetrieveMsg(1); + CheckBinary("d2", TSDB_DB_NAME_LEN - 1); + + { + SUseDbMsg* pReq = (SUseDbMsg*)rpcMallocCont(sizeof(SUseDbMsg)); + strcpy(pReq->db, "1.d2"); + pReq->vgVersion = htonl(-1); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SUseDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_USE_DB; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + + SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; + EXPECT_STREQ(pRsp->db, "1.d2"); + pRsp->vgVersion = htonl(pRsp->vgVersion); + pRsp->vgNum = htonl(pRsp->vgNum); + pRsp->hashMethod = pRsp->hashMethod; + EXPECT_EQ(pRsp->vgVersion, 1); + EXPECT_EQ(pRsp->vgNum, 2); + EXPECT_EQ(pRsp->hashMethod, 1); + + { + SVgroupInfo* pInfo = &pRsp->vgroupInfo[0]; + pInfo->vgId = htonl(pInfo->vgId); + pInfo->hashBegin = htonl(pInfo->hashBegin); + pInfo->hashEnd = htonl(pInfo->hashEnd); + EXPECT_GT(pInfo->vgId, 0); + EXPECT_EQ(pInfo->hashBegin, 0); + EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); + EXPECT_EQ(pInfo->inUse, 0); + EXPECT_EQ(pInfo->numOfEps, 1); + SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + pAddr->port = htons(pAddr->port); + EXPECT_EQ(pAddr->port, 9040); + EXPECT_STREQ(pAddr->fqdn, "localhost"); + } + + { + SVgroupInfo* pInfo = &pRsp->vgroupInfo[1]; + pInfo->vgId = htonl(pInfo->vgId); + pInfo->hashBegin = htonl(pInfo->hashBegin); + pInfo->hashEnd = htonl(pInfo->hashEnd); + EXPECT_GT(pInfo->vgId, 0); + EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); + EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); + EXPECT_EQ(pInfo->inUse, 0); + EXPECT_EQ(pInfo->numOfEps, 1); + SEpAddrMsg* pAddr = &pInfo->epAddr[0]; + pAddr->port = htons(pAddr->port); + EXPECT_EQ(pAddr->port, 9040); + EXPECT_STREQ(pAddr->fqdn, "localhost"); + } + } +} diff --git a/source/dnode/mgmt/impl/test/dnode/dnode.cpp b/source/dnode/mgmt/impl/test/dnode/dnode.cpp index 5a4512e16c..f382f385d7 100644 --- a/source/dnode/mgmt/impl/test/dnode/dnode.cpp +++ b/source/dnode/mgmt/impl/test/dnode/dnode.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file vnodeApiTests.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module dnode-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -82,11 +78,11 @@ class DndTestDnode : public ::testing::Test { ASSERT_NE(pShowRsp, nullptr); pShowRsp->showId = htonl(pShowRsp->showId); pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); pMeta->tversion = htons(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); + pMeta->tuid = htonl(pMeta->tuid); pMeta->suid = htobe64(pMeta->suid); showId = pShowRsp->showId; @@ -224,19 +220,21 @@ TEST_F(DndTestDnode, 02_ConfigDnode) { ASSERT_EQ(pMsg->code, 0); } -TEST_F(DndTestDnode, 03_CreateDnode) { - SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); - strcpy(pReq->ep, "localhost:9042"); +TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) { + { + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); + strcpy(pReq->ep, "localhost:9042"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DNODE; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } taosMsleep(1300); SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); @@ -255,21 +253,21 @@ TEST_F(DndTestDnode, 03_CreateDnode) { CheckTimestamp(); CheckBinary("", 24); CheckBinary("", 24); -} -TEST_F(DndTestDnode, 04_DropDnode) { - SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); - pReq->dnodeId = htonl(2); + { + SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(sizeof(SDropDnodeMsg)); + pReq->dnodeId = htonl(2); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropDnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropDnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_DNODE; - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "show dnodes", 7); SendThenCheckShowRetrieveMsg(1); @@ -280,9 +278,7 @@ TEST_F(DndTestDnode, 04_DropDnode) { CheckBinary("ready", 10); CheckTimestamp(); CheckBinary("", 24); -} -TEST_F(DndTestDnode, 05_CreateDnode) { { SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(sizeof(SCreateDnodeMsg)); strcpy(pReq->ep, "localhost:9043"); @@ -359,9 +355,8 @@ TEST_F(DndTestDnode, 05_CreateDnode) { CheckBinary("", 24); CheckBinary("", 24); CheckBinary("", 24); -} -TEST_F(DndTestDnode, 06_RestartDnode) { + // restart uInfo("stop all server"); stopServer(pServer1); stopServer(pServer2); diff --git a/source/dnode/mgmt/impl/test/profile/profile.cpp b/source/dnode/mgmt/impl/test/profile/profile.cpp index 2cbb9d5c2c..e2dbeb1a5b 100644 --- a/source/dnode/mgmt/impl/test/profile/profile.cpp +++ b/source/dnode/mgmt/impl/test/profile/profile.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file vnodeApiTests.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module profile-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" diff --git a/source/dnode/mgmt/impl/test/show/show.cpp b/source/dnode/mgmt/impl/test/show/show.cpp index c535386ecd..06e8d42ad4 100644 --- a/source/dnode/mgmt/impl/test/show/show.cpp +++ b/source/dnode/mgmt/impl/test/show/show.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file vnodeApiTests.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module show-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" diff --git a/source/dnode/mgmt/impl/test/user/user.cpp b/source/dnode/mgmt/impl/test/user/user.cpp index 2b30dcbb1b..74e8e0db44 100644 --- a/source/dnode/mgmt/impl/test/user/user.cpp +++ b/source/dnode/mgmt/impl/test/user/user.cpp @@ -1,16 +1,12 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. +/** + * @file vnodeApiTests.cpp + * @author slguan (slguan@taosdata.com) + * @brief DNODE module user-msg tests + * @version 0.1 + * @date 2021-12-15 * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. + * @copyright Copyright (c) 2021 * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . */ #include "deploy.h" @@ -67,10 +63,10 @@ class DndTestUser : public ::testing::Test { ASSERT_NE(pShowRsp, nullptr); pShowRsp->showId = htonl(pShowRsp->showId); pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htons(pMeta->numOfTags); - pMeta->numOfColumns = htons(pMeta->numOfColumns); - pMeta->sversion = htons(pMeta->sversion); - pMeta->tversion = htons(pMeta->tversion); + pMeta->numOfTags = htonl(pMeta->numOfTags); + pMeta->numOfColumns = htonl(pMeta->numOfColumns); + pMeta->sversion = htonl(pMeta->sversion); + pMeta->tversion = htonl(pMeta->tversion); pMeta->tuid = htobe64(pMeta->tuid); pMeta->suid = htobe64(pMeta->suid); @@ -184,7 +180,7 @@ TEST_F(DndTestUser, 01_ShowUser) { CheckBinary("root", TSDB_USER_LEN); } -TEST_F(DndTestUser, 02_CreateUser) { +TEST_F(DndTestUser, 02_Create_Drop_Alter_User) { { SCreateUserMsg* pReq = (SCreateUserMsg*)rpcMallocCont(sizeof(SCreateUserMsg)); strcpy(pReq->user, "u1"); @@ -231,23 +227,22 @@ TEST_F(DndTestUser, 02_CreateUser) { CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} -TEST_F(DndTestUser, 03_AlterUser) { - SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); - strcpy(pReq->user, "u1"); - strcpy(pReq->pass, "p2"); + { + SAlterUserMsg* pReq = (SAlterUserMsg*)rpcMallocCont(sizeof(SAlterUserMsg)); + strcpy(pReq->user, "u1"); + strcpy(pReq->pass, "p2"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SAlterUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_USER; + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); SendThenCheckShowRetrieveMsg(3); CheckBinary("u1", TSDB_USER_LEN); @@ -262,22 +257,21 @@ TEST_F(DndTestUser, 03_AlterUser) { CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} -TEST_F(DndTestUser, 04_DropUser) { - SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); - strcpy(pReq->user, "u1"); + { + SDropUserMsg* pReq = (SDropUserMsg*)rpcMallocCont(sizeof(SDropUserMsg)); + strcpy(pReq->user, "u1"); - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropUserMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropUserMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_USER; + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } SendTheCheckShowMetaMsg(TSDB_MGMT_TABLE_USER, "show users", 4); SendThenCheckShowRetrieveMsg(2); CheckBinary("root", TSDB_USER_LEN); @@ -288,9 +282,8 @@ TEST_F(DndTestUser, 04_DropUser) { CheckTimestamp(); CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); -} -TEST_F(DndTestUser, 05_RestartDnode) { + // restart stopServer(pServer); pServer = NULL; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 115d896da2..74138500bd 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -42,13 +42,6 @@ extern int32_t mDebugFlag; #define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} #define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", mDebugFlag, __VA_ARGS__); }} -typedef struct SClusterObj SClusterObj; -typedef struct SMnodeObj SMnodeObj; -typedef struct SAcctObj SAcctObj; -typedef struct SVgObj SVgObj; -typedef struct SFuncObj SFuncObj; -typedef struct SOperObj SOperObj; - typedef enum { MND_AUTH_ACCT_START = 0, MND_AUTH_ACCT_USER, @@ -99,7 +92,7 @@ typedef enum { DND_REASON_OTHERS } EDndReason; -typedef struct STrans { +typedef struct { int32_t id; ETrnStage stage; ETrnPolicy policy; @@ -111,7 +104,7 @@ typedef struct STrans { SArray *undoActions; } STrans; -typedef struct SClusterObj { +typedef struct { int32_t id; char name[TSDB_CLUSTER_ID_LEN]; int64_t createdTime; @@ -139,7 +132,7 @@ typedef struct { char ep[TSDB_EP_LEN]; } SDnodeObj; -typedef struct SMnodeObj { +typedef struct { int32_t id; int64_t createdTime; int64_t updateTime; @@ -167,7 +160,7 @@ typedef struct { int64_t compStorage; // Compressed storage on disk } SAcctInfo; -typedef struct SAcctObj { +typedef struct { char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; @@ -195,8 +188,8 @@ typedef struct { int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; + int32_t minRows; + int32_t maxRows; int32_t commitTime; int32_t fsyncPeriod; int8_t walLevel; @@ -214,7 +207,10 @@ typedef struct { int64_t createdTime; int64_t updateTime; int64_t uid; - int32_t version; + int32_t cfgVersion; + int32_t vgVersion; + int32_t numOfVgroups; + int8_t hashMethod; // default is 1 SDbCfg cfg; } SDbObj; @@ -223,12 +219,15 @@ typedef struct { ESyncState role; } SVnodeGid; -typedef struct SVgObj { +typedef struct { int32_t vgId; int64_t createdTime; int64_t updateTime; int32_t version; + uint32_t hashBegin; + uint32_t hashEnd; char dbName[TSDB_FULL_DB_NAME_LEN]; + int64_t dbUid; int32_t numOfTables; int32_t numOfTimeSeries; int64_t totalStorage; @@ -245,6 +244,7 @@ typedef struct { int64_t createdTime; int64_t updateTime; uint64_t uid; + uint64_t dbUid; int32_t version; int32_t numOfColumns; int32_t numOfTags; @@ -252,7 +252,7 @@ typedef struct { SSchema *pSchema; } SStbObj; -typedef struct SFuncObj { +typedef struct { char name[TSDB_FUNC_NAME_LEN]; int64_t createdTime; int8_t funcType; diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index 04ceb2820a..764dfbffc1 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -28,6 +28,7 @@ SDnodeObj *mndAcquireDnode(SMnode *pMnode, int32_t dnodeId); void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); +bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 21aee0f0a4..696f798c9a 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -22,10 +22,13 @@ extern "C" { #endif -int32_t mndInitVgroup(SMnode *pMnode); -void mndCleanupVgroup(SMnode *pMnode); -SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); -void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); +int32_t mndInitVgroup(SMnode *pMnode); +void mndCleanupVgroup(SMnode *pMnode); +SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId); +void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup); +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); +SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); +SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 0eaa184907..7c53a4ebd0 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -143,22 +143,22 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); strcpy(pMeta->tbFname, mndShowStr(pShow->type)); pShow->numOfColumns = cols; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9f2b2d17f3..258a96affe 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -20,6 +20,7 @@ #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" +#include "mndVgroup.h" #define TSDB_DB_VER_NUM 1 #define TSDB_DB_RESERVE_SIZE 64 @@ -74,15 +75,18 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT64(pRaw, dataPos, pDb->createdTime) SDB_SET_INT64(pRaw, dataPos, pDb->updateTime) SDB_SET_INT64(pRaw, dataPos, pDb->uid) - SDB_SET_INT32(pRaw, dataPos, pDb->version) + SDB_SET_INT32(pRaw, dataPos, pDb->cfgVersion) + SDB_SET_INT32(pRaw, dataPos, pDb->vgVersion) + SDB_SET_INT32(pRaw, dataPos, pDb->numOfVgroups) + SDB_SET_INT8(pRaw, dataPos, pDb->hashMethod) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.cacheBlockSize) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.totalBlocks) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysPerFile) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep0) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep1) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.daysToKeep2) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRowsPerFileBlock) - SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRowsPerFileBlock) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.minRows) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.maxRows) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.commitTime) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.fsyncPeriod) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.walLevel) @@ -118,15 +122,18 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->uid) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->version) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfgVersion) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->vgVersion) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->numOfVgroups) + SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->hashMethod) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.cacheBlockSize) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.totalBlocks) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysPerFile) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep0) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep1) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.daysToKeep2) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRowsPerFileBlock) - SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRowsPerFileBlock) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.minRows) + SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.maxRows) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.commitTime) SDB_GET_INT32(pRaw, pRow, dataPos, &pDb->cfg.fsyncPeriod) SDB_GET_INT8(pRaw, pRow, dataPos, &pDb->cfg.walLevel) @@ -154,6 +161,9 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); pOldDb->updateTime = pNewDb->createdTime; + pOldDb->cfgVersion = pNewDb->cfgVersion; + pOldDb->vgVersion = pNewDb->vgVersion; + pOldDb->numOfVgroups = pNewDb->numOfVgroups; memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); return 0; } @@ -184,133 +194,30 @@ static int32_t mndCheckDbName(char *dbName, SUserObj *pUser) { return 0; } -static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg, char *errMsg, int32_t len) { - if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database cache block size option", len); - return -1; - } - - if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database total blocks option", len); - return -1; - } - - if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database days option", len); - return -1; - } - - if (pCfg->daysToKeep0 < pCfg->daysPerFile) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database days option", len); - return -1; - } - - if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP || pCfg->daysToKeep0 > pCfg->daysToKeep1) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database keep0 option", len); - return -1; - } - - if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database keep1 option", len); - return -1; - } - - if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database keep2 option", len); - return -1; - } - - if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database minrows option", len); - return -1; - } - - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database maxrows option", len); - return -1; - } - - if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database minrows option", len); - return -1; - } - - if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database commit option", len); - return -1; - } - - if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database fsync option", len); - return -1; - } - - if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database wal level option", len); - return -1; - } - - if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid precision option", len); - return -1; - } - - if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database compression option", len); - return -1; - } - - if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database replication option", len); - return -1; - } - - if (pCfg->replications > mndGetDnodeSize(pMnode)) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database replication option", len); - return -1; - } - - if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database quorum option", len); - return -1; - } - - if (pCfg->quorum > pCfg->replications) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database quorum option", len); - return -1; - } - - if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database update option", len); - return -1; - } - - if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) { - terrno = TSDB_CODE_MND_INVALID_DB_OPTION; - tstrncpy(errMsg, "Invalid database cachelast option", len); - return -1; - } - +static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { + if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1; + if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1; + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; + if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; + if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1; + if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1; + if (pCfg->minRows < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRows > TSDB_MAX_MIN_ROW_FBLOCK) return -1; + if (pCfg->maxRows < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRows > TSDB_MAX_MAX_ROW_FBLOCK) return -1; + if (pCfg->minRows > pCfg->maxRows) return -1; + if (pCfg->commitTime < TSDB_MIN_COMMIT_TIME || pCfg->commitTime > TSDB_MAX_COMMIT_TIME) return -1; + if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; + if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; + if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; + if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; + if (pCfg->replications < TSDB_MIN_DB_REPLICA_OPTION || pCfg->replications > TSDB_MAX_DB_REPLICA_OPTION) return -1; + if (pCfg->replications > mndGetDnodeSize(pMnode)) return -1; + if (pCfg->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCfg->quorum > TSDB_MAX_DB_QUORUM_OPTION) return -1; + if (pCfg->quorum > pCfg->replications) return -1; + if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) return -1; + if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; return TSDB_CODE_SUCCESS; } @@ -321,8 +228,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->daysToKeep0 < 0) pCfg->daysToKeep0 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep1 < 0) pCfg->daysToKeep1 = TSDB_DEFAULT_KEEP; if (pCfg->daysToKeep2 < 0) pCfg->daysToKeep2 = TSDB_DEFAULT_KEEP; - if (pCfg->minRowsPerFileBlock < 0) pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; - if (pCfg->maxRowsPerFileBlock < 0) pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MIN_ROW_FBLOCK; + if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAX_ROW_FBLOCK; if (pCfg->commitTime < 0) pCfg->commitTime = TSDB_DEFAULT_COMMIT_TIME; if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; @@ -334,6 +241,56 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; } +static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + SSdbRaw *pDbRaw = mndDbActionEncode(pDb); + if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; + sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING); + + for (int v = 0; v < pDb->numOfVgroups; ++v) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); + if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; + sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING); + } + + return 0; +} + +static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + SSdbRaw *pDbRaw = mndDbActionEncode(pDb); + if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; + sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED); + + for (int v = 0; v < pDb->numOfVgroups; ++v) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); + if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; + sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); + } + + return 0; +} + +static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + SSdbRaw *pDbRaw = mndDbActionEncode(pDb); + if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; + sdbSetRawStatus(pDbRaw, SDB_STATUS_READY); + + for (int v = 0; v < pDb->numOfVgroups; ++v) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); + if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + } + + return 0; +} + +static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + return 0; +} + +static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + return 0; +} + static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); @@ -341,14 +298,18 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); + dbObj.numOfVgroups = pCreate->numOfVgroups; + dbObj.hashMethod = 1; + dbObj.cfgVersion = 1; + dbObj.vgVersion = 1; dbObj.cfg = (SDbCfg){.cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, .daysPerFile = pCreate->daysPerFile, .daysToKeep0 = pCreate->daysToKeep0, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRowsPerFileBlock = pCreate->minRowsPerFileBlock, - .maxRowsPerFileBlock = pCreate->maxRowsPerFileBlock, + .minRows = pCreate->minRowsPerFileBlock, + .maxRows = pCreate->maxRowsPerFileBlock, .fsyncPeriod = pCreate->fsyncPeriod, .commitTime = pCreate->commitTime, .precision = pCreate->precision, @@ -366,57 +327,69 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat return -1; } - char errMsg[TSDB_ERROR_MSG_LEN] = {0}; - if (mndCheckDbCfg(pMnode, &dbObj.cfg, errMsg, TSDB_ERROR_MSG_LEN) != 0) { + if (mndCheckDbCfg(pMnode, &dbObj.cfg) != 0) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; mError("db:%s, failed to create since %s", pCreate->db, terrstr()); return -1; } + SVgObj *pVgroups = NULL; + if (mndAllocVgroup(pMnode, &dbObj, &pVgroups) != 0) { + mError("db:%s, failed to create since %s", pCreate->db, terrstr()); + return -1; + } + + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); - SSdbRaw *pRedoRaw = mndDbActionEncode(&dbObj); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - SSdbRaw *pUndoRaw = mndDbActionEncode(&dbObj); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); - SSdbRaw *pCommitRaw = mndDbActionEncode(&dbObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; + } + + if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; + } + + if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto CREATE_DB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto CREATE_DB_OVER; } + code = 0; + +CREATE_DB_OVER: + free(pVgroups); mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCreateDbMsg *pCreate = pMsg->rpcMsg.pCont; + pCreate->numOfVgroups = htonl(pCreate->numOfVgroups); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); @@ -573,7 +546,8 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return code; } - dbObj.version++; + dbObj.cfgVersion++; + dbObj.updateTime = taosGetTimestampMs(); code = mndUpdateDb(pMnode, pMsg, pDb, &dbObj); mndReleaseDb(pMnode, pDb); @@ -658,17 +632,69 @@ static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; + SSdb *pSdb = pMnode->pSdb; SUseDbMsg *pUse = pMsg->rpcMsg.pCont; + pUse->vgVersion = htonl(pUse->vgVersion); - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); - if (pDb != NULL) { - strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); - mndReleaseDb(pMnode, pDb); - return 0; - } else { - mError("db:%s, failed to process use db msg since %s", pMsg->db, terrstr()); + SDbObj *pDb = mndAcquireDb(pMnode, pUse->db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + mError("db:%s, failed to process use db msg since %s", pUse->db, terrstr()); return -1; } + + int32_t contLen = sizeof(SUseDbRsp) + pDb->numOfVgroups * sizeof(SVgroupInfo); + SUseDbRsp *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t vindex = 0; + + if (pUse->vgVersion < pDb->vgVersion) { + void *pIter = NULL; + while (vindex < pDb->numOfVgroups) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid == pDb->uid) { + SVgroupInfo *pInfo = &pRsp->vgroupInfo[vindex]; + pInfo->vgId = htonl(pVgroup->vgId); + pInfo->hashBegin = htonl(pVgroup->hashBegin); + pInfo->hashEnd = htonl(pVgroup->hashEnd); + pInfo->numOfEps = pVgroup->replica; + for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; + SEpAddrMsg *pEpArrr = &pInfo->epAddr[gid]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode != NULL) { + memcpy(pEpArrr->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + pEpArrr->port = htons(pDnode->port); + } + mndReleaseDnode(pMnode, pDnode); + if (pVgid->role == TAOS_SYNC_STATE_LEADER) { + pInfo->inUse = gid; + } + } + vindex++; + } + + sdbRelease(pSdb, pVgroup); + } + } + + memcpy(pRsp->db, pDb->name, TSDB_FULL_DB_NAME_LEN); + pRsp->vgVersion = htonl(pDb->vgVersion); + pRsp->vgNum = htonl(vindex); + pRsp->hashMethod = pDb->hashMethod; + + pMsg->pCont = pRsp; + pMsg->contLen = contLen; + mndReleaseDb(pMnode, pDb); + + return 0; } static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { @@ -709,100 +735,106 @@ static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMe pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "vgroups"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "replica"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "quorum"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "days"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "keep0,keep1,keep2"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cache(MB)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "cache"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "blocks"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "minrows"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "maxrows"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "wallevel"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "fsync"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "comp"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "cachelast"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "precision"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "update"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -855,6 +887,10 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 *(int64_t *)pWrite = pDb->createdTime; cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDb->numOfVgroups; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; *(int16_t *)pWrite = pDb->cfg.replications; cols++; @@ -886,11 +922,11 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int3 cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.minRowsPerFileBlock; + *(int32_t *)pWrite = pDb->cfg.minRows; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDb->cfg.maxRowsPerFileBlock; + *(int32_t *)pWrite = pDb->cfg.maxRows; cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ec5f68a713..55e8b3a721 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -218,6 +218,15 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { return sdbGetSize(pSdb, SDB_DNODE); } +bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { + int64_t ms = taosGetTimestampMs(); + int64_t interval = ABS(pDnode->lastAccessTime - ms); + if (interval > 3000 * pMnode->cfg.statusInterval) { + return false; + } + return true; +} + static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t numOfEps) { SSdb *pSdb = pMnode->pSdb; @@ -552,16 +561,16 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -629,46 +638,46 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "max vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "max_vnodes"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "offline reason"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "offline_reason"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b7158bb094..a513c9d57d 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -380,46 +380,46 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "comment"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "aggregate"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "outputtype"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "code_len"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "bufsize"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 7b6804d43e..16733cbf94 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -368,34 +368,34 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "endpoint"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "role"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "role time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "role_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 45a63f2dc5..bf657fd27a 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -474,48 +474,48 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; // app name pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "program"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; // app pid pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "login_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "last_access"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -602,88 +602,88 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "queryId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 24; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "qid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "created_time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql_obj_id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ep"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_BOOL; strcpy(pSchema[cols].name, "stable_query"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "sub_queries"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sub_query_info"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -818,64 +818,64 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "streamId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "dest table"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "destination"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "created time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "exec time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "exec"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "time(us)"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "cycles"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 791b6f5d12..3330c86f92 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -74,10 +74,12 @@ static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { if (pRaw == NULL) return NULL; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN) + SDB_SET_BINARY(pRaw, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN) SDB_SET_INT64(pRaw, dataPos, pStb->createdTime) SDB_SET_INT64(pRaw, dataPos, pStb->updateTime) SDB_SET_INT64(pRaw, dataPos, pStb->uid) + SDB_SET_INT64(pRaw, dataPos, pStb->dbUid) SDB_SET_INT64(pRaw, dataPos, pStb->version) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags) @@ -113,10 +115,12 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { if (pStb == NULL) return NULL; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_NAME_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN) + SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid) + SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->dbUid) SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->version) SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfColumns) SDB_GET_INT32(pRaw, pRow, dataPos, &pStb->numOfTags) @@ -570,28 +574,28 @@ static int32_t mndGetStbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pM pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; - strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + strcpy(pSchema[cols].name, "create_time"); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "tags"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c deleted file mode 100644 index 59161b32f2..0000000000 --- a/source/dnode/mnode/impl/src/mndSync.c +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" -#include "mndTrans.h" - -int32_t mndInitSync(SMnode *pMnode) { return 0; } -void mndCleanupSync(SMnode *pMnode) {} - -int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) { - int32_t code = 0; - - // int32_t len = sdbGetRawTotalSize(pRaw); - // SSdbRaw *pReceived = calloc(1, len); - // memcpy(pReceived, pRaw, len); - // mDebug("trans:%d, data:%p recv from sync, code:0x%x pMsg:%p", pMsg->id, pReceived, code & 0xFFFF, pMsg); - - // mndTransApply(pMnode, pReceived, code); - return code; -} - -bool mndIsMaster(SMnode *pMnode) { return true; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 7219342d59..c97e1ff7d6 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -410,28 +410,28 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "privilege"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create time"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "account"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 9a462c024b..b880434bf6 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -24,11 +24,9 @@ #define TSDB_VGROUP_VER_NUM 1 #define TSDB_VGROUP_RESERVE_SIZE 64 -static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup); -static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); -static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); -static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup); +static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); +static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup); static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg); static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg); @@ -70,8 +68,8 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} -static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); +SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -79,7 +77,10 @@ static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime) SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime) SDB_SET_INT32(pRaw, dataPos, pVgroup->version) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd) SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid) SDB_SET_INT8(pRaw, dataPos, pVgroup->replica) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; @@ -92,7 +93,7 @@ static SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { return pRaw; } -static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { +SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; @@ -102,7 +103,7 @@ static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { return NULL; } - SSdbRow *pRow = sdbAllocRow(sizeof(SDbObj)); + SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); SVgObj *pVgroup = sdbGetRowObj(pRow); if (pVgroup == NULL) return NULL; @@ -111,7 +112,10 @@ static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->updateTime) SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->version) + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashBegin) + SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashEnd) SDB_GET_BINARY(pRaw, pRow, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN) + SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->dbUid) SDB_GET_INT8(pRaw, pRow, dataPos, &pVgroup->replica) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; @@ -133,12 +137,6 @@ static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup) { return 0; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } - static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNewVgroup) { mTrace("vgId:%d, perform update action", pOldVgroup->vgId); pOldVgroup->updateTime = pNewVgroup->updateTime; @@ -158,6 +156,105 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } +static int32_t mndGetDefaultVgroupSize(SMnode *pMnode) { + // todo + return 2; +} + +static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup) { + SSdb *pSdb = pMnode->pSdb; + int32_t allocedVnodes = 0; + void *pIter = NULL; + + while (allocedVnodes < pVgroup->replica) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + // todo + if (mndIsDnodeInReadyStatus(pMnode, pDnode)) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[allocedVnodes]; + pVgid->dnodeId = pDnode->id; + if (pVgroup->replica == 1) { + pVgid->role = TAOS_SYNC_STATE_LEADER; + } else { + pVgid->role = TAOS_SYNC_STATE_FOLLOWER; + } + allocedVnodes++; + } + sdbRelease(pSdb, pDnode); + } + + if (allocedVnodes != pVgroup->replica) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + return 0; +} + +int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { + if (pDb->numOfVgroups != -1 && + (pDb->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pDb->numOfVgroups > TSDB_MAX_VNODES_PER_DB)) { + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; + return -1; + } + + if (pDb->numOfVgroups == -1) { + pDb->numOfVgroups = mndGetDefaultVgroupSize(pMnode); + if (pDb->numOfVgroups < 0) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + } + + SVgObj *pVgroups = calloc(pDb->numOfVgroups, sizeof(SVgObj)); + if (pVgroups == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t alloceVgroups = 0; + int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); + uint32_t hashMin = 0; + uint32_t hashMax = UINT32_MAX; + uint32_t hashInterval = (hashMax - hashMin) / pDb->numOfVgroups; + + for (uint32_t v = 0; v < pDb->numOfVgroups; v++) { + SVgObj *pVgroup = &pVgroups[v]; + pVgroup->vgId = maxVgId++; + pVgroup->createdTime = taosGetTimestampMs(); + pVgroup->updateTime = pVgroups->createdTime; + pVgroup->version = 1; + pVgroup->dbUid = pDb->uid; + pVgroup->hashBegin = hashMin + hashInterval * v; + if (v == pDb->numOfVgroups - 1) { + pVgroup->hashEnd = hashMax; + } else { + pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1; + } + + memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN); + pVgroup->replica = pDb->cfg.replications; + + if (mndGetAvailableDnode(pMnode, pVgroup) != 0) { + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + free(pVgroups); + return -1; + } + + alloceVgroups++; + } + + *ppVgroups = pVgroups; + return 0; +} + +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } + static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { SSdb *pSdb = pMnode->pSdb; @@ -203,30 +300,30 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "tables"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; for (int32_t i = 0; i < pShow->replica; ++i) { - pShow->bytes[cols] = 4; + pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; } - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -273,10 +370,6 @@ static int32_t mndRetrieveVgroups(SMnodeMsg *pMsg, SShowObj *pShow, char *data, cols++; } - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int8_t *)pWrite = pVgroup->compact; - cols++; - sdbRelease(pSdb, pVgroup); numOfRows++; } @@ -309,16 +402,16 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); + pSchema[cols].bytes = htonl(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/vnode/impl/inc/vnodeStateMgr.h b/source/dnode/vnode/impl/inc/vnodeStateMgr.h index 788426e25e..5862b304ed 100644 --- a/source/dnode/vnode/impl/inc/vnodeStateMgr.h +++ b/source/dnode/vnode/impl/inc/vnodeStateMgr.h @@ -21,13 +21,13 @@ extern "C" { #endif typedef struct { - uint64_t processed; - uint64_t committed; - uint64_t applied; + int64_t processed; + int64_t committed; + int64_t applied; } SVState; #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_STATE_MGR_H_*/ \ No newline at end of file +#endif /*_TD_VNODE_STATE_MGR_H_*/ diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index 85460e8d91..85e044266a 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -15,16 +15,31 @@ #include "vnodeDef.h" +int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { + SVnodeReq *pVnodeReq; + + switch (pMsg->msgType) { + case TSDB_MSG_TYPE_MQ_SET: + if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { + // TODO: handle error + } + break; + } + + void *pBuf = pMsg->pCont; + return 0; +} + int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { - SRpcMsg * pMsg; + SRpcMsg *pMsg; SVnodeReq *pVnodeReq; for (int i = 0; i < taosArrayGetSize(pMsgs); i++) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void * pBuf = pMsg->pCont; - uint64_t ver = pVnode->state.processed++; + void *pBuf = pMsg->pCont; + int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { @@ -99,4 +114,4 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return 0; } -/* ------------------------ STATIC METHODS ------------------------ */ \ No newline at end of file +/* ------------------------ STATIC METHODS ------------------------ */ diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/tq/inc/tqInt.h index 100149c0ea..022b599816 100644 --- a/source/dnode/vnode/tq/inc/tqInt.h +++ b/source/dnode/vnode/tq/inc/tqInt.h @@ -22,16 +22,16 @@ extern "C" { #endif -//create persistent storage for meta info such as consuming offset -//return value > 0: cgId -//return value <= 0: error code -//int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); -//create ring buffer in memory and load consuming offset -//int tqOpenTCGroup(STQ*, const char* topic, int cgId); -//destroy ring buffer and persist consuming offset -//int tqCloseTCGroup(STQ*, const char* topic, int cgId); -//delete persistent storage for meta info -//int tqDropTCGroup(STQ*, const char* topic, int cgId); +// create persistent storage for meta info such as consuming offset +// return value > 0: cgId +// return value <= 0: error code +// int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); +// create ring buffer in memory and load consuming offset +// int tqOpenTCGroup(STQ*, const char* topic, int cgId); +// destroy ring buffer and persist consuming offset +// int tqCloseTCGroup(STQ*, const char* topic, int cgId); +// delete persistent storage for meta info +// int tqDropTCGroup(STQ*, const char* topic, int cgId); #ifdef __cplusplus } diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/tq/inc/tqMetaStore.h index 4190125e44..5bcedaed74 100644 --- a/source/dnode/vnode/tq/inc/tqMetaStore.h +++ b/source/dnode/vnode/tq/inc/tqMetaStore.h @@ -23,27 +23,22 @@ extern "C" { #endif - -STqMetaStore* tqStoreOpen(const char* path, - TqSerializeFun pSerializer, - TqDeserializeFun pDeserializer, - TqDeleteFun pDeleter, - int32_t tqConfigFlag - ); +STqMetaStore* tqStoreOpen(const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, FTqDelete pDeleter, + int32_t tqConfigFlag); int32_t tqStoreClose(STqMetaStore*); -//int32_t tqStoreDelete(TqMetaStore*); -//int32_t tqStoreCommitAll(TqMetaStore*); -int32_t tqStorePersist(STqMetaStore*); -//clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore*); +// int32_t tqStoreDelete(TqMetaStore*); +// int32_t tqStoreCommitAll(TqMetaStore*); +int32_t tqStorePersist(STqMetaStore*); +// clean deleted idx and data from persistent file +int32_t tqStoreCompact(STqMetaStore*); -void* tqHandleGet(STqMetaStore*, int64_t key); -//make it unpersist +void* tqHandleGet(STqMetaStore*, int64_t key); +// make it unpersist void* tqHandleTouchGet(STqMetaStore*, int64_t key); int32_t tqHandleMovePut(STqMetaStore*, int64_t key, void* value); int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); -//delete committed kv pair -//notice that a delete action still needs to be committed +// delete committed kv pair +// notice that a delete action still needs to be committed int32_t tqHandleDel(STqMetaStore*, int64_t key); int32_t tqHandleCommit(STqMetaStore*, int64_t key); int32_t tqHandleAbort(STqMetaStore*, int64_t key); diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 4aabf07c2d..b88ef353b0 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -16,51 +16,48 @@ #include "tqInt.h" #include "tqMetaStore.h" -//static -//read next version data +// static +// read next version data // -//send to fetch queue +// send to fetch queue // -//handle management message +// handle management message // -int tqGetgHandleSSize(const TqGroupHandle *gHandle); +int tqGetgHandleSSize(const STqGroupHandle* gHandle); int tqBufHandleSSize(); int tqBufItemSSize(); -TqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - TqGroupHandle* gHandle; +STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + STqGroupHandle* gHandle; return NULL; } -void* tqSerializeListHandle(TqListHandle* listHandle, void* ptr); -void* tqSerializeBufHandle(TqBufferHandle* bufHandle, void* ptr); -void* tqSerializeBufItem(TqBufferItem* bufItem, void* ptr); +void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr); +void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr); +void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr); -const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle* bufHandle); -const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem* bufItem); +const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle); +const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem); -STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac) { +STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); - if(pTq == NULL) { - //TODO: memory error + if (pTq == NULL) { + // TODO: memory error return NULL; } pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; - pTq->tqMemRef.pAlloctorFactory = allocFac; - // pTq->tqMemRef.pAllocator = allocFac->create(allocFac); - if(pTq->tqMemRef.pAllocator == NULL) { - //TODO + pTq->tqMemRef.pAlloctorFactory = allocFac; + pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + if (pTq->tqMemRef.pAllocator == NULL) { + // TODO } - pTq->tqMeta = tqStoreOpen(path, - (TqSerializeFun)tqSerializeGroupHandle, - (TqDeserializeFun)tqDeserializeGroupHandle, - free, - 0); - if(pTq->tqMeta == NULL) { - //TODO: free STQ + pTq->tqMeta = + tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0); + if (pTq->tqMeta == NULL) { + // TODO: free STQ return NULL; } return pTq; @@ -74,21 +71,21 @@ static int tqProtoCheck(TmqMsgHead *pMsg) { return pMsg->protoVer == 0; } -static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** ppQuery) { - //clean old item and move forward +static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) { + // clean old item and move forward int32_t consumeOffset = pAck->consumeOffset; - int idx = consumeOffset % TQ_BUFFER_SIZE; + int idx = consumeOffset % TQ_BUFFER_SIZE; ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); tfree(bHandle->buffer[idx].content); - if( 1 /* TODO: need to launch new query */) { - TqQueryMsg* pNewQuery = malloc(sizeof(TqQueryMsg)); - if(pNewQuery == NULL) { - //TODO: memory insufficient + if (1 /* TODO: need to launch new query */) { + STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); + if (pNewQuery == NULL) { + // TODO: memory insufficient return -1; } - //TODO: lock executor + // TODO: lock executor pNewQuery->exec->executor = bHandle->buffer[idx].executor; - //TODO: read from wal and assign to src + // TODO: read from wal and assign to src pNewQuery->exec->src = 0; pNewQuery->exec->dest = &bHandle->buffer[idx]; pNewQuery->next = *ppQuery; @@ -97,98 +94,94 @@ static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** return 0; } -static int tqAck(TqGroupHandle* gHandle, TmqAcks* pAcks) { - int32_t ackNum = pAcks->ackNum; - TmqOneAck *acks = pAcks->acks; - //double ptr for acks and list - int i = 0; - TqListHandle* node = gHandle->head; - int ackCnt = 0; - TqQueryMsg *pQuery = NULL; - while(i < ackNum && node->next) { - if(acks[i].topicId == node->next->bufHandle.topicId) { +static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { + int32_t ackNum = pAcks->ackNum; + TmqOneAck* acks = pAcks->acks; + // double ptr for acks and list + int i = 0; + STqListHandle* node = gHandle->head; + int ackCnt = 0; + STqQueryMsg* pQuery = NULL; + while (i < ackNum && node->next) { + if (acks[i].topicId == node->next->bufHandle.topicId) { ackCnt++; tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); - } else if(acks[i].topicId < node->next->bufHandle.topicId) { + } else if (acks[i].topicId < node->next->bufHandle.topicId) { i++; } else { node = node->next; } } - if(pQuery) { - //post message + if (pQuery) { + // post message } return ackCnt; } -static int tqCommitTCGroup(TqGroupHandle* handle) { - //persist modification into disk +static int tqCommitTCGroup(STqGroupHandle* handle) { + // persist modification into disk return 0; } -int tqCreateTCGroup(STQ *pTq, int64_t topicId, int64_t cgId, int64_t cId, TqGroupHandle** handle) { - //create in disk - TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle)); - if(gHandle == NULL) { - //TODO +int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) { + // create in disk + STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle)); + if (gHandle == NULL) { + // TODO return -1; } - memset(gHandle, 0, sizeof(TqGroupHandle)); + memset(gHandle, 0, sizeof(STqGroupHandle)); return 0; } -TqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - TqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); - if(gHandle == NULL) { +STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); + if (gHandle == NULL) { int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return NULL; } } - //create - //open + // create + // open return gHandle; } -int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - return 0; -} +int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; } int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - //delete from disk + // delete from disk return 0; } -static int tqFetch(TqGroupHandle* gHandle, void** msg) { - TqListHandle* head = gHandle->head; - TqListHandle* node = head; - int totSize = 0; - //TODO: make it a macro - int sizeLimit = 4 * 1024; +static int tqFetch(STqGroupHandle* gHandle, void** msg) { + STqListHandle* head = gHandle->head; + STqListHandle* node = head; + int totSize = 0; + // TODO: make it a macro + int sizeLimit = 4 * 1024; TmqMsgContent* buffer = malloc(sizeLimit); - if(buffer == NULL) { - //TODO:memory insufficient + if (buffer == NULL) { + // TODO:memory insufficient return -1; } - //iterate the list to get msgs of all topics - //until all topic iterated or msgs over sizeLimit - while(node->next) { + // iterate the list to get msgs of all topics + // until all topic iterated or msgs over sizeLimit + while (node->next) { node = node->next; - TqBufferHandle* bufHandle = &node->bufHandle; - int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; - if(bufHandle->buffer[idx].content != NULL && - bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset - ) { + STqBufferHandle* bufHandle = &node->bufHandle; + int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; + if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) { totSize += bufHandle->buffer[idx].size; - if(totSize > sizeLimit) { - void *ptr = realloc(buffer, totSize); - if(ptr == NULL) { + if (totSize > sizeLimit) { + void* ptr = realloc(buffer, totSize); + if (ptr == NULL) { totSize -= bufHandle->buffer[idx].size; - //TODO:memory insufficient - //return msgs already copied + // TODO:memory insufficient + // return msgs already copied break; } } @@ -198,7 +191,7 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) { buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); - if(totSize > sizeLimit) { + if (totSize > sizeLimit) { break; } } @@ -206,104 +199,102 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) { return totSize; } -TqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { - return NULL; -} +STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; } -int tqLaunchQuery(TqGroupHandle* gHandle) { - return 0; -} +int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; } -int tqSendLaunchQuery(TqGroupHandle* gHandle) { - return 0; -} +int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; } /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ - /*return 0;*/ +/*return 0;*/ /*}*/ -int tqPushMsg(STQ* pTq , void* p, int64_t version) { - //add reference - //judge and launch new query +int tqPushMsg(STQ* pTq, void* p, int64_t version) { + // add reference + // judge and launch new query return 0; } int tqCommit(STQ* pTq) { - //do nothing + // do nothing return 0; } -int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) { - if(!tqProtoCheck((TmqMsgHead *)pMsg)) { - //proto version invalid +int tqSetCursor(STQ* pTq, void* msg) { + return 0; +} + +int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { + if (!tqProtoCheck((TmqMsgHead*)pMsg)) { + // proto version invalid return -1; } - int64_t clientId = pMsg->head.clientId; - TqGroupHandle *gHandle = tqGetGroupHandle(pTq, clientId); - if(gHandle == NULL) { - //client not connect + int64_t clientId = pMsg->head.clientId; + STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId); + if (gHandle == NULL) { + // client not connect return -1; } - if(pMsg->acks.ackNum != 0) { - if(tqAck(gHandle, &pMsg->acks) != 0) { - //ack not success + if (pMsg->acks.ackNum != 0) { + if (tqAck(gHandle, &pMsg->acks) != 0) { + // ack not success return -1; } } - TmqConsumeRsp *pRsp = (TmqConsumeRsp*) pMsg; + STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - if(tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { - //fetch error + if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { + // fetch error return -1; } - //judge and launch new query - if(tqLaunchQuery(gHandle)) { - //launch query error + // judge and launch new query + if (tqLaunchQuery(gHandle)) { + // launch query error return -1; } return 0; } -int tqSerializeGroupHandle(const TqGroupHandle *gHandle, TqSerializedHead** ppHead) { - //calculate size - int sz = tqGetgHandleSSize(gHandle) + sizeof(TqSerializedHead); - if(sz > (*ppHead)->ssize) { +int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) { + // calculate size + int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead); + if (sz > (*ppHead)->ssize) { void* tmpPtr = realloc(*ppHead, sz); - if(tmpPtr == NULL) { + if (tmpPtr == NULL) { free(*ppHead); - //TODO: memory err + // TODO: memory err return -1; } *ppHead = tmpPtr; (*ppHead)->ssize = sz; } void* ptr = (*ppHead)->content; - //do serialization + // do serialization *(int64_t*)ptr = gHandle->cId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); *(int64_t*)ptr = gHandle->cgId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); *(int32_t*)ptr = gHandle->topicNum; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if(gHandle->topicNum > 0) { + if (gHandle->topicNum > 0) { tqSerializeListHandle(gHandle->head, ptr); } return 0; } -void* tqSerializeListHandle(TqListHandle *listHandle, void* ptr) { - TqListHandle *node = listHandle; +void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) { + STqListHandle* node = listHandle; ASSERT(node != NULL); - while(node) { + while (node) { ptr = tqSerializeBufHandle(&node->bufHandle, ptr); node = node->next; } return ptr; } -void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) { +void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) { *(int64_t*)ptr = bufHandle->nextConsumeOffset; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); *(int64_t*)ptr = bufHandle->topicId; @@ -312,21 +303,21 @@ void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) { ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); *(int32_t*)ptr = bufHandle->tail; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - for(int i = 0; i < TQ_BUFFER_SIZE; i++) { + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); } return ptr; } -void* tqSerializeBufItem(TqBufferItem *bufItem, void* ptr) { - //TODO: do we need serialize this? - //mainly for executor +void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) { + // TODO: do we need serialize this? + // mainly for executor return ptr; } -const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle **ppGHandle) { - TqGroupHandle *gHandle = *ppGHandle; - const void* ptr = pHead->content; +const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) { + STqGroupHandle* gHandle = *ppGHandle; + const void* ptr = pHead->content; gHandle->cId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); gHandle->cgId = *(int64_t*)ptr; @@ -335,20 +326,20 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl gHandle->topicNum = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); gHandle->head = NULL; - TqListHandle *node = gHandle->head; - for(int i = 0; i < gHandle->topicNum; i++) { - if(gHandle->head == NULL) { - if((node = malloc(sizeof(TqListHandle))) == NULL) { - //TODO: error + STqListHandle* node = gHandle->head; + for (int i = 0; i < gHandle->topicNum; i++) { + if (gHandle->head == NULL) { + if ((node = malloc(sizeof(STqListHandle))) == NULL) { + // TODO: error return NULL; } - node->next= NULL; - ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); + node->next = NULL; + ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); gHandle->head = node; } else { - node->next = malloc(sizeof(TqListHandle)); - if(node->next == NULL) { - //TODO: error + node->next = malloc(sizeof(STqListHandle)); + if (node->next == NULL) { + // TODO: error return NULL; } node->next->next = NULL; @@ -359,7 +350,7 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl return ptr; } -const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle) { +const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) { const void* ptr = pBytes; bufHandle->nextConsumeOffset = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); @@ -369,32 +360,30 @@ const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); bufHandle->tail = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - for(int i = 0; i < TQ_BUFFER_SIZE; i++) { + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); } return ptr; } -const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem *bufItem) { - return pBytes; +const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; } + +// TODO: make this a macro +int tqGetgHandleSSize(const STqGroupHandle* gHandle) { + return sizeof(int64_t) * 2 // cId + cgId + + sizeof(int32_t) // topicNum + + gHandle->topicNum * tqBufHandleSSize(); } -//TODO: make this a macro -int tqGetgHandleSSize(const TqGroupHandle *gHandle) { - return sizeof(int64_t) * 2 //cId + cgId - + sizeof(int32_t) //topicNum - + gHandle->topicNum * tqBufHandleSSize(); -} - -//TODO: make this a macro +// TODO: make this a macro int tqBufHandleSSize() { - return sizeof(int64_t) * 2 // nextConsumeOffset + topicId - + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqBufItemSSize(); + return sizeof(int64_t) * 2 // nextConsumeOffset + topicId + + sizeof(int32_t) * 2 // head + tail + + TQ_BUFFER_SIZE * tqBufItemSSize(); } int tqBufItemSSize() { - //TODO: do this need serialization? - //mainly for executor + // TODO: do this need serialization? + // mainly for executor return 0; } diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/tq/src/tqMetaStore.c index 3f40c94f24..082f0ad28e 100644 --- a/source/dnode/vnode/tq/src/tqMetaStore.c +++ b/source/dnode/vnode/tq/src/tqMetaStore.c @@ -13,20 +13,20 @@ * along with this program. If not, see . */ #include "tqMetaStore.h" -//TODO:replace by an abstract file layer -#include "osDir.h" +// TODO:replace by an abstract file layer #include #include #include +#include "osDir.h" #define TQ_META_NAME "tq.meta" -#define TQ_IDX_NAME "tq.idx" +#define TQ_IDX_NAME "tq.idx" static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value); static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key); -static inline void tqLinkUnpersist(STqMetaStore *pMeta, STqMetaList* pNode) { - if(pNode->unpersistNext == NULL) { +static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { + if (pNode->unpersistNext == NULL) { pNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pNode->unpersistPrev = pMeta->unpersistHead; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode; @@ -41,24 +41,24 @@ static inline int tqSeekLastPage(int fd) { return lseek(fd, curPageOffset, SEEK_SET); } -//TODO: the struct is tightly coupled with index entry -typedef struct TqIdxPageHead { +// TODO: the struct is tightly coupled with index entry +typedef struct STqIdxPageHead { int16_t writeOffset; int8_t unused[14]; -} TqIdxPageHead; +} STqIdxPageHead; -typedef struct TqIdxPageBuf { - TqIdxPageHead head; - char buffer[TQ_IDX_PAGE_BODY_SIZE]; -} TqIdxPageBuf; +typedef struct STqIdxPageBuf { + STqIdxPageHead head; + char buffer[TQ_IDX_PAGE_BODY_SIZE]; +} STqIdxPageBuf; -static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) { +static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { int offset = tqSeekLastPage(fd); int nBytes; - if((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { return -1; } - if(nBytes == 0) { + if (nBytes == 0) { memset(pBuf, 0, TQ_PAGE_SIZE); pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; } @@ -67,28 +67,24 @@ static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) { return lseek(fd, offset, SEEK_SET); } -STqMetaStore* tqStoreOpen(const char* path, - TqSerializeFun serializer, - TqDeserializeFun deserializer, - TqDeleteFun deleter, - int32_t tqConfigFlag - ) { - STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); - if(pMeta == NULL) { - //close +STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter, + int32_t tqConfigFlag) { + STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); + if (pMeta == NULL) { + // close return NULL; } memset(pMeta, 0, sizeof(STqMetaStore)); - //concat data file name and index file name + // concat data file name and index file name size_t pathLen = strlen(path); - pMeta->dirPath = malloc(pathLen+1); - if(pMeta->dirPath != NULL) { - //TODO: memory insufficient + pMeta->dirPath = malloc(pathLen + 1); + if (pMeta->dirPath != NULL) { + // TODO: memory insufficient } strcpy(pMeta->dirPath, path); - - char name[pathLen+10]; + + char name[pathLen + 10]; strcpy(name, path); if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { @@ -96,98 +92,96 @@ STqMetaStore* tqStoreOpen(const char* path, } strcat(name, "/" TQ_IDX_NAME); int idxFd = open(name, O_RDWR | O_CREAT, 0755); - if(idxFd < 0) { + if (idxFd < 0) { ASSERT(false); - //close file - //free memory + // close file + // free memory return NULL; } pMeta->idxFd = idxFd; pMeta->unpersistHead = malloc(sizeof(STqMetaList)); - if(pMeta->unpersistHead == NULL) { + if (pMeta->unpersistHead == NULL) { ASSERT(false); - //close file - //free memory + // close file + // free memory return NULL; } memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); - pMeta->unpersistHead->unpersistNext - = pMeta->unpersistHead->unpersistPrev - = pMeta->unpersistHead; + pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; strcpy(name, path); strcat(name, "/" TQ_META_NAME); int fileFd = open(name, O_RDWR | O_CREAT, 0755); - if(fileFd < 0){ + if (fileFd < 0) { ASSERT(false); return NULL; } pMeta->fileFd = fileFd; - + pMeta->pSerializer = serializer; pMeta->pDeserializer = deserializer; pMeta->pDeleter = deleter; pMeta->tqConfigFlag = tqConfigFlag; - //read idx file and load into memory - TqIdxPageBuf idxBuf; - TqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); - if(serializedObj == NULL) { - //TODO:memory insufficient + // read idx file and load into memory + STqIdxPageBuf idxBuf; + STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); + if (serializedObj == NULL) { + // TODO:memory insufficient } - int idxRead; - int allocated = TQ_PAGE_SIZE; + int idxRead; + int allocated = TQ_PAGE_SIZE; bool readEnd = false; - while((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { - if(idxRead == -1) { - //TODO: handle error + while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { + if (idxRead == -1) { + // TODO: handle error ASSERT(false); } ASSERT(idxBuf.head.writeOffset == idxRead); - //loop read every entry - for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList *pNode = malloc(sizeof(STqMetaList)); - if(pNode == NULL) { - //TODO: free memory and return error + // loop read every entry + for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { + STqMetaList* pNode = malloc(sizeof(STqMetaList)); + if (pNode == NULL) { + // TODO: free memory and return error } memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); lseek(fileFd, pNode->handle.offset, SEEK_SET); - if(allocated < pNode->handle.serializedSize) { - void *ptr = realloc(serializedObj, pNode->handle.serializedSize); - if(ptr == NULL) { - //TODO: memory insufficient + if (allocated < pNode->handle.serializedSize) { + void* ptr = realloc(serializedObj, pNode->handle.serializedSize); + if (ptr == NULL) { + // TODO: memory insufficient } serializedObj = ptr; allocated = pNode->handle.serializedSize; } serializedObj->ssize = pNode->handle.serializedSize; - if(read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { - //TODO: read error + if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { + // TODO: read error } - if(serializedObj->action == TQ_ACTION_INUSE) { - if(serializedObj->ssize != sizeof(TqSerializedHead)) { + if (serializedObj->action == TQ_ACTION_INUSE) { + if (serializedObj->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } - } else if(serializedObj->action == TQ_ACTION_INTXN) { - if(serializedObj->ssize != sizeof(TqSerializedHead)) { + } else if (serializedObj->action == TQ_ACTION_INTXN) { + if (serializedObj->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; } - } else if(serializedObj->action == TQ_ACTION_INUSE_CONT) { - if(serializedObj->ssize != sizeof(TqSerializedHead)) { + } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) { + if (serializedObj->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } - TqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); - if(ptr->ssize != sizeof(TqSerializedHead)) { + STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); + if (ptr->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; @@ -196,22 +190,21 @@ STqMetaStore* tqStoreOpen(const char* path, ASSERT(0); } - //put into list - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; + // put into list + int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; STqMetaList* pBucketNode = pMeta->bucket[bucketKey]; - if(pBucketNode == NULL) { + if (pBucketNode == NULL) { pMeta->bucket[bucketKey] = pNode; - } else if(pBucketNode->handle.key == pNode->handle.key) { + } else if (pBucketNode->handle.key == pNode->handle.key) { pNode->next = pBucketNode->next; pMeta->bucket[bucketKey] = pNode; } else { - while(pBucketNode->next && - pBucketNode->next->handle.key != pNode->handle.key) { - pBucketNode = pBucketNode->next; + while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) { + pBucketNode = pBucketNode->next; } - if(pBucketNode->next) { + if (pBucketNode->next) { ASSERT(pBucketNode->next->handle.key == pNode->handle.key); - STqMetaList *pNodeFound = pBucketNode->next; + STqMetaList* pNodeFound = pBucketNode->next; pNode->next = pNodeFound->next; pBucketNode->next = pNode; pBucketNode = pNodeFound; @@ -221,13 +214,11 @@ STqMetaStore* tqStoreOpen(const char* path, pBucketNode = NULL; } } - if(pBucketNode) { - if(pBucketNode->handle.valueInUse - && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pBucketNode) { + if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pBucketNode->handle.valueInUse); } - if(pBucketNode->handle.valueInTxn - && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pBucketNode->handle.valueInTxn); } free(pBucketNode); @@ -239,23 +230,21 @@ STqMetaStore* tqStoreOpen(const char* path, } int32_t tqStoreClose(STqMetaStore* pMeta) { - //commit data and idx + // commit data and idx tqStorePersist(pMeta); - ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next==NULL); + ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); close(pMeta->fileFd); close(pMeta->idxFd); - //free memory - for(int i = 0; i < TQ_BUCKET_SIZE; i++) { + // free memory + for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; - while(pNode) { + while (pNode) { ASSERT(pNode->unpersistNext == NULL); ASSERT(pNode->unpersistPrev == NULL); - if(pNode->handle.valueInTxn - && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; @@ -272,17 +261,15 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { int32_t tqStoreDelete(STqMetaStore* pMeta) { close(pMeta->fileFd); close(pMeta->idxFd); - //free memory - for(int i = 0; i < TQ_BUCKET_SIZE; i++) { + // free memory + for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; pMeta->bucket[i] = NULL; - while(pNode) { - if(pNode->handle.valueInTxn - && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; @@ -297,26 +284,26 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) { return 0; } -//TODO: wrap in tfile +// TODO: wrap in tfile int32_t tqStorePersist(STqMetaStore* pMeta) { - TqIdxPageBuf idxBuf; - int64_t* bufPtr = (int64_t*)idxBuf.buffer; - STqMetaList *pHead = pMeta->unpersistHead; - STqMetaList *pNode = pHead->unpersistNext; - TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead)); - if(pSHead == NULL) { - //TODO: memory error + STqIdxPageBuf idxBuf; + int64_t* bufPtr = (int64_t*)idxBuf.buffer; + STqMetaList* pHead = pMeta->unpersistHead; + STqMetaList* pNode = pHead->unpersistNext; + STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); + if (pSHead == NULL) { + // TODO: memory error return -1; } pSHead->ver = TQ_SVER; pSHead->checksum = 0; - pSHead->ssize = sizeof(TqSerializedHead); - int allocatedSize = sizeof(TqSerializedHead); + pSHead->ssize = sizeof(STqSerializedHead); + int allocatedSize = sizeof(STqSerializedHead); int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); tqReadLastPage(pMeta->idxFd, &idxBuf); - if(idxBuf.head.writeOffset == TQ_PAGE_SIZE) { + if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { lseek(pMeta->idxFd, 0, SEEK_END); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; @@ -324,18 +311,18 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset); } - while(pHead != pNode) { + while (pHead != pNode) { int nBytes = 0; - if(pNode->handle.valueInUse) { - if(pNode->handle.valueInTxn) { + if (pNode->handle.valueInUse) { + if (pNode->handle.valueInTxn) { pSHead->action = TQ_ACTION_INUSE_CONT; } else { pSHead->action = TQ_ACTION_INUSE; } - if(pNode->handle.valueInUse == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(TqSerializedHead); + if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) { + pSHead->ssize = sizeof(STqSerializedHead); } else { pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); } @@ -343,10 +330,10 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { ASSERT(nBytes == pSHead->ssize); } - if(pNode->handle.valueInTxn) { + if (pNode->handle.valueInTxn) { pSHead->action = TQ_ACTION_INTXN; - if(pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(TqSerializedHead); + if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { + pSHead->ssize = sizeof(STqSerializedHead); } else { pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); } @@ -357,42 +344,39 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pNode->handle.offset = offset; offset += nBytes; - //write idx file - //TODO: endian check and convert + // write idx file + // TODO: endian check and convert *(bufPtr++) = pNode->handle.key; *(bufPtr++) = pNode->handle.offset; *(bufPtr++) = (int64_t)nBytes; idxBuf.head.writeOffset += TQ_IDX_SIZE; - if(idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { + if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); - //TODO: handle error with tfile + // TODO: handle error with tfile ASSERT(nBytes == TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; bufPtr = (int64_t*)&idxBuf.buffer; } - //remove from unpersist list + // remove from unpersist list pHead->unpersistNext = pNode->unpersistNext; pHead->unpersistNext->unpersistPrev = pHead; pNode->unpersistPrev = pNode->unpersistNext = NULL; pNode = pHead->unpersistNext; - //remove from bucket - if(pNode->handle.valueInUse == TQ_DELETE_TOKEN && - pNode->handle.valueInTxn == NULL - ) { - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; + // remove from bucket + if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) { + int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; STqMetaList* pBucketHead = pMeta->bucket[bucketKey]; - if(pBucketHead == pNode) { + if (pBucketHead == pNode) { pMeta->bucket[bucketKey] = pNode->next; } else { STqMetaList* pBucketNode = pBucketHead; - while(pBucketNode->next != NULL - && pBucketNode->next != pNode) { - pBucketNode = pBucketNode->next; + while (pBucketNode->next != NULL && pBucketNode->next != pNode) { + pBucketNode = pBucketNode->next; } - //impossible for pBucket->next == NULL + // impossible for pBucket->next == NULL ASSERT(pBucketNode->next == pNode); pBucketNode->next = pNode->next; } @@ -400,46 +384,45 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } } - //write left bytes + // write left bytes free(pSHead); - //TODO: write new version in tfile - if((char*)bufPtr != idxBuf.buffer) { + // TODO: write new version in tfile + if ((char*)bufPtr != idxBuf.buffer) { int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); - //TODO: handle error in tfile + // TODO: handle error in tfile ASSERT(nBytes == idxBuf.head.writeOffset); } - //TODO: using fsync in tfile + // TODO: using fsync in tfile fsync(pMeta->idxFd); fsync(pMeta->fileFd); return 0; } static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - //TODO: think about thread safety - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + // TODO: think about thread safety + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } - //change pointer ownership + // change pointer ownership pNode->handle.valueInUse = value; return 0; } else { pNode = pNode->next; } } - STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); - if(pNewNode == NULL) { - //TODO: memory error + STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + if (pNewNode == NULL) { + // TODO: memory error return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInUse = value; - //put into unpersist list + // put into unpersist list pNewNode->unpersistPrev = pMeta->unpersistHead; pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode; @@ -448,12 +431,11 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu } void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInUse != NULL - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { return pNode->handle.valueInUse; } else { return NULL; @@ -466,12 +448,11 @@ void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { } void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInUse != NULL - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { tqLinkUnpersist(pMeta, pNode); return pNode->handle.valueInUse; } else { @@ -485,16 +466,16 @@ void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { } static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - //TODO: think about thread safety - if(pNode->handle.valueInTxn) { - if(TqDupIntxnReject(pMeta->tqConfigFlag)) { + while (pNode) { + if (pNode->handle.key == key) { + // TODO: think about thread safety + if (pNode->handle.valueInTxn) { + if (TqDupIntxnReject(pMeta->tqConfigFlag)) { return -2; } - if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } } @@ -505,9 +486,9 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va pNode = pNode->next; } } - STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); - if(pNewNode == NULL) { - //TODO: memory error + STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + if (pNewNode == NULL) { + // TODO: memory error return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -519,14 +500,12 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va return 0; } -int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { - return tqHandlePutImpl(pMeta, key, value); -} +int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { - void *vmem = malloc(vsize); - if(vmem == NULL) { - //TODO: memory error + void* vmem = malloc(vsize); + if (vmem == NULL) { + // TODO: memory error return -1; } memcpy(vmem, value, vsize); @@ -534,12 +513,11 @@ int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vs } static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInTxn != NULL - && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { return pNode->handle.valueInTxn; } else { return NULL; @@ -552,15 +530,14 @@ static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { } int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInTxn == NULL) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn == NULL) { return -1; } - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } pNode->handle.valueInUse = pNode->handle.valueInTxn; @@ -575,12 +552,12 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { } int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInTxn) { - if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } pNode->handle.valueInTxn = NULL; @@ -596,13 +573,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { } int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - if(pNode->handle.valueInTxn) { + while (pNode) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn) { pMeta->pDeleter(pNode->handle.valueInTxn); } + pNode->handle.valueInTxn = TQ_DELETE_TOKEN; tqLinkUnpersist(pMeta, pNode); return 0; @@ -610,11 +588,9 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - //no such key + // no such key return -1; } -//TODO: clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore *pMeta) { - return 0; -} +// TODO: clean deleted idx and data from persistent file +int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; } diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/tq/test/tqMetaTest.cpp index bbd436ab1a..58263efa71 100644 --- a/source/dnode/vnode/tq/test/tqMetaTest.cpp +++ b/source/dnode/vnode/tq/test/tqMetaTest.cpp @@ -9,17 +9,17 @@ struct Foo { int32_t a; }; -int FooSerializer(const void* pObj, TqSerializedHead** ppHead) { +int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { Foo* foo = (Foo*) pObj; - if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(TqSerializedHead) + sizeof(int32_t)) { - *ppHead = (TqSerializedHead*)realloc(*ppHead, sizeof(TqSerializedHead) + sizeof(int32_t)); - (*ppHead)->ssize = sizeof(TqSerializedHead) + sizeof(int32_t); + if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { + *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); + (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } *(int32_t*)(*ppHead)->content = foo->a; return (*ppHead)->ssize; } -const void* FooDeserializer(const TqSerializedHead* pHead, void** ppObj) { +const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { if(*ppObj == NULL) { *ppObj = realloc(*ppObj, sizeof(int32_t)); } diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index 25c80d502a..e6311152d6 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -10,3 +10,5 @@ target_link_libraries( catalog PRIVATE os util common transport query ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 455c82b1bc..7b26079f11 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -70,9 +70,10 @@ extern int32_t ctgDebugFlag; #define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0) -#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) -#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); return _code; } } while (0) -#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) +#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) +#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) +#define CTG_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { ctgError(__VA_ARGS__); terrno = _code; return _code; } } while (0) +#define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #ifdef __cplusplus diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a670d9d639..32cedb82b0 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -20,50 +20,7 @@ SCatalogMgmt ctgMgmt = {0}; -int32_t ctgGetVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SVgroupListInfo** pVgroup) { - char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; - int32_t msgLen = 0; - - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST](NULL, &msg, 0, &msgLen); - if (code) { - return code; - } - - SRpcMsg rpcMsg = { - .msgType = TSDB_MSG_TYPE_VGROUP_LIST, - .pCont = msg, - .contLen = msgLen, - }; - - SRpcMsg rpcRsp = {0}; - - rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - - code = queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST](pVgroup, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - return code; - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetVgroupFromCache(struct SCatalog* pCatalog, SHashObj** pVgroupList, int32_t* exist) { - if (NULL == pCatalog->vgroupCache.cache || pCatalog->vgroupCache.vgroupVersion < 0) { - *exist = 0; - return TSDB_CODE_SUCCESS; - } - - if (pVgroupList) { - *pVgroupList = pCatalog->vgroupCache.cache; - } - - *exist = 1; - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, int32_t *exist) { +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo *dbInfo, int32_t *exist) { if (NULL == pCatalog->dbCache.cache) { *exist = 0; return TSDB_CODE_SUCCESS; @@ -71,28 +28,13 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S SDBVgroupInfo *info = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (NULL == info || info->vgroupVersion < pCatalog->vgroupCache.vgroupVersion) { + if (NULL == info) { *exist = 0; return TSDB_CODE_SUCCESS; } if (dbInfo) { - *dbInfo = calloc(1, sizeof(**dbInfo)); - if (NULL == *dbInfo) { - ctgError("calloc size[%d] failed", (int32_t)sizeof(**dbInfo)); - return TSDB_CODE_CTG_MEM_ERROR; - } - - (*dbInfo)->vgId = taosArrayDup(info->vgId); - if (NULL == (*dbInfo)->vgId) { - ctgError("taos array duplicate failed"); - tfree(*dbInfo); - return TSDB_CODE_CTG_MEM_ERROR; - } - - (*dbInfo)->vgroupVersion = info->vgroupVersion; - (*dbInfo)->hashRange = info->hashRange; - (*dbInfo)->hashType = info->hashType; + *dbInfo = *info; } *exist = 1; @@ -107,26 +49,35 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen); - if (code) { - return code; + CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_USE_DB](input, &msg, 0, &msgLen)); + + char *pMsg = rpcMallocCont(msgLen); + if (NULL == pMsg) { + ctgError("rpc malloc %d failed", msgLen); + tfree(msg); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } + memcpy(pMsg, msg, msgLen); + + tfree(msg); + SRpcMsg rpcMsg = { .msgType = TSDB_MSG_TYPE_USE_DB, - .pCont = msg, + .pCont = pMsg, .contLen = msgLen, }; SRpcMsg rpcRsp = {0}; rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); - - code = queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - return code; + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for use db, code:%x", rpcRsp.code); + CTG_ERR_RET(rpcRsp.code); } + CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen)); + return TSDB_CODE_SUCCESS; } @@ -157,14 +108,14 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, if ((*stbMeta)->suid != tbMeta->suid) { ctgError("stable cache error, expected suid:%"PRId64 ",actual suid:%"PRId64, tbMeta->suid, (*stbMeta)->suid); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); *pTableMeta = calloc(1, metaSize); if (NULL == *pTableMeta) { ctgError("calloc size[%d] failed", metaSize); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(*pTableMeta, tbMeta, sizeof(SCTableMeta)); @@ -174,7 +125,7 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const char *dbName, *pTableMeta = calloc(1, metaSize); if (NULL == *pTableMeta) { ctgError("calloc size[%d] failed", metaSize); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(*pTableMeta, tbMeta, metaSize); @@ -198,7 +149,7 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } char tbFullName[TSDB_TABLE_FNAME_LEN]; @@ -210,10 +161,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - int32_t code = queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen); - if (code) { - return code; - } + CTG_ERR_RET(queryBuildMsg[TSDB_MSG_TYPE_TABLE_META](&bInput, &msg, 0, &msgLen)); SRpcMsg rpcMsg = { .msgType = TSDB_MSG_TYPE_TABLE_META, @@ -227,23 +175,20 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE ctgGenEpSet(&epSet, vgroupInfo); rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp); - + if (TSDB_CODE_SUCCESS != rpcRsp.code) { - ctgError("get table meta from mnode failed, error code:%d", rpcRsp.code); - return rpcRsp.code; + ctgError("error rsp for table meta, code:%x", rpcRsp.code); + CTG_ERR_RET(rpcRsp.code); } - code = queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen); - if (code) { - return code; - } + CTG_ERR_RET(queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen)); return TSDB_CODE_SUCCESS; } -int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) { - switch (hashType) { +int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { + switch (hashMethod) { default: *fp = MurmurHash3_32; break; @@ -252,129 +197,68 @@ int32_t ctgGetHashFunction(int32_t hashType, tableNameHashFp *fp) { return TSDB_CODE_SUCCESS; } -int32_t ctgGetVgroupFromVgId(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, int32_t vgId, SVgroupInfo *pVgroup) { +int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SDBVgroupInfo *dbInfo, SArray* vgroupList) { SHashObj *vgroupHash = NULL; - - CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); - if (NULL == vgroupHash) { - ctgError("get empty vgroup cache"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } + SVgroupInfo *vgInfo = NULL; - if (NULL == taosHashGetClone(vgroupHash, &vgId, sizeof(vgId), pVgroup)) { - ctgError("vgId[%d] not found in vgroup list", vgId); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } + void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + while (pIter) { + vgInfo = pIter; - return TSDB_CODE_SUCCESS; -} - -int32_t ctgGetVgroupFromVgIdBatch(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, SArray* vgIds, SArray* vgroupList) { - SHashObj *vgroupHash = NULL; - SVgroupInfo pVgroup = {0}; - int32_t vgIdNum = taosArrayGetSize(vgIds); - - CTG_ERR_RET(catalogGetVgroup(pCatalog, pRpc, pMgmtEps, &vgroupHash)); - if (NULL == vgroupHash) { - ctgError("get empty vgroup cache"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - for (int32_t i = 0; i < vgIdNum; ++i) { - int32_t *vgId = taosArrayGet(vgIds, i); + if (NULL == taosArrayPush(vgroupList, vgInfo)) { + ctgError("taosArrayPush failed"); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } - if (NULL == taosHashGetClone(vgroupHash, vgId, sizeof(*vgId), &pVgroup)) { - ctgError("vgId[%d] not found in vgroup list", vgId); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - if (NULL == taosArrayPush(vgroupList, &pVgroup)) { - ctgError("push vgroup to array failed, idx:%d", i); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } + pIter = taosHashIterate(dbInfo->vgInfo, pIter); + vgInfo = NULL; } return TSDB_CODE_SUCCESS; } - - -int32_t ctgGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { - SDBVgroupInfo *dbInfo = NULL; - int32_t code = 0; - - CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); - - if (NULL == dbInfo) { - ctgWarn("db[%s] vgroup info not found", pDBName); - return TSDB_CODE_TSC_DB_NOT_SELECTED; - } - - if (dbInfo->vgroupVersion < 0 || NULL == dbInfo->vgId) { - ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgId:%p", pDBName, dbInfo->vgroupVersion, dbInfo->vgId); - CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); - } - - int32_t vgNum = taosArrayGetSize(dbInfo->vgId); +int32_t ctgGetVgInfoFromHashValue(SDBVgroupInfo *dbInfo, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); if (vgNum <= 0) { ctgError("db[%s] vgroup cache invalid, vgroup number:%d", pDBName, vgNum); - CTG_ERR_JRET(TSDB_CODE_TSC_DB_NOT_SELECTED); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); } tableNameHashFp fp = NULL; + SVgroupInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashType, &fp)); + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); char tbFullName[TSDB_TABLE_FNAME_LEN]; snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName); uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); - uint32_t hashUnit = dbInfo->hashRange / vgNum; - uint32_t vgId = hashValue / hashUnit; - CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, vgId, pVgroup)); - -_return: - if (dbInfo && dbInfo->vgId) { - taosArrayDestroy(dbInfo->vgId); - dbInfo->vgId = NULL; - } - - tfree(dbInfo); - - return code; -} - - - -STableMeta* ctgCreateSTableMeta(STableMetaMsg* pChild) { - assert(pChild != NULL); - int32_t total = pChild->numOfColumns + pChild->numOfTags; - - STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + sizeof(SSchema) * total); - pTableMeta->tableType = TSDB_SUPER_TABLE; - pTableMeta->tableInfo.numOfTags = pChild->numOfTags; - pTableMeta->tableInfo.numOfColumns = pChild->numOfColumns; - pTableMeta->tableInfo.precision = pChild->precision; - - pTableMeta->uid = pChild->suid; - pTableMeta->tversion = pChild->tversion; - pTableMeta->sversion = pChild->sversion; - - memcpy(pTableMeta->schema, pChild->pSchema, sizeof(SSchema) * total); - - int32_t num = pTableMeta->tableInfo.numOfColumns; - for(int32_t i = 0; i < num; ++i) { - pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + break; + } + + pIter = taosHashIterate(dbInfo->vgInfo, pIter); + vgInfo = NULL; } - return pTableMeta; + if (NULL == vgInfo) { + ctgError("no hash range found for hashvalue[%u]", hashValue); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + *pVgroup = *vgInfo; + + return TSDB_CODE_SUCCESS; } int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, bool forceUpdate, STableMeta** pTableMeta) { if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; @@ -393,7 +277,7 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* if (0 == exist) { ctgError("get table meta from cache failed, but fetch succeed"); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } return TSDB_CODE_SUCCESS; @@ -403,19 +287,19 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { if (output->metaNum != 1 && output->metaNum != 2) { ctgError("invalid table meta number[%d] got from meta rsp", output->metaNum); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == output->tbMeta) { ctgError("no valid table meta got from meta rsp"); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (NULL == pCatalog->tableCache.cache) { pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -423,13 +307,13 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.cache) { ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->tableCache.stableCache) { ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -467,7 +351,7 @@ error_exit: pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t catalogInit(SCatalogCfg *cfg) { @@ -489,12 +373,12 @@ int32_t catalogInit(SCatalogCfg *cfg) { int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) { if (NULL == clusterId || NULL == catalogHandle) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (NULL == ctgMgmt.pCluster) { ctgError("cluster cache are not ready"); - return TSDB_CODE_CTG_NOT_READY; + CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } size_t clen = strlen(clusterId); @@ -508,7 +392,7 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) clusterCtg = calloc(1, sizeof(*clusterCtg)); if (NULL == clusterCtg) { ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg)); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; @@ -516,7 +400,7 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) { ctgError("put cluster %s cache to hash failed", clusterId); tfree(clusterCtg); - return TSDB_CODE_CTG_INTERNAL_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } *catalogHandle = clusterCtg; @@ -524,98 +408,9 @@ int32_t catalogGetHandle(const char *clusterId, struct SCatalog** catalogHandle) return TSDB_CODE_SUCCESS; } - -int32_t catalogGetVgroupVersion(struct SCatalog* pCatalog, int32_t* version) { - if (NULL == pCatalog || NULL == version) { - return TSDB_CODE_CTG_INVALID_INPUT; - } - - *version = pCatalog->vgroupCache.vgroupVersion; - - return TSDB_CODE_SUCCESS; -} - - - -int32_t catalogUpdateVgroupCache(struct SCatalog* pCatalog, SVgroupListInfo* pVgroup) { - if (NULL == pVgroup) { - ctgError("no valid vgroup list info to update"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - if (pVgroup->vgroupVersion < 0) { - ctgError("vgroup version[%d] is invalid", pVgroup->vgroupVersion); - return TSDB_CODE_CTG_INVALID_INPUT; - } - - if (NULL == pCatalog->vgroupCache.cache) { - pCatalog->vgroupCache.cache = taosHashInit(CTG_DEFAULT_CACHE_VGROUP_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == pCatalog->vgroupCache.cache) { - ctgError("init hash[%d] for cluster cache failed", CTG_DEFAULT_CACHE_VGROUP_NUMBER); - return TSDB_CODE_CTG_MEM_ERROR; - } - } else { - taosHashClear(pCatalog->vgroupCache.cache); - } - - SVgroupInfo *vInfo = NULL; - for (int32_t i = 0; i < pVgroup->vgroupNum; ++i) { - if (taosHashPut(pCatalog->vgroupCache.cache, &pVgroup->vgroupInfo[i].vgId, sizeof(pVgroup->vgroupInfo[i].vgId), &pVgroup->vgroupInfo[i], sizeof(pVgroup->vgroupInfo[i])) != 0) { - ctgError("push to vgroup hash cache failed"); - goto error_exit; - } - } - - pCatalog->vgroupCache.vgroupVersion = pVgroup->vgroupVersion; - - return TSDB_CODE_SUCCESS; - -error_exit: - if (pCatalog->vgroupCache.cache) { - taosHashCleanup(pCatalog->vgroupCache.cache); - pCatalog->vgroupCache.cache = NULL; - } - - pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION; - - return TSDB_CODE_CTG_INTERNAL_ERROR; -} - -int32_t catalogGetVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SHashObj** pVgroupHash) { - if (NULL == pCatalog || NULL == pMgmtEps || NULL == pRpc) { - return TSDB_CODE_CTG_INVALID_INPUT; - } - - int32_t exist = 0; - - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); - - if (exist) { - return TSDB_CODE_SUCCESS; - } - - SVgroupListInfo *pVgroup = NULL; - - CTG_ERR_RET(ctgGetVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &pVgroup)); - - CTG_ERR_RET(catalogUpdateVgroupCache(pCatalog, pVgroup)); - - if (pVgroupHash) { - CTG_ERR_RET(ctgGetVgroupFromCache(pCatalog, pVgroupHash, &exist)); - } - - if (0 == exist) { - ctgError("catalog fetched but get from cache failed"); - return TSDB_CODE_CTG_INTERNAL_ERROR; - } - - return TSDB_CODE_SUCCESS; -} - - int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, int32_t* version) { if (NULL == pCatalog || NULL == dbName || NULL == version) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } if (NULL == pCatalog->dbCache.cache) { @@ -629,17 +424,17 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, return TSDB_CODE_SUCCESS; } - *version = dbInfo->vgroupVersion; + *version = dbInfo->vgVersion; return TSDB_CODE_SUCCESS; } int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == dbInfo) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (dbInfo->vgroupVersion < 0) { + if (dbInfo->vgVersion < 0) { if (pCatalog->dbCache.cache) { taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName)); } @@ -652,13 +447,19 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == pCatalog->dbCache.cache) { ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + } else { + SDBVgroupInfo *oldInfo = taosHashGet(pCatalog->dbCache.cache, dbName, strlen(dbName)); + if (oldInfo && oldInfo->vgInfo) { + taosHashCleanup(oldInfo->vgInfo); + oldInfo->vgInfo = NULL; } } if (taosHashPut(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo)) != 0) { ctgError("push to vgroup hash cache failed"); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } return TSDB_CODE_SUCCESS; @@ -667,13 +468,12 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName -int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo** dbInfo) { +int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, int32_t forceUpdate, SDBVgroupInfo* dbInfo) { if (NULL == pCatalog || NULL == dbName || NULL == pRpc || NULL == pMgmtEps) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t exist = 0; - int32_t code = 0; if (0 == forceUpdate) { CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &exist)); @@ -688,29 +488,17 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* strncpy(input.db, dbName, sizeof(input.db)); input.db[sizeof(input.db) - 1] = 0; - input.vgroupVersion = pCatalog->vgroupCache.vgroupVersion; - input.dbGroupVersion = CTG_DEFAULT_INVALID_VERSION; + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - if (DbOut.vgroupList) { - CTG_ERR_JRET(catalogUpdateVgroupCache(pCatalog, DbOut.vgroupList)); - } - - if (DbOut.dbVgroup) { - CTG_ERR_JRET(catalogUpdateDBVgroupCache(pCatalog, dbName, DbOut.dbVgroup)); - } + CTG_ERR_RET(catalogUpdateDBVgroupCache(pCatalog, dbName, &DbOut.dbVgroup)); if (dbInfo) { *dbInfo = DbOut.dbVgroup; - DbOut.dbVgroup = NULL; } -_return: - tfree(DbOut.dbVgroup); - tfree(DbOut.vgroupList); - - return code; + return TSDB_CODE_SUCCESS; } int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) { @@ -719,12 +507,12 @@ int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) { if (NULL == pCatalog || NULL == pDBName || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } SVgroupInfo vgroupInfo = {0}; - CTG_ERR_RET(ctgGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo)); + CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo)); STableMetaOutput output = {0}; @@ -741,24 +529,28 @@ int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, true, pTableMeta); } -int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) { +int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, SArray* pVgroupList) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == pVgroupList) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; - SDBVgroupInfo *dbVgroup = NULL; + SDBVgroupInfo dbVgroup = {0}; CTG_ERR_JRET(catalogGetTableMeta(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &tbMeta)); - if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); + CTG_ERR_JRET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbVgroup)); - CTG_ERR_JRET(ctgGetVgroupFromVgIdBatch(pCatalog, pRpc, pMgmtEps, dbVgroup->vgId, pVgroupList)); + if (tbMeta->tableType == TSDB_SUPER_TABLE) { + CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, &dbVgroup, pVgroupList)); } else { - CTG_ERR_JRET(ctgGetVgroupFromVgId(pCatalog, pRpc, pMgmtEps, tbMeta->vgId, &vgroupInfo)); + int32_t vgId = tbMeta->vgId; + if (NULL == taosHashGetClone(dbVgroup.vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { + ctgError("vgId[%d] not found in vgroup list", vgId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } if (NULL == taosArrayPush(pVgroupList, &vgroupInfo)) { ctgError("push vgroupInfo to array failed"); @@ -768,20 +560,32 @@ int32_t catalogGetTableVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSe _return: tfree(tbMeta); - if (dbVgroup && dbVgroup->vgId) { - taosArrayDestroy(dbVgroup->vgId); - dbVgroup->vgId = NULL; + + CTG_RET(code); +} + + +int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pRpc, const SEpSet *pMgmtEps, const char *pDBName, const char *pTableName, SVgroupInfo *pVgroup) { + SDBVgroupInfo dbInfo = {0}; + int32_t code = 0; + int32_t vgId = 0; + + CTG_ERR_RET(catalogGetDBVgroup(pCatalog, pRpc, pMgmtEps, pDBName, false, &dbInfo)); + + if (dbInfo.vgVersion < 0 || NULL == dbInfo.vgInfo) { + ctgError("db[%s] vgroup cache invalid, vgroup version:%d, vgInfo:%p", pDBName, dbInfo.vgVersion, dbInfo.vgInfo); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); } - - tfree(dbVgroup); - - return code; + + CTG_ERR_RET(ctgGetVgInfoFromHashValue(&dbInfo, pDBName, pTableName, pVgroup)); + + CTG_RET(code); } int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pReq || NULL == pRsp) { - return TSDB_CODE_CTG_INVALID_INPUT; + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; @@ -793,7 +597,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* p pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES); if (NULL == pRsp->pTableMeta) { ctgError("taosArrayInit num[%d] failed", tbNum); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } } @@ -827,7 +631,7 @@ _return: taosArrayDestroy(pRsp->pTableMeta); } - return code; + CTG_RET(code); } void catalogDestroy(void) { diff --git a/source/libs/catalog/test/CMakeLists.txt b/source/libs/catalog/test/CMakeLists.txt new file mode 100644 index 0000000000..176978cc7f --- /dev/null +++ b/source/libs/catalog/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build catalog unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(catalogTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + catalogTest + PUBLIC os util common catalog transport gtest query taos +) + +TARGET_INCLUDE_DIRECTORIES( + catalogTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/catalog/inc" +) diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e69de29bb2..e14c58d412 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" +#include "tep.h" +#include "trpc.h" + +typedef struct SAppInstInfo { + int64_t numOfConns; + SCorEpSet mgmtEp; +} SAppInstInfo; + +typedef struct STscObj { + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char acctId[TSDB_ACCT_ID_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; + uint32_t connId; + uint64_t id; // ref ID returned by taosAddRef +// struct SSqlObj *sqlList; + void *pTransporter; + pthread_mutex_t mutex; // used to protect the operation on db + int32_t numOfReqs; // number of sqlObj from this tscObj + SAppInstInfo *pAppInfo; +} STscObj; + +namespace { + +void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { + SCreateDbMsg* pReq = (SCreateDbMsg*)rpcMallocCont(sizeof(SCreateDbMsg)); + strcpy(pReq->db, "1.db1"); + pReq->numOfVgroups = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRowsPerFileBlock = htonl(100); + pReq->maxRowsPerFileBlock = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replications = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->ignoreExist = 1; + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateDbMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_DB; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp); + + ASSERT_EQ(rpcRsp.code, 0); +} + +} + +TEST(testCase, normalCase) { + STscObj* pConn = (STscObj *)taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + struct SCatalog* pCtg = NULL; + void *mockPointer = (void *)0x1; + SVgroupInfo vgInfo = {0}; + + msgInit(); + + sendCreateDbMsg(pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet); + + int32_t code = catalogInit(NULL); + ASSERT_EQ(code, 0); + + code = catalogGetHandle(clusterId, &pCtg); + ASSERT_EQ(code, 0); + + code = catalogGetTableHashVgroup(pCtg, pConn->pTransporter, &pConn->pAppInfo->mgmtEp.epSet, dbname, tablename, &vgInfo); + ASSERT_EQ(code, 0); + + taos_close(pConn); +} + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a6862c05c8..f6ff9bc139 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -17,7 +17,10 @@ #define _TD_INDEX_INT_H_ #include "index.h" +#include "index_fst.h" #include "tlog.h" +#include "thash.h" +#include "taos.h" #ifdef USE_LUCENE #include @@ -32,12 +35,20 @@ struct SIndex { #ifdef USE_LUCENE index_t *index; #endif + void *cache; + void *tindex; + SHashObj *fieldObj; // + uint64_t suid; + int fieldId; + pthread_mutex_t mtx; }; struct SIndexOpts { #ifdef USE_LUCENE void *opts; #endif + int32_t numOfItermLimit; + int8_t mergeInterval; }; struct SIndexMultiTermQuery { diff --git a/source/libs/index/inc/index_cache.h b/source/libs/index/inc/index_cache.h new file mode 100644 index 0000000000..27e095ff31 --- /dev/null +++ b/source/libs/index/inc/index_cache.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_CACHE_H__ +#define __INDEX_CACHE_H__ + +#include "index.h" +#include "tlockfree.h" +// ----------------- row structure in skiplist --------------------- + +/* A data row, the format is like below: + * |<--totalLen-->|<-- fieldId-->|<-- value len--->|<-- value-->|<--version--->|<-- itermType -->| + * + */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct IndexCache { + T_REF_DECLARE() + int cVersion; // +} IndexCache; + + +// +IndexCache *indexCacheCreate(); + +void indexCacheDestroy(IndexCache *cache); + +int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldVale, int32_t fvlen, uint64_t uid, int8_t operaType); + +int indexCacheGet(IndexCache *cache, uint64_t *rst); +int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result); + +#ifdef __cplusplus +} +#endif + + + +#endif diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index f8f4311a4a..c011411189 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -15,39 +15,58 @@ #include "index.h" #include "indexInt.h" +#include "index_cache.h" #ifdef USE_LUCENE #include "lucene++/Lucene_c.h" #endif -static pthread_once_t isInit = PTHREAD_ONCE_INIT; +typedef struct SIdxFieldInfo { + int id; // generated by index internal + int type; // field type +} SIdxFieldInfo; + +static pthread_once_t isInit = PTHREAD_ONCE_INIT; static void indexInit(); +static int indexMergeCacheIntoTindex(struct SIndex *sIdx) { + if (sIdx == NULL) { + return -1; + } + indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid); + return 0; +} SIndex *indexOpen(SIndexOpts *opts, const char *path) { pthread_once(&isInit, indexInit); + SIndex *sIdx = malloc(sizeof(SIndex)); + #ifdef USE_LUCENE index_t *index = index_open(path); - SIndex *p = malloc(sizeof(SIndex)); - p->index = index; - return p; + sIdx->index = index; #endif - return NULL; + + sIdx->cache = (void*)indexCacheCreate(); + sIdx->tindex = NULL; + sIdx->fieldObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pthread_mutex_init(&sIdx->mtx, NULL); + return sIdx; } -void indexClose(SIndex *index) { +void indexClose(SIndex *sIdx) { #ifdef USE_LUCENE - index_close(index->index); - index->index = NULL; + index_close(sIdex->index); + sIdx->index = NULL; #endif - free(index); + indexCacheDestroy(sIdx->cache); + taosHashCleanup(sIdx->fieldObj); + pthread_mutex_destroy(&sIdx->mtx); + free(sIdx); return; - } -#ifdef USE_LUCENE -#endif int indexPut(SIndex *index, SArray* field_vals, int uid) { + #ifdef USE_LUCENE index_document_t *doc = index_document_create(); @@ -63,6 +82,8 @@ int indexPut(SIndex *index, SArray* field_vals, int uid) { index_put(index->index, doc); index_document_destroy(doc); #endif + pthread_mutex_lock(&index->mtx); + pthread_mutex_unlock(&index->mtx); return 1; } @@ -105,7 +126,9 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result return 1; } + int indexDelete(SIndex *index, SIndexMultiTermQuery *query) { + return 1; } int indexRebuild(SIndex *index, SIndexOpts *opts); diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c new file mode 100644 index 0000000000..7c355b0f0a --- /dev/null +++ b/source/libs/index/src/index_cache.c @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "index_cache.h" + +static int32_t compareKey(const void *l, const void *r) { + char *lp = (char *)l; + char *rp = (char *)r; + + // skip total len + int32_t ll, rl; // len + memcpy(&ll, lp, sizeof(int32_t)); + memcpy(&rl, rp, sizeof(int32_t)); + lp += sizeof(int32_t); + rp += sizeof(int32_t); + + // compare field id + int32_t lf, rf; // field id + memcpy(&lf, lp, sizeof(lf)); + memcpy(&rf, rp, sizeof(rf)); + if (lf != rf) { + return lf < rf ? -1: 1; + } + lp += sizeof(lf); + rp += sizeof(rf); + + // compare field value + int32_t lfl, rfl; + memcpy(&lfl, lp, sizeof(lfl)); + memcpy(&rfl, rp, sizeof(rfl)); + lp += sizeof(lfl); + rp += sizeof(rfl); + + //refator later + int32_t i, j; + for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) { + if (lp[i] == rp[j]) { continue; } + else { return lp[i] < rp[j] ? -1 : 1;} + } + if (i < lfl) { return 1;} + else if (j < rfl) { return -1; } + lp += lfl; + rp += rfl; + + // compare version + int32_t lv, rv; + memcpy(&lv, lp, sizeof(lv)); + memcpy(&rv, rp, sizeof(rv)); + if (lv != rv) { + return lv > rv ? -1 : 1; + } + lp += sizeof(lv); + rp += sizeof(rv); + + + return 0; + +} +IndexCache *indexCacheCreate() { + IndexCache *cache = calloc(1, sizeof(IndexCache)); + return cache; +} + +void indexCacheDestroy(IndexCache *cache) { + free(cache); +} + +int indexCachePut(IndexCache *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) { + if (cache == NULL) { return -1;} + int32_t version = T_REF_INC(cache); + + int32_t total = sizeof(int32_t) + sizeof(fieldId) + 4 + fvlen + sizeof(version) + sizeof(uid) + sizeof(operType); + + char *buf = calloc(1, total); + char *p = buf; + + memcpy(buf, &total, sizeof(total)); + total += total; + + memcpy(buf, &fieldId, sizeof(fieldId)); + buf += sizeof(fieldId); + + memcpy(buf, &fvlen, sizeof(fvlen)); + buf += sizeof(fvlen); + memcpy(buf, fieldValue, fvlen); + buf += fvlen; + + memcpy(buf, &version, sizeof(version)); + buf += sizeof(version); + + memcpy(buf, &uid, sizeof(uid)); + buf += sizeof(uid); + + memcpy(buf, &operType, sizeof(operType)); + buf += sizeof(operType); + + +} +int indexCacheSearch(IndexCache *cache, SIndexMultiTermQuery *query, SArray *result) { + + return 0; +} + + + + + diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 9cb4ac6836..7aaa498864 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1330,7 +1330,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb SArray *nodes = taosArrayInit(8, sizeof(FstNode *)); while (taosArrayGetSize(sws->stack) > 0) { StreamState *p = (StreamState *)taosArrayPop(sws->stack); - if (p->trans >= FST_NODE_LEN(p->node) || automFuncs[aut->type].canMatch(aut, p->autState)) { + if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index f70b90041b..07ad45079b 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -87,9 +87,18 @@ static void* prefixAccept(AutomationCtx *ctx, void *state, uint8_t byte) { if (ssv == NULL || ctx == NULL) {return NULL;} char *data = ctx->data; + if (ssv->kind == Done) { + return startWithStateValueCreate(Done, FST_INT, &ssv->val); + } if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { int val = ssv->val + 1; - return startWithStateValueCreate(Running, FST_INT, &val); + StartWithStateValue *nsv = startWithStateValueCreate(Running, FST_INT, &val); + if (prefixIsMatch(ctx, nsv)) { + nsv->kind = Done; + } else { + nsv->kind = Running; + } + return nsv; } return NULL; } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h new file mode 100644 index 0000000000..223d5a5768 --- /dev/null +++ b/source/libs/parser/inc/astToMsg.h @@ -0,0 +1,9 @@ +#ifndef TDENGINE_ASTTOMSG_H +#define TDENGINE_ASTTOMSG_H + +#include "parserInt.h" +#include "taosmsg.h" + +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); + +#endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index e3925d3446..4f506e631f 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -53,6 +53,15 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t */ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen); +/** + * validate the ddl ast, and convert the ast to the corresponding message format + * @param pSqlInfo + * @param output + * @param type + * @return + */ +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); + /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. * @param pNode diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 6c95c4327b..b402621903 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -45,6 +45,8 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo); SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index); int32_t parserValidateIdToken(SToken* pToken); +int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf); + int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); @@ -63,6 +65,8 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo); STableMeta* tableMetaDup(const STableMeta* pTableMeta); +bool isDclSqlStatement(SSqlInfo* pSqlInfo); + #ifdef __cplusplus } #endif diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c new file mode 100644 index 0000000000..6bfbd5ebed --- /dev/null +++ b/source/libs/parser/src/astToMsg.c @@ -0,0 +1,23 @@ +#include "parserInt.h" + +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { + SCreateUserMsg *pMsg = (SCreateUserMsg *)calloc(1, sizeof(SCreateUserMsg)); + if (pMsg == NULL) { +// tscError("0x%" PRIx64 " failed to malloc for query msg", id); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SUserInfo *pUser = &pInfo->pMiscInfo->user; + strncpy(pMsg->user, pUser->user.z, pUser->user.n); + pMsg->type = pUser->type; + pMsg->superUser = (int8_t)pUser->type; + + if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { +// pMsg->privilege = (char)pCmd->count; + } else { + strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); + } + + return pMsg; +} \ No newline at end of file diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 9f1363a9b4..2ee95732d0 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "astGenerator.h" #include "function.h" #include "parserInt.h" @@ -23,6 +22,7 @@ #include "tglobal.h" #include "tmsgtype.h" #include "ttime.h" +#include "astToMsg.h" #define TSQL_TBNAME_L "tbname" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -3636,11 +3636,14 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) } int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msgBuf, int32_t msgBufLen) { - //1. if it is a query, get the meta info and continue. assert(pCatalog != NULL && pInfo != NULL); int32_t code = 0; -#if 0 + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf *pMsgBuf = &m; + switch (pInfo->type) { +#if 0 case TSDB_SQL_DROP_TABLE: case TSDB_SQL_DROP_USER: case TSDB_SQL_DROP_ACCT: @@ -3651,14 +3654,14 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (parserValidateIdToken(pzName) != TSDB_CODE_SUCCESS)) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } if (pInfo->type == TSDB_SQL_DROP_DB) { assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { @@ -3675,7 +3678,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer strncpy(pCmd->payload, pzName->z, pzName->n); } else { // drop user/account if (pzName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } strncpy(pCmd->payload, pzName->z, pzName->n); @@ -3689,12 +3692,12 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); if (ret != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } break; @@ -3729,19 +3732,19 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } char buf[TSDB_DB_NAME_LEN] = {0}; SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); if (tscValidateName(&token) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token); if (ret != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) { @@ -3755,7 +3758,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer const char* msg = "invalid host name (ip address)"; if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -3779,11 +3782,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } if (pName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; @@ -3793,7 +3796,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { } else { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } } @@ -3805,7 +3808,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } // additional msg has been attached already code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); @@ -3821,7 +3824,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); @@ -3836,11 +3839,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } if (pToken->n > TSDB_DB_NAME_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); } @@ -3853,7 +3856,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer /* validate the parameter names and options */ if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } char* pMsg = pCmd->payload; @@ -3867,7 +3870,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer strncpy(pCfg->ep, t0->z, t0->n); if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } strncpy(pCfg->config, t1->z, t1->n); @@ -3882,65 +3885,13 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer break; } - case TSDB_SQL_CREATE_USER: - case TSDB_SQL_ALTER_USER: { - const char* msg2 = "invalid user/account name"; - const char* msg3 = "name too long"; - const char* msg5 = "invalid user rights"; - const char* msg7 = "not support options"; - - pCmd->command = pInfo->type; - - SUserInfo* pUser = &pInfo->pMiscInfo->user; - SToken* pName = &pUser->user; - SToken* pPwd = &pUser->passwd; - - if (pName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); - } - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); - } - - if (pCmd->command == TSDB_SQL_CREATE_USER) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - if (pUser->type == TSDB_ALTER_USER_PASSWD) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - assert(pPwd->type == TSDB_DATA_TYPE_NULL); - - SToken* pPrivilege = &pUser->privilege; - - if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 1; - } else if (strncasecmp(pPrivilege->z, "read", 4) == 0 && pPrivilege->n == 4) { - pCmd->count = 2; - } else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 3; - } else { - return setInvalidOperatorMsg(pMsgBuf, msg5); - } - } else { - return setInvalidOperatorMsg(pMsgBuf, msg7); - } - } - - break; - } - case TSDB_SQL_CFG_LOCAL: { SMiscInfo *pMiscInfo = pInfo->pMiscInfo; const char *msg = "invalid configure options or values"; // validate the parameter names and options if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); @@ -3995,7 +3946,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } // normalizeSqlNode(pSqlNode); // normalize the column name in each function @@ -4061,21 +4012,22 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } break; } case TSDB_SQL_COMPACT_VNODE:{ const char* msg = "invalid compact"; if (setCompactVnodeInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } break; } + #endif default: - return setInvalidOperatorMsg(pMsgBuf, "not support sql expression"); + return buildInvalidOperationMsg(pMsgBuf, "not support sql expression"); } -#endif + SCatalogReq req = {0}; SMetaData data = {0}; @@ -4114,5 +4066,159 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } + return code; +} + +// todo remove it +static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLen, SMsgBuf* pMsgBuf) { + const char* msg1 = "invalid name"; + const char* msg2 = "wildcard string should be less than %d characters"; + const char* msg3 = "database name too long"; + const char* msg4 = "pattern is invalid"; + const char* msg5 = "database name is empty"; + const char* msg6 = "pattern string is empty"; + + /* + * database prefix in pInfo->pMiscInfo->a[0] + * wildcard in like clause in pInfo->pMiscInfo->a[1] + */ + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; + int16_t showType = pShowInfo->showType; + if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_VGROUP) { + SToken* pDbPrefixToken = &pShowInfo->prefix; + if (pDbPrefixToken->type != 0) { + if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (pDbPrefixToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken); + // if (ret != TSDB_CODE_SUCCESS) { + // return buildInvalidOperationMsg(pMsgBuf, msg1); + // } + } + + // show table/stable like 'xxxx', set the like pattern for show tables + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type != 0) { + if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + pPattern->n = strdequote(pPattern->z); + if (pPattern->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + if (pPattern->n > tsMaxWildCardsLen) { + char tmp[64] = {0}; + sprintf(tmp, msg2, tsMaxWildCardsLen); + return buildInvalidOperationMsg(pMsgBuf, tmp); + } + } + } else if (showType == TSDB_MGMT_TABLE_VNODES) { + if (pShowInfo->prefix.type == 0) { + return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep"); + } + + if (pShowInfo->prefix.type == TK_STRING) { + pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); + } + } + + SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SToken* pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + *output = pShowMsg; + *msgLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + return TSDB_CODE_SUCCESS; +} + +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { + int32_t code = 0; + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf *pMsgBuf = &m; + + *type = pInfo->type; + + switch (pInfo->type) { + case TSDB_SQL_CREATE_USER: + case TSDB_SQL_ALTER_USER: { + const char* msg1 = "not support options"; + const char* msg2 = "invalid user/account name"; + const char* msg3 = "name too long"; + const char* msg4 = "invalid user rights"; + + SUserInfo* pUser = &pInfo->pMiscInfo->user; + SToken* pName = &pUser->user; + SToken* pPwd = &pUser->passwd; + + if (pName->n >= TSDB_USER_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + if (pInfo->type == TSDB_SQL_CREATE_USER) { + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } else { + if (pUser->type == TSDB_ALTER_USER_PASSWD) { + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + assert(pPwd->type == TSDB_DATA_TYPE_NULL); + + SToken* pPrivilege = &pUser->privilege; + if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { + // pCmd->count = 1; + } else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) { + // pCmd->count = 2; + } else { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + } else { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } + + *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_SHOW: { + code = setShowInfo(pInfo, output, outputLen, pMsgBuf); + break; + } + default: + break; + } + return code; } diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 219dd42f78..f63f13d8ac 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -71,8 +71,7 @@ typedef struct SInsertParseContext { const char* pSql; SMsgBuf msg; struct SCatalog* pCatalog; - SMetaData meta; // need release - const STableMeta* pTableMeta; + STableMeta* pTableMeta; SHashObj* pTableBlockHashObj; // data block for each table. need release int32_t totalNum; SInsertStmtInfo* pOutput; @@ -165,29 +164,29 @@ static int32_t skipInsertInto(SInsertParseContext* pCxt) { return TSDB_CODE_SUCCESS; } -static int32_t buildTableName(SInsertParseContext* pCxt, SToken* pStname, SArray* tableNameList) { +static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullDbName, char* tableName) { if (parserValidateIdToken(pStname) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(&pCxt->msg, "invalid table name", pStname->z); } - SName name = {0}; - strcpy(name.dbname, pCxt->pComCxt->pDbname); - strncpy(name.tname, pStname->z, pStname->n); - taosArrayPush(tableNameList, &name); - + char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); + if (NULL != p) { // db.table + strcpy(fullDbName, pCxt->pComCxt->pAcctId); + fullDbName[strlen(pCxt->pComCxt->pAcctId)] = TS_PATH_DELIMITER[0]; + strncpy(fullDbName, pStname->z, p - pStname->z); + strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); + } else { + snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%s.%s", pCxt->pComCxt->pAcctId, pCxt->pComCxt->pDbname); + strncpy(tableName, pStname->z, pStname->n); + } return TSDB_CODE_SUCCESS; } -static int32_t buildMetaReq(SInsertParseContext* pCxt, SToken* pStname, SCatalogReq* pMetaReq) { - pMetaReq->pTableName = taosArrayInit(4, sizeof(SName)); - return buildTableName(pCxt, pStname, pMetaReq->pTableName); -} - static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { - SCatalogReq req; - CHECK_CODE(buildMetaReq(pCxt, pTname, &req)); - CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, NULL, NULL, NULL, NULL, NULL)); //TODO - pCxt->pTableMeta = (STableMeta*)taosArrayGetP(pCxt->meta.pTableMeta, 0); + char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0}; + char tableName[TSDB_TABLE_NAME_LEN] = {0}; + CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName)); + CHECK_CODE(catalogGetTableMeta(pCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta)); return TSDB_CODE_SUCCESS; } @@ -868,12 +867,11 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { .pOutput = *pInfo }; - CHECK_CODE(catalogGetHandle(NULL, &context.pCatalog)); //TODO - if (NULL == context.pTableBlockHashObj) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } + CHECK_CODE(catalogGetHandle(pContext->pClusterId, &context.pCatalog)); CHECK_CODE(skipInsertInto(&context)); CHECK_CODE(parseInsertBody(&context)); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d167f7ad3c..5e08859a66 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -31,25 +31,36 @@ bool qIsInsertSql(const char* pStr, size_t length) { } while (1); } -int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen) { - *pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); - if (*pQueryInfo == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. - } - +int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { SSqlInfo info = doGenerateAST(pStr); if (!info.valid) { strncpy(msg, info.msg, msgLen); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + return terrno; } - struct SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(NULL, &pCatalog); - if (code) { - return code; + if (isDclSqlStatement(&info)) { + int32_t code = qParserValidateDclSqlNode(&info, id, pOutput, outputLen, type, msg, msgLen); + if (code == TSDB_CODE_SUCCESS) { + // do nothing + } + } else { + SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. + return terrno; + } + + struct SCatalog* pCatalog = NULL; + int32_t code = catalogGetHandle(NULL, &pCatalog); + code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen); + if (code == TSDB_CODE_SUCCESS) { + *pOutput = pQueryInfo; + } } - - return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); + + destroySqlInfo(&info); + return TSDB_CODE_SUCCESS; } int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 7fcb7ea304..3a61f5912b 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "parserUtil.h" + #include "taosmsg.h" #include "parser.h" @@ -22,6 +22,8 @@ #include "thash.h" #include "tbuffer.h" #include "parserInt.h" +#include "parserUtil.h" +#include "tmsgtype.h" #include "queryInfoUtil.h" #include "function.h" @@ -97,12 +99,35 @@ int32_t parserValidateIdToken(SToken* pToken) { return TSDB_CODE_SUCCESS; } +int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) { + const char* msg1 = "password can not be empty"; + const char* msg2 = "name or password too long"; + const char* msg3 = "password needs single quote marks enclosed"; + + if (pToken->type != TK_STRING) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + strdequote(pToken->z); + + pToken->n = (uint32_t)strtrim(pToken->z); // trim space before and after passwords + if (pToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + if (pToken->n >= TSDB_USET_PASSWORD_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + return TSDB_CODE_SUCCESS; +} + int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { strncpy(pBuf->buf, msg, pBuf->len); return TSDB_CODE_TSC_INVALID_OPERATION; } -int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { +int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { const char* msgFormat1 = "syntax error near \'%s\'"; const char* msgFormat2 = "syntax error near \'%s\' (%s)"; const char* msgFormat3 = "%s"; @@ -584,21 +609,6 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } -int32_t getFirstInvisibleFieldPos(SQueryStmtInfo* pQueryInfo) { - if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) { - return 0; - } - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (!pField->visible) { - return i; - } - } - - return pQueryInfo->fieldsInfo.numOfOutput; -} - SInternalField* appendFieldInfo(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { assert(pFieldInfo != NULL); pFieldInfo->numOfOutput++; @@ -1583,6 +1593,10 @@ uint32_t convertRelationalOperator(SToken *pToken) { } } +bool isDclSqlStatement(SSqlInfo* pSqlInfo) { + return (pSqlInfo->type != TSDB_SQL_SELECT); +} + #if 0 int32_t tscCreateQueryFromQueryInfo(SQueryStmtInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) { memset(pQueryAttr, 0, sizeof(SQueryAttr)); diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 4b9e586be3..03b76152da 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -6,13 +6,16 @@ SET(CMAKE_CXX_STANDARD 11) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(parserTest ${SOURCE_LIST}) -TARGET_LINK_LIBRARIES( - parserTest - PUBLIC os util common parser catalog transport gtest function planner query -) TARGET_INCLUDE_DIRECTORIES( parserTest PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc" ) + +TARGET_LINK_LIBRARIES( + parserTest + PUBLIC os util common parser catalog transport gtest function planner query +) + +TARGET_LINK_OPTIONS(parserTest PRIVATE -Wl,-wrap,malloc) diff --git a/source/libs/parser/test/insertTest.cpp b/source/libs/parser/test/insertTest.cpp index 9cf48da4eb..5877adf41c 100644 --- a/source/libs/parser/test/insertTest.cpp +++ b/source/libs/parser/test/insertTest.cpp @@ -27,6 +27,27 @@ namespace { } } +extern "C" { + +#include + +void *__real_malloc(size_t); + +void *__wrap_malloc(size_t c) { + // printf("My MALLOC called: %d\n", c); + // void *array[32]; + // int size = backtrace(array, 32); + // char **symbols = backtrace_symbols(array, size); + // for (int i = 0; i < size; ++i) { + // cout << symbols[i] << endl; + // } + // free(symbols); + + return __real_malloc(c); +} + +} + // syntax: // INSERT INTO // tb_name diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 6a402cd620..396e8a12fe 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -29,6 +29,7 @@ #include "taos.h" #include "tdef.h" #include "tvariant.h" +#include "parserUtil.h" namespace { void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_t colId) { @@ -700,5 +701,47 @@ TEST(testCase, function_Test6) { destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, show_user_Test) { + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + char sql1[] = "show users"; + SSqlInfo info1 = doGenerateAST(sql1); + ASSERT_EQ(info1.valid, true); + + void* output = NULL; + int32_t type = 0; + int32_t len = 0; + int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + ASSERT_EQ(code, 0); + + // convert the show command to be the select query + // select name, privilege, create_time, account from information_schema.users; +} + +TEST(testCase, create_user_Test) { + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + char sql[] = {"create user abc pass 'abc'"}; + + SSqlInfo info1 = doGenerateAST(sql); + ASSERT_EQ(info1.valid, true); + + ASSERT_EQ(isDclSqlStatement(&info1), true); + + void* output = NULL; + int32_t type = 0; + int32_t len = 0; + int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + ASSERT_EQ(code, 0); + destroySqlInfo(&info1); } \ No newline at end of file diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 2231c93362..38b399fb0b 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -23,25 +23,23 @@ extern "C" { #include "common.h" #include "tarray.h" #include "planner.h" +#include "parser.h" #include "taosmsg.h" -enum LOGIC_PLAN_E { - LP_SCAN = 1, - LP_SESSION = 2, - LP_STATE = 3, - LP_INTERVAL = 4, - LP_FILL = 5, - LP_AGG = 6, - LP_JOIN = 7, - LP_PROJECT = 8, - LP_DISTINCT = 9, - LP_ORDER = 10 -}; - -typedef struct SQueryNodeBasicInfo { - int32_t type; // operator type - char *name; // operator name -} SQueryNodeBasicInfo; +#define QNODE_TAGSCAN 1 +#define QNODE_TABLESCAN 2 +#define QNODE_PROJECT 3 +#define QNODE_AGGREGATE 4 +#define QNODE_GROUPBY 5 +#define QNODE_LIMIT 6 +#define QNODE_JOIN 7 +#define QNODE_DISTINCT 8 +#define QNODE_SORT 9 +#define QNODE_UNION 10 +#define QNODE_TIMEWINDOW 11 +#define QNODE_SESSIONWINDOW 12 +#define QNODE_STATEWINDOW 13 +#define QNODE_FILL 14 typedef struct SQueryDistPlanNodeInfo { bool stableQuery; // super table query or not @@ -52,8 +50,9 @@ typedef struct SQueryDistPlanNodeInfo { } SQueryDistPlanNodeInfo; typedef struct SQueryTableInfo { - char *tableName; - uint64_t uid; + char *tableName; // to be deleted + uint64_t uid; // to be deleted + STableMetaInfo* pMeta; STimeWindow window; } SQueryTableInfo; @@ -64,50 +63,12 @@ typedef struct SQueryPlanNode { SArray *pExpr; // the query functions or sql aggregations int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information - // previous operator to generated result for current node to process + // children operator to generated result for current node to process // in case of join, multiple prev nodes exist. - SArray *pPrevNodes; // upstream nodes - struct SQueryPlanNode *nextNode; + SArray *pChildren; // upstream nodes + struct SQueryPlanNode *pParent; } SQueryPlanNode; -typedef SSchema SSlotSchema; - -typedef struct SDataBlockSchema { - int32_t index; - SSlotSchema *pSchema; - int32_t numOfCols; // number of columns -} SDataBlockSchema; - -typedef struct SPhyNode { - SQueryNodeBasicInfo info; - SArray *pTargets; // target list to be computed or scanned at this node - SArray *pConditions; // implicitly-ANDed qual conditions - SDataBlockSchema targetSchema; - // children plan to generated result for current node to process - // in case of join, multiple plan nodes exist. - SArray *pChildren; -} SPhyNode; - -typedef struct SScanPhyNode { - SPhyNode node; - uint64_t uid; // unique id of the table -} SScanPhyNode; - -typedef SScanPhyNode STagScanPhyNode; - -typedef SScanPhyNode SSystemTableScanPhyNode; - -typedef struct SMultiTableScanPhyNode { - SScanPhyNode scan; - SArray *pTagsConditions; // implicitly-ANDed tag qual conditions -} SMultiTableScanPhyNode; - -typedef SMultiTableScanPhyNode SMultiTableSeqScanPhyNode; - -typedef struct SProjectPhyNode { - SPhyNode node; -} SProjectPhyNode; - /** * Optimize the query execution plan, currently not implement yet. * @param pQueryNode @@ -138,6 +99,8 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); */ int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag); + /** * Convert to physical plan to string to enable to print it out in the shell. * @param pPhyNode @@ -150,7 +113,7 @@ int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str); * Destroy the query plan object. * @return */ -void* destroyQueryPlan(struct SQueryPlanNode* pQueryNode); +void destroyQueryPlan(struct SQueryPlanNode* pQueryNode); /** * Destroy the physical plan. diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c new file mode 100644 index 0000000000..ce923314bd --- /dev/null +++ b/source/libs/planner/src/logicPlan.c @@ -0,0 +1,604 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "function.h" +#include "os.h" +#include "parser.h" +#include "plannerInt.h" + +typedef struct SFillEssInfo { + int32_t fillType; // fill type + int64_t *val; // fill value +} SFillEssInfo; + +typedef struct SJoinCond { + bool tagExists; // denote if tag condition exists or not + SColumn *tagCond[2]; + SColumn *colCond[2]; +} SJoinCond; + +static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); +static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); + +int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len); +int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { + return 0; +} + +int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { + SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); + assert(taosArrayGetSize(upstream) == 1); + + *pQueryNode = taosArrayGetP(upstream, 0); + + taosArrayDestroy(upstream); + return TSDB_CODE_SUCCESS; +} + +int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { + return 0; +} + +void destroyQueryPlan(SQueryPlanNode* pQueryNode) { + if (pQueryNode == NULL) { + return; + } + + doDestroyQueryNode(pQueryNode); +} + +//====================================================================================================================== + +static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, + SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { + SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); + + pNode->info.type = type; + pNode->info.name = strdup(name); + + pNode->numOfExpr = numOfOutput; + pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); + + for(int32_t i = 0; i < numOfOutput; ++i) { + taosArrayPush(pNode->pExpr, &pExpr[i]); + } + + pNode->pChildren = taosArrayInit(4, POINTER_BYTES); + for(int32_t i = 0; i < numOfPrev; ++i) { + taosArrayPush(pNode->pChildren, &prev[i]); + } + + switch(type) { + case QNODE_TAGSCAN: + case QNODE_TABLESCAN: { + SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); + memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); + info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName); + pNode->pExtInfo = info; + break; + } + + case QNODE_TIMEWINDOW: { + SInterval* pInterval = calloc(1, sizeof(SInterval)); + pNode->pExtInfo = pInterval; + memcpy(pInterval, pExtInfo, sizeof(SInterval)); + break; + } + + case QNODE_STATEWINDOW: { + SColumn* psw = calloc(1, sizeof(SColumn)); + pNode->pExtInfo = psw; + memcpy(psw, pExtInfo, sizeof(SColumn)); + break; + } + + case QNODE_SESSIONWINDOW: { + SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow)); + pNode->pExtInfo = pSessionWindow; + memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow)); + break; + } + + case QNODE_GROUPBY: { + SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; + + SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); + pGroupbyExpr->groupbyTag = p->groupbyTag; + pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); + + pNode->pExtInfo = pGroupbyExpr; + break; + } + + case QNODE_FILL: { // todo !! + pNode->pExtInfo = pExtInfo; + break; + } + + case QNODE_LIMIT: { + pNode->pExtInfo = calloc(1, sizeof(SLimit)); + memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); + break; + } + + case QNODE_SORT: { + pNode->pExtInfo = taosArrayDup(pExtInfo); + break; + } + + default: + break; + } + + return pNode; +} + +static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, + SArray* pExprs, SArray* tableCols) { + if (pQueryInfo->info.onlyTagQuery) { + int32_t num = (int32_t) taosArrayGetSize(pExprs); + SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info); + + if (pQueryInfo->info.distinct) { + pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); + } + + return pNode; + } + + SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); + + if (pQueryInfo->info.projectionQuery) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL); + } else { + STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); + + // table source column projection, generate the projection expr + int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); + SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(tableCols, i); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, pCol); + SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); + SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); + pExpr[i] = p; + } + + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); + tfree(pExpr); + } + + return pNode; +} + +static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { + // group by column not by tag + size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); + + // check for aggregation + int32_t level = getExprFunctionLevel(pQueryInfo); + + for(int32_t i = level - 1; i >= 0; --i) { + SArray* p = pQueryInfo->exprList[i]; + size_t num = taosArrayGetSize(p); + + bool aggregateFunc = false; + for(int32_t j = 0; j < num; ++j) { + SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); + if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { + continue; + } + + aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); + if (aggregateFunc) { + break; + } + } + + if (aggregateFunc) { + if (pQueryInfo->interval.interval > 0) { + pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval); + } else if (pQueryInfo->sessionWindow.gap > 0) { + pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow); + } else if (pQueryInfo->stateWindow.col.info.colId > 0) { + pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow); + } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { + pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr); + } else { + pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL); + } + } else { + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); + } + } + + if (pQueryInfo->havingFieldNum > 0) { +// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); +// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL); + } + + if (pQueryInfo->fillType != TSDB_FILL_NONE) { + SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); + pInfo->fillType = pQueryInfo->fillType; + pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); + memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); + + SArray* p = pQueryInfo->exprList[0]; // top expression in select clause + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); + } + + if (pQueryInfo->order != NULL) { + SArray* pList = pQueryInfo->exprList[0]; + pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order); + } + + if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { + pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); + } + + return pNode; +} + +static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, + SArray* tableCols) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); + + SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; + + // handle the only tag query + SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); + if (pQueryInfo->info.onlyTagQuery) { + tfree(info.tableName); + return pNode; + } + + SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); + tfree(info.tableName); + return pNode1; +} + +static bool isAllAggExpr(SArray* pList) { + assert(pList != NULL); + + for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { + SExprInfo* p = taosArrayGetP(pList, k); + if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { + return false; + } + } + + return true; +} + +SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { + SArray* upstream = NULL; + + if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause + upstream = taosArrayInit(4, POINTER_BYTES); + + size_t size = taosArrayGetSize(pQueryInfo->pUpstream); + for(int32_t i = 0; i < size; ++i) { + SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); + SArray* p = createQueryPlanImpl(pq); + taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); + } + } + + if (pQueryInfo->numOfTables > 1) { // it is a join query + // 1. separate the select clause according to table + taosArrayDestroy(upstream); + upstream = taosArrayInit(5, POINTER_BYTES); + + for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; + uint64_t uid = pTableMetaInfo->pTableMeta->uid; + + SArray* exprList = taosArrayInit(4, POINTER_BYTES); + if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; +// dropAllExprInfo(exprList); + exit(-1); + } + + // 2. create the query execution node + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; + + // 3. get the required table column list + SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); + columnListCopy(tableColumnList, pQueryInfo->colList, uid); + + // 4. add the projection query node + SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); + columnListDestroy(tableColumnList); +// dropAllExprInfo(exprList); + taosArrayPush(upstream, &pNode); + } + + // 3. add the join node here + SQueryTableInfo info = {0}; + int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); + SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, + pQueryInfo->exprList[0]->pData, num, NULL); + + // 4. add the aggregation or projection execution node + pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } else { // only one table, normal query process + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } + + return upstream; +} + +static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { + tfree(pQueryNode->pExtInfo); + tfree(pQueryNode->pSchema); + tfree(pQueryNode->info.name); +// dropAllExprInfo(pQueryNode->pExpr); + + if (pQueryNode->pChildren != NULL) { + int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); + for(int32_t i = 0; i < size; ++i) { + SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i); + doDestroyQueryNode(p); + } + + taosArrayDestroy(pQueryNode->pChildren); + } + + tfree(pQueryNode); +} + +static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { + if (level > 0) { + sprintf(buf + totalLen, "%*c", level, ' '); + totalLen += level; + } + + int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); + int32_t len = len1 + totalLen; + + switch(pQueryNode->info.type) { + case QNODE_TABLESCAN: { + SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid, + pInfo->window.skey, pInfo->window.ekey); + assert(len1 > 0); + len += len1; + + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); + len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); + + assert(len1 > 0); + len += len1; + } + + len1 = sprintf(buf + len, "\n"); + assert(len1 > 0); + + len += len1; + break; + } + + case QNODE_PROJECT: { + len1 = sprintf(buf + len, "cols:"); + assert(len1 > 0); + len += len1; + + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ")"); + len += len1; + + // todo print filter info + len1 = sprintf(buf + len, " filters:(nil)\n"); + len += len1; + break; + } + + case QNODE_AGGREGATE: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ")\n"); + len += len1; + + break; + } + + case QNODE_TIMEWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ") "); + len += len1; + + SInterval* pInterval = pQueryNode->pExtInfo; + + // todo dynamic return the time precision + len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n", + pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, + TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); + len += len1; + + break; + } + + case QNODE_STATEWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ") "); + len += len1; + + SColumn* pCol = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId); + len += len1; + break; + } + + case QNODE_SESSIONWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + + len1 = sprintf(buf + len, ") "); + len += len1; + + struct SSessionWindow* ps = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); + len += len1; + break; + } + + case QNODE_GROUPBY: { + len = printExprInfo(buf, pQueryNode, len); + + SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, ") groupby_col: "); + len += len1; + + for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { + SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); + len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); + len += len1; + } + + len += sprintf(buf + len, "\n"); + break; + } + + case QNODE_FILL: { + SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%d", pEssInfo->fillType); + len += len1; + + if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { + len1 = sprintf(buf + len, ", val:"); + len += len1; + + // todo get the correct fill data type + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]); + len += len1; + + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + + case QNODE_LIMIT: { + SLimit* pVal = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset); + len += len1; + break; + } + + case QNODE_DISTINCT: + case QNODE_TAGSCAN: { + len1 = sprintf(buf + len, "cols: "); + len += len1; + + len = printExprInfo(buf, pQueryNode, len); + + len1 = sprintf(buf + len, ")\n"); + len += len1; + + break; + } + + case QNODE_SORT: { + len1 = sprintf(buf + len, "cols:"); + len += len1; + + SArray* pSort = pQueryNode->pExtInfo; + for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) { + SOrder* p = taosArrayGet(pSort, i); + len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC"); + + len += len1; + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + + case QNODE_JOIN: { + // print join condition + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + } + + return len; +} + +int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len) { + int32_t len1 = 0; + + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); + + SSqlExpr* pExpr = &pExprInfo->base; + len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + assert(len1 > 0); + + len += len1; + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + + return len; +} + +int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { + int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); + + for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) { + SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i); + int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); + len = len1; + } + + return len; +} + +int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { + assert(pQueryNode); + + *str = calloc(1, 4096); + + int32_t len = sprintf(*str, "===== logic plan =====\n"); + queryPlanToStringImpl(*str, pQueryNode, 0, len); + + return TSDB_CODE_SUCCESS; +} + +SQueryPlanNode* queryPlanFromString() { + return NULL; +} diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 2bdc159af8..277703f5c0 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -14,23 +14,208 @@ */ #include "plannerInt.h" +#include "parser.h" -SPhyNode* createScanNode(SQueryPlanNode* pPlanNode) { - return NULL; +#define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan +#define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _ + +static const char* gOpName[] = { + "Unknown", +#define INCLUDE_AS_NAME +#include "plannerOp.h" +#undef INCLUDE_AS_NAME +}; + +typedef struct SPlanContext { + struct SCatalog* pCatalog; + struct SQueryDag* pDag; + SSubplan* pCurrentSubplan; + SSubplanId nextId; +} SPlanContext; + +static void toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { + SWAP(dataBlockSchema->pSchema, pPlanNode->pSchema, SSchema*); + dataBlockSchema->numOfCols = pPlanNode->numOfCols; } -SPhyNode* createPhyNode(SQueryPlanNode* node) { - switch (node->info.type) { - case LP_SCAN: - return createScanNode(node); +static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t size) { + SPhyNode* node = (SPhyNode*)calloc(1, size); + node->info.type = type; + node->info.name = gOpName[type]; + SWAP(node->pTargets, pPlanNode->pExpr, SArray*); + toDataBlockSchema(pPlanNode, &(node->targetSchema)); +} + +static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { + SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size); + node->uid = pTable->pMeta->pTableMeta->uid; + node->tableType = pTable->pMeta->pTableMeta->tableType; + return (SPhyNode*)node; +} + +static SPhyNode* createPseudoScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { + return initScanNode(pPlanNode, pTable, op, sizeof(SScanPhyNode)); +} + +static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { + SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; + return createPseudoScanNode(pPlanNode, pTable, OP_TagScan); +} + +static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + // todo + return MASTER_SCAN; +} + +static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { + STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode)); + node->scanFlag = getScanFlag(pPlanNode, pTable); + node->window = pTable->window; + // todo tag cond + return (SPhyNode*)node; +} + +static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); +} + +static bool isSystemTable(SQueryTableInfo* pTable) { + // todo + return false; +} + +static bool needSeqScan(SQueryPlanNode* pPlanNode) { + // todo + return false; +} + +static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + if (isSystemTable(pTable)) { + return createPseudoScanNode(pPlanNode, pTable, OP_SystemTableScan); + } else if (needSeqScan(pPlanNode)) { + return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan); } - return NULL; + return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan); } -SPhyNode* createSubplan(SQueryPlanNode* pSubquery) { - return NULL; +static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { + SSubplan* subplan = calloc(1, sizeof(SSubplan)); + subplan->id = pCxt->nextId; + ++(pCxt->nextId.subplanId); + subplan->type = type; + subplan->level = 0; + if (NULL != pCxt->pCurrentSubplan) { + subplan->level = pCxt->pCurrentSubplan->level + 1; + if (NULL == pCxt->pCurrentSubplan->pChildern) { + pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + } + taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); + subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); + } + SArray* currentLevel; + if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { + currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); + } else { + currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); + } + taosArrayPush(currentLevel, &subplan); + pCxt->pCurrentSubplan = subplan; + return subplan; } -int32_t createDag(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { - return 0; +static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { + epSet->inUse = 0; // todo + epSet->numOfEps = vg->numOfEps; + for (int8_t i = 0; i < vg->numOfEps; ++i) { + epSet->port[i] = vg->epAddr[i].port; + strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn); + } + return; +} + +static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList; + for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { + STORE_CURRENT_SUBPLAN(pCxt); + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); + vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); + subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + RECOVERY_CURRENT_SUBPLAN(pCxt); + } + return pCxt->nextId.templateId++; +} + +static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) { + SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode)); + node->srcTemplateId = srcTemplateId; + return (SPhyNode*)node; +} + +static bool needMultiNodeScan(SQueryTableInfo* pTable) { + // todo system table, for instance, user_tables + return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType); +} + +static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; + if (needMultiNodeScan(pTable)) { + return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); + } + return createSingleTableScanNode(pPlanNode, pTable); +} + +static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SPhyNode* node = NULL; + switch (pPlanNode->info.type) { + case QNODE_TAGSCAN: + node = createTagScanNode(pPlanNode); + break; + case QNODE_TABLESCAN: + node = createTableScanNode(pCxt, pPlanNode); + break; + default: + assert(false); + } + if (pPlanNode->pChildren != NULL && taosArrayGetSize(pPlanNode->pChildren) > 0) { + node->pChildren = taosArrayInit(4, POINTER_BYTES); + size_t size = taosArrayGetSize(pPlanNode->pChildren); + for(int32_t i = 0; i < size; ++i) { + SPhyNode* child = createPhyNode(pCxt, taosArrayGet(pPlanNode->pChildren, i)); + child->pParent = node; + taosArrayPush(node->pChildren, &child); + } + } + return node; +} + +static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { + SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); + ++(pCxt->nextId.templateId); + subplan->pNode = createPhyNode(pCxt, pRoot); + SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + taosArrayPush(l0, &subplan); + taosArrayPush(pCxt->pDag->pSubplans, &l0); + // todo deal subquery +} + +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag) { + SPlanContext context = { + .pCatalog = pCatalog, + .pDag = calloc(1, sizeof(SQueryDag)), + .pCurrentSubplan = NULL, + .nextId = {0} // todo queryid + }; + if (NULL == context.pDag) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + context.pDag->pSubplans = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + createSubplanByLevel(&context, pQueryNode); + *pDag = context.pDag; + return TSDB_CODE_SUCCESS; +} + +int32_t subPlanToString(struct SSubplan *pPhyNode, char** str) { + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index e54b847230..744a849e2d 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -13,620 +13,31 @@ * along with this program. If not, see . */ -#include "function.h" -#include "os.h" #include "parser.h" #include "plannerInt.h" -#define QNODE_TAGSCAN 1 -#define QNODE_TABLESCAN 2 -#define QNODE_PROJECT 3 -#define QNODE_AGGREGATE 4 -#define QNODE_GROUPBY 5 -#define QNODE_LIMIT 6 -#define QNODE_JOIN 7 -#define QNODE_DISTINCT 8 -#define QNODE_SORT 9 -#define QNODE_UNION 10 -#define QNODE_TIMEWINDOW 11 -#define QNODE_SESSIONWINDOW 12 -#define QNODE_STATEWINDOW 13 -#define QNODE_FILL 14 - -typedef struct SFillEssInfo { - int32_t fillType; // fill type - int64_t *val; // fill value -} SFillEssInfo; - -typedef struct SJoinCond { - bool tagExists; // denote if tag condition exists or not - SColumn *tagCond[2]; - SColumn *colCond[2]; -} SJoinCond; - -static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); -static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); - -int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len); -int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { - return 0; +void qDestroyQueryDag(struct SQueryDag* pDag) { + // todo } -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { - SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); - assert(taosArrayGetSize(upstream) == 1); - - *pQueryNode = taosArrayGetP(upstream, 0); - - taosArrayDestroy(upstream); +int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) { + SQueryPlanNode* logicPlan; + int32_t code = createQueryPlan(pQueryInfo, &logicPlan); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + return code; + } + code = optimizeQueryPlan(logicPlan); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + return code; + } + code = createDag(logicPlan, NULL, pDag); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + qDestroyQueryDag(*pDag); + return code; + } + destroyQueryPlan(logicPlan); return TSDB_CODE_SUCCESS; } - -int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { - return 0; -} - -int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { - - return 0; -} - -int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str) { - return 0; -} - -void* destroyQueryPlan(SQueryPlanNode* pQueryNode) { - if (pQueryNode == NULL) { - return NULL; - } - - doDestroyQueryNode(pQueryNode); - return NULL; -} - -void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode) { - return NULL; -} - -//====================================================================================================================== - -static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { - SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); - - pNode->info.type = type; - pNode->info.name = strdup(name); - - pNode->numOfExpr = numOfOutput; - pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); - - for(int32_t i = 0; i < numOfOutput; ++i) { - taosArrayPush(pNode->pExpr, &pExpr[i]); - } - - pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES); - for(int32_t i = 0; i < numOfPrev; ++i) { - taosArrayPush(pNode->pPrevNodes, &prev[i]); - } - - switch(type) { - case QNODE_TABLESCAN: { - SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); - memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); - info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName); - pNode->pExtInfo = info; - break; - } - - case QNODE_TIMEWINDOW: { - SInterval* pInterval = calloc(1, sizeof(SInterval)); - pNode->pExtInfo = pInterval; - memcpy(pInterval, pExtInfo, sizeof(SInterval)); - break; - } - - case QNODE_STATEWINDOW: { - SColumn* psw = calloc(1, sizeof(SColumn)); - pNode->pExtInfo = psw; - memcpy(psw, pExtInfo, sizeof(SColumn)); - break; - } - - case QNODE_SESSIONWINDOW: { - SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow)); - pNode->pExtInfo = pSessionWindow; - memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow)); - break; - } - - case QNODE_GROUPBY: { - SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; - - SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - pGroupbyExpr->groupbyTag = p->groupbyTag; - pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); - - pNode->pExtInfo = pGroupbyExpr; - break; - } - - case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; - break; - } - - case QNODE_LIMIT: { - pNode->pExtInfo = calloc(1, sizeof(SLimit)); - memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); - break; - } - - case QNODE_SORT: { - pNode->pExtInfo = taosArrayDup(pExtInfo); - break; - } - - default: - break; - } - - return pNode; -} - -static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, - SArray* pExprs, SArray* tableCols) { - if (pQueryInfo->info.onlyTagQuery) { - int32_t num = (int32_t) taosArrayGetSize(pExprs); - SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, NULL); - - if (pQueryInfo->info.distinct) { - pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); - } - - return pNode; - } - - SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); - - if (pQueryInfo->info.projectionQuery) { - int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL); - } else { - STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); - - // table source column projection, generate the projection expr - int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(tableCols, i); - - SSourceParam param = {0}; - addIntoSourceParam(¶m, NULL, pCol); - SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); - SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); - pExpr[i] = p; - } - - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); - tfree(pExpr); - } - - return pNode; -} - -static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { - // group by column not by tag - size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); - - // check for aggregation - int32_t level = getExprFunctionLevel(pQueryInfo); - - for(int32_t i = level - 1; i >= 0; --i) { - SArray* p = pQueryInfo->exprList[i]; - size_t num = taosArrayGetSize(p); - - bool aggregateFunc = false; - for(int32_t j = 0; j < num; ++j) { - SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); - if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { - continue; - } - - aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); - if (aggregateFunc) { - break; - } - } - - if (aggregateFunc) { - if (pQueryInfo->interval.interval > 0) { - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval); - } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow); - } else if (pQueryInfo->stateWindow.col.info.colId > 0) { - pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow); - } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr); - } else { - pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL); - } - } else { - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); - } - } - - if (pQueryInfo->havingFieldNum > 0) { -// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); -// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL); - } - - if (pQueryInfo->fillType != TSDB_FILL_NONE) { - SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); - pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); - memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); - - SArray* p = pQueryInfo->exprList[0]; // top expression in select clause - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); - } - - if (pQueryInfo->order != NULL) { - SArray* pList = pQueryInfo->exprList[0]; - pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order); - } - - if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { - pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); - } - - return pNode; -} - -static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, - SArray* tableCols) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); - - SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; - - // handle the only tag query - SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); - if (pQueryInfo->info.onlyTagQuery) { - tfree(info.tableName); - return pNode; - } - - SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); - tfree(info.tableName); - return pNode1; -} - -static bool isAllAggExpr(SArray* pList) { - assert(pList != NULL); - - for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { - SExprInfo* p = taosArrayGetP(pList, k); - if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { - return false; - } - } - - return true; -} - -SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { - SArray* upstream = NULL; - - if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause - upstream = taosArrayInit(4, POINTER_BYTES); - - size_t size = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < size; ++i) { - SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); - SArray* p = createQueryPlanImpl(pq); - taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); - } - } - - if (pQueryInfo->numOfTables > 1) { // it is a join query - // 1. separate the select clause according to table - taosArrayDestroy(upstream); - upstream = taosArrayInit(5, POINTER_BYTES); - - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; - uint64_t uid = pTableMetaInfo->pTableMeta->uid; - - SArray* exprList = taosArrayInit(4, POINTER_BYTES); - if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; -// dropAllExprInfo(exprList); - exit(-1); - } - - // 2. create the query execution node - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; - - // 3. get the required table column list - SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); - columnListCopy(tableColumnList, pQueryInfo->colList, uid); - - // 4. add the projection query node - SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); - columnListDestroy(tableColumnList); -// dropAllExprInfo(exprList); - taosArrayPush(upstream, &pNode); - } - - // 3. add the join node here - SQueryTableInfo info = {0}; - int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); - SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, - pQueryInfo->exprList[0]->pData, num, NULL); - - // 4. add the aggregation or projection execution node - pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } else { // only one table, normal query process - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } - - return upstream; -} - -static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - tfree(pQueryNode->pExtInfo); - tfree(pQueryNode->pSchema); - tfree(pQueryNode->info.name); -// dropAllExprInfo(pQueryNode->pExpr); - - if (pQueryNode->pPrevNodes != NULL) { - int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes); - for(int32_t i = 0; i < size; ++i) { - SQueryPlanNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i); - doDestroyQueryNode(p); - } - - taosArrayDestroy(pQueryNode->pPrevNodes); - } - - tfree(pQueryNode); -} - -static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { - if (level > 0) { - sprintf(buf + totalLen, "%*c", level, ' '); - totalLen += level; - } - - int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); - int32_t len = len1 + totalLen; - - switch(pQueryNode->info.type) { - case QNODE_TABLESCAN: { - SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid, - pInfo->window.skey, pInfo->window.ekey); - assert(len1 > 0); - len += len1; - - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); - len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); - - assert(len1 > 0); - len += len1; - } - - len1 = sprintf(buf + len, "\n"); - assert(len1 > 0); - - len += len1; - break; - } - - case QNODE_PROJECT: { - len1 = sprintf(buf + len, "cols:"); - assert(len1 > 0); - len += len1; - - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ")"); - len += len1; - - // todo print filter info - len1 = sprintf(buf + len, " filters:(nil)\n"); - len += len1; - break; - } - - case QNODE_AGGREGATE: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ")\n"); - len += len1; - - break; - } - - case QNODE_TIMEWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ") "); - len += len1; - - SInterval* pInterval = pQueryNode->pExtInfo; - - // todo dynamic return the time precision - len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n", - pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, - TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); - len += len1; - - break; - } - - case QNODE_STATEWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ") "); - len += len1; - - SColumn* pCol = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId); - len += len1; - break; - } - - case QNODE_SESSIONWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - - len1 = sprintf(buf + len, ") "); - len += len1; - - struct SSessionWindow* ps = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); - len += len1; - break; - } - - case QNODE_GROUPBY: { - len = printExprInfo(buf, pQueryNode, len); - - SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, ") groupby_col: "); - len += len1; - - for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { - SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); - len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); - len += len1; - } - - len += sprintf(buf + len, "\n"); - break; - } - - case QNODE_FILL: { - SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%d", pEssInfo->fillType); - len += len1; - - if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { - len1 = sprintf(buf + len, ", val:"); - len += len1; - - // todo get the correct fill data type - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]); - len += len1; - - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_LIMIT: { - SLimit* pVal = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset); - len += len1; - break; - } - - case QNODE_DISTINCT: - case QNODE_TAGSCAN: { - len1 = sprintf(buf + len, "cols: "); - len += len1; - - len = printExprInfo(buf, pQueryNode, len); - - len1 = sprintf(buf + len, ")\n"); - len += len1; - - break; - } - - case QNODE_SORT: { - len1 = sprintf(buf + len, "cols:"); - len += len1; - - SArray* pSort = pQueryNode->pExtInfo; - for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) { - SOrder* p = taosArrayGet(pSort, i); - len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC"); - - len += len1; - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_JOIN: { - // print join condition - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - } - - return len; -} - -int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len) { - int32_t len1 = 0; - - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - - SSqlExpr* pExpr = &pExprInfo->base; - len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - assert(len1 > 0); - - len += len1; - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - - return len; -} - -int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { - int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); - - for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) { - SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i); - int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); - len = len1; - } - - return len; -} - -int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { - assert(pQueryNode); - - *str = calloc(1, 4096); - - int32_t len = sprintf(*str, "===== logic plan =====\n"); - queryPlanToStringImpl(*str, pQueryNode, 0, len); - - return TSDB_CODE_SUCCESS; -} - -SQueryPlanNode* queryPlanFromString() { - return NULL; -} diff --git a/source/libs/query/inc/queryInt.h b/source/libs/query/inc/queryInt.h index f3204b3785..75c1e134cd 100644 --- a/source/libs/query/inc/queryInt.h +++ b/source/libs/query/inc/queryInt.h @@ -21,17 +21,6 @@ extern "C" { #endif -#include "tlog.h" - -extern int32_t qDebugFlag; - -#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) -#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", qDebugFlag, __VA_ARGS__); }} while(0) #ifdef __cplusplus } diff --git a/source/libs/query/src/querymsg.c b/source/libs/query/src/querymsg.c index c5864fd41b..7409922149 100644 --- a/source/libs/query/src/querymsg.c +++ b/source/libs/query/src/querymsg.c @@ -21,17 +21,6 @@ int32_t (*queryBuildMsg[TSDB_MSG_TYPE_MAX])(void* input, char **msg, int32_t msg int32_t (*queryProcessMsgRsp[TSDB_MSG_TYPE_MAX])(void* output, char *msg, int32_t msgSize) = {0}; - -int32_t queryBuildVgroupListReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { - if (NULL == msg || NULL == msgLen) { - return TSDB_CODE_TSC_INVALID_INPUT; - } - - *msgLen = 0; - - return TSDB_CODE_SUCCESS; -} - int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -81,8 +70,7 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms strncpy(bMsg->db, bInput->db, sizeof(bMsg->db)); bMsg->db[sizeof(bMsg->db) - 1] = 0; - bMsg->vgroupVersion = bInput->vgroupVersion; - bMsg->dbGroupVersion = bInput->dbGroupVersion; + bMsg->vgVersion = bInput->vgVersion; *msgLen = (int32_t)sizeof(*bMsg); @@ -90,58 +78,12 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms } - -int32_t queryProcessVgroupListRsp(void* output, char *msg, int32_t msgSize) { - if (NULL == output || NULL == msg || msgSize <= 0) { - return TSDB_CODE_TSC_INVALID_INPUT; - } - - SVgroupListRspMsg *pRsp = (SVgroupListRspMsg *)msg; - - pRsp->vgroupNum = htonl(pRsp->vgroupNum); - pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); - - if (pRsp->vgroupNum < 0) { - qError("vgroup number[%d] in rsp is invalid", pRsp->vgroupNum); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - if (pRsp->vgroupVersion < 0) { - qError("vgroup vgroupVersion[%d] in rsp is invalid", pRsp->vgroupVersion); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - if (msgSize != (pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp))) { - qError("vgroup list msg size mis-match, msgSize:%d, vgroup number:%d", msgSize, pRsp->vgroupNum); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - // keep SVgroupListInfo/SVgroupListRspMsg the same - *(SVgroupListInfo **)output = (SVgroupListInfo *)msg; - - if (pRsp->vgroupNum == 0) { - return TSDB_CODE_SUCCESS; - } - - for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { - pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); - } - } - - return TSDB_CODE_SUCCESS; -} - - - - int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { if (NULL == output || NULL == msg || msgSize <= 0) { return TSDB_CODE_TSC_INVALID_INPUT; } - SUseDbRspMsg *pRsp = (SUseDbRspMsg *)msg; + SUseDbRsp *pRsp = (SUseDbRsp *)msg; SUseDbOutput *pOut = (SUseDbOutput *)output; int32_t code = 0; @@ -150,104 +92,52 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - pRsp->vgroupVersion = htonl(pRsp->vgroupVersion); - pRsp->dbVgroupVersion = htonl(pRsp->dbVgroupVersion); + pRsp->vgVersion = htonl(pRsp->vgVersion); + pRsp->vgNum = htonl(pRsp->vgNum); - pRsp->vgroupNum = htonl(pRsp->vgroupNum); - pRsp->dbVgroupNum = htonl(pRsp->dbVgroupNum); - - if (pRsp->vgroupNum < 0) { - qError("invalid vgroup number[%d]", pRsp->vgroupNum); + if (pRsp->vgNum < 0) { + qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum); return TSDB_CODE_TSC_INVALID_VALUE; } - if (pRsp->dbVgroupNum < 0) { - qError("invalid db vgroup number[%d]", pRsp->dbVgroupNum); - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t expectSize = pRsp->vgroupNum * sizeof(pRsp->vgroupInfo[0]) + pRsp->dbVgroupNum * sizeof(int32_t) + sizeof(*pRsp); + int32_t expectSize = pRsp->vgNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp); if (msgSize != expectSize) { - qError("vgroup list msg size mis-match, msgSize:%d, expected:%d, vgroup number:%d, db vgroup number:%d", msgSize, expectSize, pRsp->vgroupNum, pRsp->dbVgroupNum); + qError("use db rsp size mis-match, msgSize:%d, expected:%d, vgnumber:%d", msgSize, expectSize, pRsp->vgNum); return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - if (pRsp->vgroupVersion < 0) { - qInfo("no new vgroup list info"); - if (pRsp->vgroupNum != 0) { - qError("invalid vgroup number[%d] for no new vgroup list case", pRsp->vgroupNum); - return TSDB_CODE_TSC_INVALID_VALUE; - } - } else { - int32_t s = sizeof(*pOut->vgroupList) + sizeof(pOut->vgroupList->vgroupInfo[0]) * pRsp->vgroupNum; - pOut->vgroupList = calloc(1, s); - if (NULL == pOut->vgroupList) { - qError("calloc size[%d] failed", s); - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - pOut->vgroupList->vgroupNum = pRsp->vgroupNum; - pOut->vgroupList->vgroupVersion = pRsp->vgroupVersion; - - for (int32_t i = 0; i < pRsp->vgroupNum; ++i) { - pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = htonl(pRsp->vgroupInfo[i].epAddr[n].port); - } - - memcpy(&pOut->vgroupList->vgroupInfo[i], &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i])); - } + pOut->dbVgroup.vgVersion = pRsp->vgVersion; + pOut->dbVgroup.hashMethod = pRsp->hashMethod; + pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pOut->dbVgroup.vgInfo) { + qError("hash init[%d] failed", pRsp->vgNum); + return TSDB_CODE_TSC_OUT_OF_MEMORY; } - int32_t *vgIdList = (int32_t *)((char *)pRsp->vgroupInfo + sizeof(pRsp->vgroupInfo[0]) * pRsp->vgroupNum); + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + pRsp->vgroupInfo[i].vgId = htonl(pRsp->vgroupInfo[i].vgId); + pRsp->vgroupInfo[i].hashBegin = htonl(pRsp->vgroupInfo[i].hashBegin); + pRsp->vgroupInfo[i].hashEnd = htonl(pRsp->vgroupInfo[i].hashEnd); + + for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { + pRsp->vgroupInfo[i].epAddr[n].port = htons(pRsp->vgroupInfo[i].epAddr[n].port); + } + + if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { + qError("hash push failed"); + goto _return; + } + } memcpy(pOut->db, pRsp->db, sizeof(pOut->db)); - - if (pRsp->dbVgroupVersion < 0) { - qInfo("no new vgroup info for db[%s]", pRsp->db); - } else { - pOut->dbVgroup = calloc(1, sizeof(*pOut->dbVgroup)); - if (NULL == pOut->dbVgroup) { - qError("calloc size[%d] failed", (int32_t)sizeof(*pOut->dbVgroup)); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _exit; - } - - pOut->dbVgroup->vgId = taosArrayInit(pRsp->dbVgroupNum, sizeof(int32_t)); - if (NULL == pOut->dbVgroup->vgId) { - qError("taosArrayInit size[%d] failed", pRsp->dbVgroupNum); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto _exit; - } - - pOut->dbVgroup->vgroupVersion = pRsp->dbVgroupVersion; - pOut->dbVgroup->hashRange = htonl(pRsp->dbHashRange); - pOut->dbVgroup->hashType = htonl(pRsp->dbHashType); - - if (pOut->dbVgroup->hashRange < 0) { - qError("invalid hashRange[%d] for db[%s]", pOut->dbVgroup->hashRange, pRsp->db); - code = TSDB_CODE_TSC_INVALID_INPUT; - goto _exit; - } - - for (int32_t i = 0; i < pRsp->dbVgroupNum; ++i) { - *(vgIdList + i) = htonl(*(vgIdList + i)); - - taosArrayPush(pOut->dbVgroup->vgId, vgIdList + i) ; - } - } return code; -_exit: - if (pOut->dbVgroup && pOut->dbVgroup->vgId) { - taosArrayDestroy(pOut->dbVgroup->vgId); - pOut->dbVgroup->vgId = NULL; +_return: + if (pOut) { + tfree(pOut->dbVgroup.vgInfo); } - tfree(pOut->dbVgroup); - tfree(pOut->vgroupList); - return code; } @@ -375,11 +265,9 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { void msgInit() { queryBuildMsg[TSDB_MSG_TYPE_TABLE_META] = queryBuildTableMetaReqMsg; - queryBuildMsg[TSDB_MSG_TYPE_VGROUP_LIST] = queryBuildVgroupListReqMsg; queryBuildMsg[TSDB_MSG_TYPE_USE_DB] = queryBuildUseDbMsg; queryProcessMsgRsp[TSDB_MSG_TYPE_TABLE_META] = queryProcessTableMetaRsp; - queryProcessMsgRsp[TSDB_MSG_TYPE_VGROUP_LIST] = queryProcessVgroupListRsp; queryProcessMsgRsp[TSDB_MSG_TYPE_USE_DB] = queryProcessUseDBRsp; /* @@ -417,7 +305,7 @@ void msgInit() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -435,7 +323,7 @@ void msgInit() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index 770a6b02c2..6675b7f5ec 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -9,5 +9,5 @@ target_include_directories( target_link_libraries( scheduler - PRIVATE os util planner + PRIVATE os util planner common query ) \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index b1b128e200..8e30ce1403 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -24,15 +24,57 @@ extern "C" { #include "tarray.h" #include "planner.h" #include "scheduler.h" +#include "thash.h" + +#define SCHEDULE_DEFAULT_JOB_NUMBER 1000 + +enum { + SCH_STATUS_NOT_START = 1, + SCH_STATUS_EXECUTING, + SCH_STATUS_SUCCEED, + SCH_STATUS_FAILED, + SCH_STATUS_CANCELLING, + SCH_STATUS_CANCELLED +}; + +typedef struct SSchedulerMgmt { + SHashObj *Jobs; // key: queryId, value: SQueryJob* +} SSchedulerMgmt; + +typedef struct SQueryTask { + uint64_t taskId; // task id + char *pSubplan; // operator tree + int8_t status; // task status + SQueryProfileSummary summary; // task execution summary +} SQueryTask; + +typedef struct SQueryLevel { + int8_t status; + int32_t taskNum; + + SArray *subTasks; // Element is SQueryTask + SArray *subPlans; // Element is SSubplan +} SQueryLevel; + +typedef struct SQueryJob { + uint64_t queryId; + int32_t levelNum; + int32_t levelIdx; + int8_t status; + SQueryProfileSummary summary; + + SArray *levels; // Element is SQueryLevel, starting from 0. + SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. +} SQueryJob; + + +#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { return _code; } } while (0) +#define SCH_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); return _code; } } while (0) +#define SCH_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { goto _return; } } while (0) -typedef struct SQuery { - SArray **pSubquery; - int32_t numOfLevels; - int32_t currentLevel; -} SQuery; #ifdef __cplusplus } #endif -#endif /*_TD_SCHEDULER_INT_H_*/ \ No newline at end of file +#endif /*_TD_SCHEDULER_INT_H_*/ diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 37f6240f9b..8d2e1ed916 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -13,4 +13,132 @@ * along with this program. If not, see . */ -#include "schedulerInt.h" \ No newline at end of file +#include "schedulerInt.h" +#include "taosmsg.h" +#include "query.h" + +SSchedulerMgmt schMgmt = {0}; + + +int32_t schBuildAndSendRequest(void *pRpc, const SEpSet* pMgmtEps, __taos_async_fn_t fp) { +/* + SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT); + if (pRequest == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SRequestMsgBody body = {0}; + buildConnectMsg(pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyConnectMsg(&body); + + if (pRequest->code != TSDB_CODE_SUCCESS) { + const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); + printf("failed to connect to server, reason: %s\n\n", errorMsg); + + destroyRequest(pRequest); + taos_close(pTscObj); + pTscObj = NULL; + } else { + tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter); + destroyRequest(pRequest); + } +*/ +} + +int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { + int32_t levelNum = (int32_t)taosArrayGetSize(dag->pSubplans); + if (levelNum <= 0) { + qError("invalid level num:%d", levelNum); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + job->levels = taosArrayInit(levelNum, sizeof(SQueryLevel)); + if (NULL == job->levels) { + qError("taosArrayInit %d failed", levelNum); + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + job->levelNum = levelNum; + job->levelIdx = levelNum - 1; + job->status = SCH_STATUS_NOT_START; + + job->subPlans = dag->pSubplans; + + SQueryLevel level = {0}; + SArray *levelPlans = NULL; + int32_t levelPlanNum = 0; + + for (int32_t i = 0; i < levelNum; ++i) { + levelPlans = taosArrayGetP(dag->pSubplans, i); + if (NULL == levelPlans) { + qError("no level plans for level %d", i); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + levelPlanNum = (int32_t)taosArrayGetSize(levelPlans); + if (levelPlanNum <= 0) { + qError("invalid level plans number:%d, level:%d", levelPlanNum, i); + return TSDB_CODE_QRY_INVALID_INPUT; + } + + for (int32_t n = 0; n < levelPlanNum; ++n) { + + } + } + + return TSDB_CODE_SUCCESS; +} + + + +int32_t schedulerInit(SSchedulerCfg *cfg) { + schMgmt.Jobs = taosHashInit(SCHEDULE_DEFAULT_JOB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + if (NULL == schMgmt.Jobs) { + SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", SCHEDULE_DEFAULT_JOB_NUMBER); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t scheduleQueryJob(SQueryDag* pDag, void** pJob) { + if (NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { + return TSDB_CODE_QRY_INVALID_INPUT; + } + + + SQueryJob *job = calloc(1, sizeof(SQueryJob)); + if (NULL == job) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + schValidateAndBuildJob(pDag, job); + + + + + + *(SQueryJob **)pJob = job; + + + + +} + +int32_t scheduleFetchRows(void *pJob, void *data); + +int32_t scheduleCancelJob(void *pJob); + +void schedulerDestroy(void) { + if (schMgmt.Jobs) { + taosHashCleanup(schMgmt.Jobs); //TBD + schMgmt.Jobs = NULL; + } +} + + diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 819afcc411..1579cad7b6 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -16,72 +16,70 @@ #ifndef _TD_WAL_INT_H_ #define _TD_WAL_INT_H_ -#include "wal.h" #include "compare.h" #include "tchecksum.h" +#include "wal.h" #ifdef __cplusplus extern "C" { #endif -//meta section begin +// meta section begin typedef struct WalFileInfo { int64_t firstVer; int64_t lastVer; int64_t createTs; int64_t closeTs; int64_t fileSize; -} WalFileInfo; +} SWalFileInfo; typedef struct WalIdxEntry { int64_t ver; int64_t offset; -} WalIdxEntry; +} SWalIdxEntry; static inline int32_t compareWalFileInfo(const void* pLeft, const void* pRight) { - WalFileInfo* pInfoLeft = (WalFileInfo*)pLeft; - WalFileInfo* pInfoRight = (WalFileInfo*)pRight; + SWalFileInfo* pInfoLeft = (SWalFileInfo*)pLeft; + SWalFileInfo* pInfoRight = (SWalFileInfo*)pRight; return compareInt64Val(&pInfoLeft->firstVer, &pInfoRight->firstVer); } static inline int64_t walGetLastFileSize(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); return pInfo->fileSize; } static inline int64_t walGetLastFileFirstVer(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); return pInfo->firstVer; } static inline int64_t walGetCurFileFirstVer(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->firstVer; } static inline int64_t walGetCurFileLastVer(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->firstVer; } static inline int64_t walGetCurFileOffset(SWal* pWal) { - WalFileInfo* pInfo = (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); + SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); return pInfo->fileSize; } -static inline bool walCurFileClosed(SWal* pWal) { - return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; +static inline bool walCurFileClosed(SWal* pWal) { return taosArrayGetSize(pWal->fileInfoSet) != pWal->writeCur; } + +static inline SWalFileInfo* walGetCurFileInfo(SWal* pWal) { + return (SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); } -static inline WalFileInfo* walGetCurFileInfo(SWal* pWal) { - return (WalFileInfo*)taosArrayGet(pWal->fileInfoSet, pWal->writeCur); -} - -static inline int walBuildLogName(SWal*pWal, int64_t fileFirstVer, char* buf) { +static inline int walBuildLogName(SWal* pWal, int64_t fileFirstVer, char* buf) { return sprintf(buf, "%s/%020" PRId64 "." WAL_LOG_SUFFIX, pWal->path, fileFirstVer); } -static inline int walBuildIdxName(SWal*pWal, int64_t fileFirstVer, char* buf) { +static inline int walBuildIdxName(SWal* pWal, int64_t fileFirstVer, char* buf) { return sprintf(buf, "%s/%020" PRId64 "." WAL_INDEX_SUFFIX, pWal->path, fileFirstVer); } @@ -93,11 +91,11 @@ static inline int walValidBodyCksum(SWalHead* pHead) { return taosCheckChecksum((uint8_t*)pHead->head.body, pHead->head.len, pHead->cksumBody); } -static inline int walValidCksum(SWalHead *pHead, void* body, int64_t bodyLen) { +static inline int walValidCksum(SWalHead* pHead, void* body, int64_t bodyLen) { return walValidHeadCksum(pHead) && walValidBodyCksum(pHead); } -static inline uint32_t walCalcHeadCksum(SWalHead *pHead) { +static inline uint32_t walCalcHeadCksum(SWalHead* pHead) { return taosCalcChecksum(0, (uint8_t*)&pHead->head, sizeof(SWalReadHead)); } @@ -106,7 +104,7 @@ static inline uint32_t walCalcBodyCksum(const void* body, uint32_t len) { } static inline int64_t walGetVerIdxOffset(SWal* pWal, int64_t ver) { - return (ver - walGetCurFileFirstVer(pWal)) * sizeof(WalIdxEntry); + return (ver - walGetCurFileFirstVer(pWal)) * sizeof(SWalIdxEntry); } static inline void walResetVer(SWalVer* pVer) { @@ -126,16 +124,16 @@ int walCheckAndRepairMeta(SWal* pWal); int walCheckAndRepairIdx(SWal* pWal); char* walMetaSerialize(SWal* pWal); -int walMetaDeserialize(SWal* pWal, const char* bytes); -//meta section end +int walMetaDeserialize(SWal* pWal, const char* bytes); +// meta section end -//seek section -int walChangeFile(SWal *pWal, int64_t ver); -//seek section end +// seek section +int walChangeFile(SWal* pWal, int64_t ver); +// seek section end int64_t walGetSeq(); -int walSeekVer(SWal *pWal, int64_t ver); -int walRoll(SWal *pWal); +int walSeekVer(SWal* pWal, int64_t ver); +int walRoll(SWal* pWal); #ifdef __cplusplus } diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e2715d51f1..0f155f9553 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -14,27 +14,21 @@ */ #define _DEFAULT_SOURCE +#include "cJSON.h" #include "os.h" #include "taoserror.h" -#include "tref.h" #include "tfile.h" -#include "cJSON.h" +#include "tref.h" #include "walInt.h" #include #include -int64_t inline walGetFirstVer(SWal *pWal) { - return pWal->vers.firstVer; -} +int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } -int64_t inline walGetSnaphostVer(SWal *pWal) { - return pWal->vers.snapshotVer; -} +int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; } -int64_t inline walGetLastVer(SWal *pWal) { - return pWal->vers.lastVer; -} +int64_t inline walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; } static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); @@ -44,31 +38,30 @@ int walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; - regex_t logRegPattern; - regex_t idxRegPattern; - SArray* pLogArray = taosArrayInit(8, sizeof(int64_t)); + regex_t logRegPattern; + regex_t idxRegPattern; + SArray* pLogArray = taosArrayInit(8, sizeof(int64_t)); regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); - - DIR *dir = opendir(pWal->path); - if(dir == NULL) { + + DIR* dir = opendir(pWal->path); + if (dir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } struct dirent* ent; - while((ent = readdir(dir)) != NULL) { - char *name = basename(ent->d_name); - int code = regexec(&logRegPattern, name, 0, NULL, 0); - if(code == 0) { + while ((ent = readdir(dir)) != NULL) { + char* name = basename(ent->d_name); + int code = regexec(&logRegPattern, name, 0, NULL, 0); + if (code == 0) { int64_t firstVer; sscanf(name, "%" PRId64 ".log", &firstVer); taosArrayPush(pLogArray, &firstVer); } } - // load meta // if not match, or meta missing // rebuild meta @@ -85,15 +78,15 @@ int walRollFileInfo(SWal* pWal) { int64_t ts = taosGetTimestampSec(); SArray* pArray = pWal->fileInfoSet; - if(taosArrayGetSize(pArray) != 0) { - WalFileInfo *pInfo = taosArrayGetLast(pArray); + if (taosArrayGetSize(pArray) != 0) { + SWalFileInfo* pInfo = taosArrayGetLast(pArray); pInfo->lastVer = pWal->vers.lastVer; pInfo->closeTs = ts; } - //TODO: change to emplace back - WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo)); - if(pNewInfo == NULL) { + // TODO: change to emplace back + SWalFileInfo* pNewInfo = malloc(sizeof(SWalFileInfo)); + if (pNewInfo == NULL) { return -1; } pNewInfo->firstVer = pWal->vers.lastVer + 1; @@ -109,13 +102,13 @@ int walRollFileInfo(SWal* pWal) { char* walMetaSerialize(SWal* pWal) { char buf[30]; ASSERT(pWal->fileInfoSet); - int sz = pWal->fileInfoSet->size; + int sz = pWal->fileInfoSet->size; cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); cJSON* pFiles = cJSON_CreateArray(); cJSON* pField; - if(pRoot == NULL || pMeta == NULL || pFiles == NULL) { - //TODO + if (pRoot == NULL || pMeta == NULL || pFiles == NULL) { + // TODO return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); @@ -129,16 +122,16 @@ char* walMetaSerialize(SWal* pWal) { cJSON_AddStringToObject(pMeta, "lastVer", buf); cJSON_AddItemToObject(pRoot, "files", pFiles); - WalFileInfo* pData = pWal->fileInfoSet->pData; - for(int i = 0; i < sz; i++) { - WalFileInfo* pInfo = &pData[i]; + SWalFileInfo* pData = pWal->fileInfoSet->pData; + for (int i = 0; i < sz; i++) { + SWalFileInfo* pInfo = &pData[i]; cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()); - if(pField == NULL) { + if (pField == NULL) { cJSON_Delete(pRoot); return NULL; } - //cjson only support int32_t or double - //string are used to prohibit the loss of precision + // cjson only support int32_t or double + // string are used to prohibit the loss of precision sprintf(buf, "%" PRId64, pInfo->firstVer); cJSON_AddStringToObject(pField, "firstVer", buf); sprintf(buf, "%" PRId64, pInfo->lastVer); @@ -171,13 +164,13 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pFiles = cJSON_GetObjectItem(pRoot, "files"); int sz = cJSON_GetArraySize(pFiles); - //deserialize + // deserialize SArray* pArray = pWal->fileInfoSet; taosArrayEnsureCap(pArray, sz); - WalFileInfo *pData = pArray->pData; - for(int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); - WalFileInfo* pInfo = &pData[i]; + SWalFileInfo* pData = pArray->pData; + for (int i = 0; i < sz; i++) { + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + SWalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); pField = cJSON_GetObjectItem(pInfoJson, "lastVer"); @@ -196,24 +189,24 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { } static int walFindCurMetaVer(SWal* pWal) { - const char * pattern = "^meta-ver[0-9]+$"; - regex_t walMetaRegexPattern; + const char* pattern = "^meta-ver[0-9]+$"; + regex_t walMetaRegexPattern; regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED); - DIR *dir = opendir(pWal->path); - if(dir == NULL) { + DIR* dir = opendir(pWal->path); + if (dir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } struct dirent* ent; - //find existing meta-ver[x].json + // find existing meta-ver[x].json int metaVer = -1; - while((ent = readdir(dir)) != NULL) { - char *name = basename(ent->d_name); - int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); - if(code == 0) { + while ((ent = readdir(dir)) != NULL) { + char* name = basename(ent->d_name); + int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); + if (code == 0) { sscanf(name, "meta-ver%d", &metaVer); break; } @@ -224,23 +217,23 @@ static int walFindCurMetaVer(SWal* pWal) { } int walSaveMeta(SWal* pWal) { - int metaVer = walFindCurMetaVer(pWal); + int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; - walBuildMetaName(pWal, metaVer+1, fnameStr); + walBuildMetaName(pWal, metaVer + 1, fnameStr); int metaTfd = tfOpenCreateWrite(fnameStr); - if(metaTfd < 0) { + if (metaTfd < 0) { return -1; } char* serialized = walMetaSerialize(pWal); - int len = strlen(serialized); - if(len != tfWrite(metaTfd, serialized, len)) { - //TODO:clean file + int len = strlen(serialized); + if (len != tfWrite(metaTfd, serialized, len)) { + // TODO:clean file return -1; } - + tfClose(metaTfd); - //delete old file - if(metaVer > -1) { + // delete old file + if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); remove(fnameStr); } @@ -250,30 +243,30 @@ int walSaveMeta(SWal* pWal) { int walLoadMeta(SWal* pWal) { ASSERT(pWal->fileInfoSet->size == 0); - //find existing meta file + // find existing meta file int metaVer = walFindCurMetaVer(pWal); - if(metaVer == -1) { + if (metaVer == -1) { return 0; } char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); - //read metafile + // read metafile struct stat statbuf; stat(fnameStr, &statbuf); - int size = statbuf.st_size; + int size = statbuf.st_size; char* buf = malloc(size + 5); - if(buf == NULL) { + if (buf == NULL) { return -1; } - memset(buf, 0, size+5); + memset(buf, 0, size + 5); int tfd = tfOpenRead(fnameStr); - if(tfRead(tfd, buf, size) != size) { + if (tfRead(tfd, buf, size) != size) { free(buf); return -1; } - //load into fileInfoSet + // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - if(code != 0) { + if (code != 0) { free(buf); return -1; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 06df3eb3be..189881c86d 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE +#include "compare.h" #include "os.h" #include "taoserror.h" -#include "tref.h" #include "tfile.h" -#include "compare.h" +#include "tref.h" #include "walInt.h" typedef struct { @@ -34,16 +34,14 @@ static int32_t walCreateThread(); static void walStopThread(); static void walFreeObj(void *pWal); -int64_t walGetSeq() { - return (int64_t)atomic_load_32(&tsWal.seq); -} +int64_t walGetSeq() { return (int64_t)atomic_load_32(&tsWal.seq); } int32_t walInit() { int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); - if(old == 1) return 0; + if (old == 1) return 0; int code = tfInit(); - if(code != 0) { + if (code != 0) { wError("failed to init tfile since %s", tstrerror(code)); atomic_store_8(&tsWal.inited, 0); return code; @@ -63,7 +61,7 @@ int32_t walInit() { void walCleanUp() { int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); - if(old == 0) { + if (old == 0) { return; } walStopThread(); @@ -78,52 +76,52 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - //set config + // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; - if(pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; + if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; tstrncpy(pWal->path, path, sizeof(pWal->path)); - if(taosMkDir(pWal->path) != 0) { + if (taosMkDir(pWal->path) != 0) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return NULL; } - //open meta + // open meta walResetVer(&pWal->vers); pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; pWal->writeCur = -1; - pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); - if(pWal->fileInfoSet == NULL) { + pWal->fileInfoSet = taosArrayInit(8, sizeof(SWalFileInfo)); + if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); free(pWal); return NULL; } - //init status + // init status pWal->totSize = 0; pWal->lastRollSeq = -1; - //init write buffer + // init write buffer memset(&pWal->writeHead, 0, sizeof(SWalHead)); pWal->writeHead.head.headVer = WAL_HEAD_VER; - if(pthread_mutex_init(&pWal->mutex, NULL) < 0) { + if (pthread_mutex_init(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; } pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if(pWal->refId < 0) { + if (pWal->refId < 0) { pthread_mutex_destroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; } - if(walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { + if (walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); pthread_mutex_destroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); @@ -131,11 +129,11 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if(walCheckAndRepairIdx(pWal) < 0) { - + if (walCheckAndRepairIdx(pWal) < 0) { } - wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, + pWal->cfg.fsyncPeriod); return pWal; } @@ -203,10 +201,12 @@ static void walFsyncAll() { SWal *pWal = taosIterateRef(tsWal.refSetId, 0); while (pWal) { if (walNeedFsync(pWal)) { - wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, + atomic_load_32(&tsWal.seq)); int32_t code = tfFsync(pWal->writeLogTfd); if (code != 0) { - wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); + wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(code)); } } pWal = taosIterateRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 42fcb8c375..48eb84b536 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ -#include "walInt.h" #include "tfile.h" +#include "walInt.h" -SWalReadHandle* walOpenReadHandle(SWal* pWal) { +SWalReadHandle *walOpenReadHandle(SWal *pWal) { SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); - if(pRead == NULL) { + if (pRead == NULL) { return NULL; } pRead->pWal = pWal; @@ -29,7 +29,7 @@ SWalReadHandle* walOpenReadHandle(SWal* pWal) { pRead->capacity = 0; pRead->status = 0; pRead->pHead = malloc(sizeof(SWalHead)); - if(pRead->pHead == NULL) { + if (pRead->pHead == NULL) { free(pRead); return NULL; } @@ -43,27 +43,25 @@ void walCloseReadHandle(SWalReadHandle *pRead) { free(pRead); } -int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { - return 0; -} +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; int64_t idxTfd = pRead->readIdxTfd; int64_t logTfd = pRead->readLogTfd; - - //seek position - int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry); + + // seek position + int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); code = tfLseek(idxTfd, offset, SEEK_SET); - if(code < 0) { + if (code < 0) { return -1; } - WalIdxEntry entry; - if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + SWalIdxEntry entry; + if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { return -1; } - //TODO:deserialize + // TODO:deserialize ASSERT(entry.ver == ver); code = tfLseek(logTfd, entry.offset, SEEK_SET); if (code < 0) { @@ -80,13 +78,13 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); int64_t logTfd = tfOpenRead(fnameStr); - if(logTfd < 0) { + if (logTfd < 0) { return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); int64_t idxTfd = tfOpenRead(fnameStr); - if(idxTfd < 0) { + if (idxTfd < 0) { return -1; } @@ -96,76 +94,75 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { } static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { - int code; + int code; SWal *pWal = pRead->pWal; - if(ver == pRead->curVersion) { + if (ver == pRead->curVersion) { return 0; } - if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { return -1; } - if(ver < pWal->vers.snapshotVer) { - + if (ver < pWal->vers.snapshotVer) { } - WalFileInfo tmpInfo; + SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; - //bsearch in fileSet - WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + // bsearch in fileSet + SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); - if(pRead->curFileFirstVer != pRet->firstVer) { + if (pRead->curFileFirstVer != pRet->firstVer) { code = walReadChangeFile(pRead, pRet->firstVer); - if(code < 0) { - //TODO: set error flag + if (code < 0) { + // TODO: set error flag return -1; } } code = walReadSeekFilePos(pRead, pRet->firstVer, ver); - if(code < 0) { + if (code < 0) { return -1; } pRead->curVersion = ver; - + return 0; } int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int code; - //TODO: check wal life - if(pRead->curVersion != ver) { - code = walReadSeekVer(pRead, ver); - if(code != 0) { + // TODO: check wal life + if (pRead->curVersion != ver) { + code = walReadSeekVer(pRead, ver); + if (code != 0) { return -1; } } - if(!tfValid(pRead->readLogTfd)) return -1; + if (!tfValid(pRead->readLogTfd)) return -1; code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); - if(code != sizeof(SWalHead)) { + if (code != sizeof(SWalHead)) { return -1; } code = walValidHeadCksum(pRead->pHead); - if(code != 0) { + if (code != 0) { return -1; } - if(pRead->capacity < pRead->pHead->head.len) { - void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); - if(ptr == NULL) { + if (pRead->capacity < pRead->pHead->head.len) { + void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); + if (ptr == NULL) { return -1; } pRead->pHead = ptr; pRead->capacity = pRead->pHead->head.len; } - if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } - /*code = walValidBodyCksum(pRead->pHead);*/ ASSERT(pRead->pHead->head.version == ver); - if(code != 0) { + code = walValidBodyCksum(pRead->pHead); + if (code != 0) { return -1; } pRead->curVersion++; @@ -176,40 +173,40 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int code; code = walSeekVer(pWal, ver); - if(code != 0) { + if (code != 0) { return code; } - if(*ppHead == NULL) { - void* ptr = realloc(*ppHead, sizeof(SWalHead)); - if(ptr == NULL) { + if (*ppHead == NULL) { + void *ptr = realloc(*ppHead, sizeof(SWalHead)); + if (ptr == NULL) { return -1; } *ppHead = ptr; } - if(tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { return -1; } - //TODO: endian compatibility processing after read - if(walValidHeadCksum(*ppHead) != 0) { + // TODO: endian compatibility processing after read + if (walValidHeadCksum(*ppHead) != 0) { return -1; } - void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); - if(ptr == NULL) { + void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); + if (ptr == NULL) { free(*ppHead); *ppHead = NULL; return -1; } - if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { + if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } - //TODO: endian compatibility processing after read - if(walValidBodyCksum(*ppHead) != 0) { + // TODO: endian compatibility processing after read + if (walValidBodyCksum(*ppHead) != 0) { return -1; } - + return 0; } /*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/ - /*return 0;*/ +/*return 0;*/ /*}*/ diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 769b2fd2e4..1f9b7b6e58 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tref.h" #include "tfile.h" +#include "tref.h" #include "walInt.h" static int walSeekFilePos(SWal* pWal, int64_t ver) { @@ -25,17 +25,17 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { int64_t idxTfd = pWal->writeIdxTfd; int64_t logTfd = pWal->writeLogTfd; - - //seek position + + // seek position int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = tfLseek(idxTfd, idxOff, SEEK_SET); - if(code != 0) { + if (code != 0) { return -1; } - WalIdxEntry entry; - //TODO:deserialize - code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); - if(code != 0) { + SWalIdxEntry entry; + // TODO:deserialize + code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); + if (code != 0) { return -1; } ASSERT(entry.ver == ver); @@ -46,51 +46,51 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { return code; } -int walChangeFileToLast(SWal *pWal) { - int64_t idxTfd, logTfd; - WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); +int walChangeFileToLast(SWal* pWal) { + int64_t idxTfd, logTfd; + SWalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenReadWrite(fnameStr); - if(idxTfd < 0) { + if (idxTfd < 0) { return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); logTfd = tfOpenReadWrite(fnameStr); - if(logTfd < 0) { + if (logTfd < 0) { return -1; } - //switch file + // switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; return 0; } -int walChangeFile(SWal *pWal, int64_t ver) { - int code = 0; +int walChangeFile(SWal* pWal, int64_t ver) { + int code = 0; int64_t idxTfd, logTfd; - char fnameStr[WAL_FILE_LEN]; + char fnameStr[WAL_FILE_LEN]; code = tfClose(pWal->writeLogTfd); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return -1; } code = tfClose(pWal->writeIdxTfd); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return -1; } - WalFileInfo tmpInfo; + SWalFileInfo tmpInfo; tmpInfo.firstVer = ver; - //bsearch in fileSet - WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + // bsearch in fileSet + SWalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; - //closed - if(taosArrayGetLast(pWal->fileInfoSet) != pRet) { + // closed + if (taosArrayGetLast(pWal->fileInfoSet) != pRet) { walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenRead(fnameStr); walBuildLogName(pWal, fileFirstVer, fnameStr); @@ -107,27 +107,26 @@ int walChangeFile(SWal *pWal, int64_t ver) { return fileFirstVer; } -int walSeekVer(SWal *pWal, int64_t ver) { +int walSeekVer(SWal* pWal, int64_t ver) { int code; - if(ver == pWal->vers.lastVer) { + if (ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { return -1; } - if(ver < pWal->vers.snapshotVer) { - + if (ver < pWal->vers.snapshotVer) { } - if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { + if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeFile(pWal, ver); - if(code != 0) { + if (code != 0) { return -1; } } code = walSeekFilePos(pWal, ver); - if(code != 0) { + if (code != 0) { return -1; } - + return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 67b70f249d..c8ffd9d07d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -24,7 +24,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) { ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); - if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { + if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { return -1; } pWal->vers.commitVer = ver; @@ -32,57 +32,57 @@ int32_t walCommit(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { - int code; + int code; char fnameStr[WAL_FILE_LEN]; - if(ver == pWal->vers.lastVer) { + if (ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { return -1; } pthread_mutex_lock(&pWal->mutex); - //find correct file - if(ver < walGetLastFileFirstVer(pWal)) { - //close current files + // find correct file + if (ver < walGetLastFileFirstVer(pWal)) { + // close current files tfClose(pWal->writeIdxTfd); tfClose(pWal->writeLogTfd); - //open old files + // open old files code = walChangeFile(pWal, ver); - if(code != 0) { + if (code != 0) { return -1; } - //delete files + // delete files int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); - for(int i = pWal->writeCur; i < fileSetSize; i++) { - walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + for (int i = pWal->writeCur; i < fileSetSize; i++) { + walBuildLogName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); remove(fnameStr); - walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + walBuildIdxName(pWal, ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); remove(fnameStr); } - //pop from fileInfoSet + // pop from fileInfoSet taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); int64_t idxTfd = tfOpenReadWrite(fnameStr); - //TODO:change to deserialize function - if(idxTfd < 0) { + // TODO:change to deserialize function + if (idxTfd < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = tfLseek(idxTfd, idxOff, SEEK_SET); - if(code < 0) { + if (code < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; } - //read idx file and get log file pos - //TODO:change to deserialize function - WalIdxEntry entry; - if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + // read idx file and get log file pos + // TODO:change to deserialize function + SWalIdxEntry entry; + if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } @@ -90,56 +90,56 @@ int32_t walRollback(SWal *pWal, int64_t ver) { walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); int64_t logTfd = tfOpenReadWrite(fnameStr); - if(logTfd < 0) { - //TODO + if (logTfd < 0) { + // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } code = tfLseek(logTfd, entry.offset, SEEK_SET); - if(code < 0) { - //TODO + if (code < 0) { + // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } - //validate offset + // validate offset SWalHead head; ASSERT(tfValid(logTfd)); int size = tfRead(logTfd, &head, sizeof(SWalHead)); - if(size != sizeof(SWalHead)) { + if (size != sizeof(SWalHead)) { return -1; } code = walValidHeadCksum(&head); ASSERT(code == 0); - if(code != 0) { + if (code != 0) { return -1; } - if(head.head.version != ver) { - //TODO + if (head.head.version != ver) { + // TODO return -1; } - //truncate old files + // truncate old files code = tfFtruncate(logTfd, entry.offset); - if(code < 0) { + if (code < 0) { return -1; } code = tfFtruncate(idxTfd, idxOff); - if(code < 0) { + if (code < 0) { return -1; } pWal->vers.lastVer = ver - 1; - ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; - ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; + ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; - //unlock + // unlock pthread_mutex_unlock(&pWal->mutex); return 0; } -int32_t walBeginSnapshot(SWal* pWal, int64_t ver) { +int32_t walBeginSnapshot(SWal *pWal, int64_t ver) { pWal->vers.verInSnapshotting = ver; - //check file rolling - if(pWal->cfg.retentionPeriod == 0) { + // check file rolling + if (pWal->cfg.retentionPeriod == 0) { walRoll(pWal); } @@ -148,54 +148,54 @@ int32_t walBeginSnapshot(SWal* pWal, int64_t ver) { int32_t walEndSnapshot(SWal *pWal) { int64_t ver = pWal->vers.verInSnapshotting; - if(ver == -1) return -1; + if (ver == -1) return -1; pWal->vers.snapshotVer = ver; int ts = taosGetTimestampSec(); - int deleteCnt = 0; - int64_t newTotSize = pWal->totSize; - WalFileInfo tmp; + int deleteCnt = 0; + int64_t newTotSize = pWal->totSize; + SWalFileInfo tmp; tmp.firstVer = ver; - //find files safe to delete - WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); - if(ver >= pInfo->lastVer) { + // find files safe to delete + SWalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); + if (ver >= pInfo->lastVer) { pInfo++; } - //iterate files, until the searched result - for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if(pWal->totSize > pWal->cfg.retentionSize || - iter->closeTs + pWal->cfg.retentionPeriod > ts) { - //delete according to file size or close time + // iterate files, until the searched result + for (SWalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { + if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) { + // delete according to file size or close time deleteCnt++; newTotSize -= iter->fileSize; } } char fnameStr[WAL_FILE_LEN]; - //remove file - for(int i = 0; i < deleteCnt; i++) { - WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); - walBuildLogName(pWal, pInfo->firstVer, fnameStr); + // remove file + for (int i = 0; i < deleteCnt; i++) { + SWalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i); + walBuildLogName(pWal, pInfo->firstVer, fnameStr); remove(fnameStr); - walBuildIdxName(pWal, pInfo->firstVer, fnameStr); + walBuildIdxName(pWal, pInfo->firstVer, fnameStr); remove(fnameStr); } - //make new array, remove files - taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); - if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + // make new array, remove files + taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); + if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->writeCur = -1; pWal->vers.firstVer = -1; } else { - pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.firstVer = ((SWalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } - pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;; + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; + ; pWal->totSize = newTotSize; pWal->vers.verInSnapshotting = -1; - //save snapshot ver, commit ver + // save snapshot ver, commit ver int code = walSaveMeta(pWal); - if(code != 0) { + if (code != 0) { return -1; } @@ -204,41 +204,41 @@ int32_t walEndSnapshot(SWal *pWal) { int walRoll(SWal *pWal) { int code = 0; - if(pWal->writeIdxTfd != -1) { + if (pWal->writeIdxTfd != -1) { code = tfClose(pWal->writeIdxTfd); - if(code != 0) { + if (code != 0) { return -1; } } - if(pWal->writeLogTfd != -1) { + if (pWal->writeLogTfd != -1) { code = tfClose(pWal->writeLogTfd); - if(code != 0) { + if (code != 0) { return -1; } } int64_t idxTfd, logTfd; - //create new file + // create new file int64_t newFileFirstVersion = pWal->vers.lastVer + 1; - char fnameStr[WAL_FILE_LEN]; + char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); idxTfd = tfOpenCreateWriteAppend(fnameStr); - if(idxTfd < 0) { + if (idxTfd < 0) { ASSERT(0); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); logTfd = tfOpenCreateWriteAppend(fnameStr); - if(logTfd < 0) { + if (logTfd < 0) { ASSERT(0); return -1; } code = walRollFileInfo(pWal); - if(code != 0) { + if (code != 0) { ASSERT(0); return -1; } - //switch file + // switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; @@ -248,10 +248,10 @@ int walRoll(SWal *pWal) { } static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - WalIdxEntry entry = { .ver = ver, .offset = offset }; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); - if(size != sizeof(WalIdxEntry)) { - //TODO truncate + SWalIdxEntry entry = {.ver = ver, .offset = offset}; + int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(SWalIdxEntry)); + if (size != sizeof(SWalIdxEntry)) { + // TODO truncate return -1; } return 0; @@ -265,21 +265,21 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; if (index == pWal->vers.lastVer + 1) { - if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->vers.firstVer = index; code = walRoll(pWal); ASSERT(code == 0); } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { + if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { walRoll(pWal); - } else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { + } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { walRoll(pWal); } } } else { - //reject skip log or rewrite log - //must truncate explicitly first + // reject skip log or rewrite log + // must truncate explicitly first return -1; } /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ @@ -294,28 +294,30 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { - //ftruncate + // ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) { - //ftruncate + if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) { + // ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(errno)); } code = walWriteIndex(pWal, index, offset); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return -1; } - //set status + // set status pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalHead) + bodyLen; walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; - + pthread_mutex_unlock(&pWal->mutex); return code; @@ -323,33 +325,34 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); + wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (tfFsync(pWal->writeLogTfd) < 0) { - wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), + strerror(errno)); } } } /*static int walValidateOffset(SWal* pWal, int64_t ver) {*/ - /*int code = 0;*/ - /*SWalHead *pHead = NULL;*/ - /*code = (int)walRead(pWal, &pHead, ver);*/ - /*if(pHead->head.version != ver) {*/ - /*return -1;*/ - /*}*/ - /*return 0;*/ +/*int code = 0;*/ +/*SWalHead *pHead = NULL;*/ +/*code = (int)walRead(pWal, &pHead, ver);*/ +/*if(pHead->head.version != ver) {*/ +/*return -1;*/ +/*}*/ +/*return 0;*/ /*}*/ /*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/ - /*int code = walSeekVer(pWal, ver);*/ - /*if(code != 0) {*/ - /*return -1;*/ - /*}*/ - - /*code = walValidateOffset(pWal, ver);*/ - /*if(code != 0) {*/ - /*return -1;*/ - /*}*/ - - /*return 0;*/ +/*int code = walSeekVer(pWal, ver);*/ +/*if(code != 0) {*/ +/*return -1;*/ +/*}*/ + +/*code = walValidateOffset(pWal, ver);*/ +/*if(code != 0) {*/ +/*return -1;*/ +/*}*/ + +/*return 0;*/ /*}*/ diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index d06388201e..cd082a3a43 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -6,111 +6,105 @@ #include "walInt.h" const char* ranStr = "tvapq02tcp"; -const int ranStrLen = strlen(ranStr); +const int ranStrLen = strlen(ranStr); class WalCleanEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(); - ASSERT(code == 0); - } + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } - static void TearDownTestCase() { - walCleanUp(); - } + static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - free(pCfg); - ASSERT(pWal != NULL); - } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + free(pCfg); + ASSERT(pWal != NULL); + } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } - SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; }; class WalCleanDeleteEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(); - ASSERT(code == 0); - } + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } - static void TearDownTestCase() { - walCleanUp(); - } + static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - free(pCfg); - ASSERT(pWal != NULL); - } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + free(pCfg); + ASSERT(pWal != NULL); + } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } - SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; }; class WalKeepEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(); - ASSERT(code == 0); - } + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } - static void TearDownTestCase() { - walCleanUp(); - } + static void TearDownTestCase() { walCleanUp(); } - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } - void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - free(pCfg); - ASSERT(pWal != NULL); - } + void SetUp() override { + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + free(pCfg); + ASSERT(pWal != NULL); + } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } - SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; }; TEST_F(WalCleanEnv, createNew) { @@ -139,7 +133,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); code = walRollFileInfo(pWal); ASSERT(code == 0); - char*ss = walMetaSerialize(pWal); + char* ss = walMetaSerialize(pWal); printf("%s\n", ss); free(ss); code = walSaveMeta(pWal); @@ -162,11 +156,11 @@ TEST_F(WalKeepEnv, readOldMeta) { walResetEnv(); int code; - for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + for (int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); + code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -182,7 +176,7 @@ TEST_F(WalKeepEnv, readOldMeta) { int len = strlen(oldss); ASSERT_EQ(len, strlen(newss)); - for(int i = 0; i < len; i++) { + for (int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); } free(oldss); @@ -191,11 +185,11 @@ TEST_F(WalKeepEnv, readOldMeta) { TEST_F(WalCleanEnv, write) { int code; - for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + for (int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); + code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -205,8 +199,8 @@ TEST_F(WalCleanEnv, write) { TEST_F(WalCleanEnv, rollback) { int code; - for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + for (int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -223,7 +217,7 @@ TEST_F(WalCleanEnv, rollback) { TEST_F(WalCleanDeleteEnv, roll) { int code; int i; - for(i = 0; i < 100; i++) { + for (i = 0; i < 100; i++) { code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); @@ -231,16 +225,16 @@ TEST_F(WalCleanDeleteEnv, roll) { ASSERT_EQ(pWal->vers.commitVer, i); } - walBeginSnapshot(pWal, i-1); - ASSERT_EQ(pWal->vers.verInSnapshotting, i-1); + walBeginSnapshot(pWal, i - 1); + ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); walEndSnapshot(pWal); - ASSERT_EQ(pWal->vers.snapshotVer, i-1); + ASSERT_EQ(pWal->vers.snapshotVer, i - 1); ASSERT_EQ(pWal->vers.verInSnapshotting, -1); code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); ASSERT_NE(code, 0); - for(; i < 200; i++) { + for (; i < 200; i++) { code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); code = walCommit(pWal, i); @@ -255,36 +249,36 @@ TEST_F(WalCleanDeleteEnv, roll) { TEST_F(WalKeepEnv, readHandleRead) { walResetEnv(); - int code; + int code; SWalReadHandle* pRead = walOpenReadHandle(pWal); ASSERT(pRead != NULL); - int i ; - for(i = 0; i < 100; i++) { + int i; + for (i = 0; i < 100; i++) { char newStr[100]; sprintf(newStr, "%s-%d", ranStr, i); int len = strlen(newStr); code = walWrite(pWal, i, 0, newStr, len); ASSERT_EQ(code, 0); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ver = rand() % 100; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); - //printf("rrbody: \n"); - //for(int i = 0; i < pRead->pHead->head.len; i++) { - //printf("%d ", pRead->pHead->head.body[i]); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); //} - //printf("\n"); + // printf("\n"); ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver+1); + ASSERT_EQ(pRead->curVersion, ver + 1); char newStr[100]; sprintf(newStr, "%s-%d", ranStr, ver); int len = strlen(newStr); ASSERT_EQ(pRead->pHead->head.len, len); - for(int j = 0; j < len; j++) { + for (int j = 0; j < len; j++) { EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); } } diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index f892b4d8c0..cb231e15a0 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -1120,11 +1120,11 @@ SysNameInfo taosGetSysNameInfo() { struct utsname uts; if (!uname(&uts)) { - info.sysname = strdup(uts.sysname); - info.nodename = strdup(uts.nodename); - info.release = strdup(uts.release); - info.version = strdup(uts.version); - info.machine = strdup(uts.machine); + tstrncpy(info.sysname, uts.sysname, sizeof(info.sysname)); + tstrncpy(info.nodename, uts.nodename, sizeof(info.nodename)); + tstrncpy(info.release, uts.release, sizeof(info.release)); + tstrncpy(info.version, uts.version, sizeof(info.version)); + tstrncpy(info.machine, uts.machine, sizeof(info.machine)); } return info; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 581a797343..cc8d6646b6 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -17,7 +17,7 @@ #include "tarray.h" #include "talgo.h" -void* taosArrayInit(size_t size, size_t elemSize) { +SArray* taosArrayInit(size_t size, size_t elemSize) { assert(elemSize > 0); if (size < TARRAY_MIN_SIZE) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 70a3dc622f..5518ec2a31 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -322,6 +322,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit ha TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION, "One valid time range condition expected") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_INPUT, "invalid input") // grant diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 4a621d47c0..b2f8eda474 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -183,7 +183,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo pSql->fp = fp; if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { @@ -265,7 +265,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { } return; - } else if (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { + } else if (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE) { // in case of show command, return no data (*pSql->fetchFp)(param, pSql, 0); } else { @@ -273,7 +273,7 @@ void taos_fetch_rows_a(TAOS_RES *tres, __async_cb_func_t fp, void *param) { } } else { // current query is not completed, continue retrieve from node if (pCmd->command != TSDB_SQL_RETRIEVE_GLOBALMERGE && pCmd->command < TSDB_SQL_LOCAL) { - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } SQueryInfo* pQueryInfo1 = tscGetQueryInfo(&pSql->cmd); diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 0b4f399a1f..dcfb2d6a87 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -322,7 +322,7 @@ TAOS_ROW tscFetchRow(void *param) { // current data set are exhausted, fetch more data from node if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE || + (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_FETCH || diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index caa334aaed..7a754ee698 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -539,7 +539,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) { if (pCmd->command == TSDB_SQL_SELECT || pCmd->command == TSDB_SQL_FETCH || - pCmd->command == TSDB_SQL_RETRIEVE || + pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_INSERT || pCmd->command == TSDB_SQL_CONNECT || pCmd->command == TSDB_SQL_HB || @@ -2749,7 +2749,7 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { } STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if ((pCmd->command == TSDB_SQL_RETRIEVE) || + if ((pCmd->command == TSDB_SQL_RETRIEVE_MNODE) || ((UTIL_TABLE_IS_CHILD_TABLE(pTableMetaInfo) || UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) && !TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY)) || (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && @@ -3174,7 +3174,7 @@ void tscInitMsgsFp() { tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg; tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg; - tscBuildMsg[TSDB_SQL_RETRIEVE] = tscBuildRetrieveFromMgmtMsg; + tscBuildMsg[TSDB_SQL_RETRIEVE_MNODE] = tscBuildRetrieveFromMgmtMsg; tscBuildMsg[TSDB_SQL_KILL_QUERY] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_STREAM] = tscBuildKillMsg; tscBuildMsg[TSDB_SQL_KILL_CONNECTION] = tscBuildKillMsg; @@ -3192,7 +3192,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp; tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp; - tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. + tscProcessMsgRsp[TSDB_SQL_RETRIEVE_MNODE] = tscProcessRetrieveRspFromNode; // rsp handled by same function. tscProcessMsgRsp[TSDB_SQL_DESCRIBE_TABLE] = tscProcessDescribeTableRsp; tscProcessMsgRsp[TSDB_SQL_CURRENT_DB] = tscProcessLocalRetrieveRsp; @@ -3214,7 +3214,7 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; tscKeepConn[TSDB_SQL_SHOW] = 1; - tscKeepConn[TSDB_SQL_RETRIEVE] = 1; + tscKeepConn[TSDB_SQL_RETRIEVE_MNODE] = 1; tscKeepConn[TSDB_SQL_SELECT] = 1; tscKeepConn[TSDB_SQL_FETCH] = 1; tscKeepConn[TSDB_SQL_HB] = 1; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 5fdaad0d66..ab1fffd5a2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -460,7 +460,7 @@ static bool needToFetchNewBlock(SSqlObj* pSql) { SSqlCmd *pCmd = &pSql->cmd; return (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && - (pCmd->command == TSDB_SQL_RETRIEVE || + (pCmd->command == TSDB_SQL_RETRIEVE_MNODE || pCmd->command == TSDB_SQL_RETRIEVE_GLOBALMERGE || pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE || pCmd->command == TSDB_SQL_FETCH || @@ -582,10 +582,10 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) { if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) && (cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_SHOW || - cmd == TSDB_SQL_RETRIEVE || + cmd == TSDB_SQL_RETRIEVE_MNODE || cmd == TSDB_SQL_FETCH)) { pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; tscDebug("0x%"PRIx64" send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql->self, sqlCmd[pCmd->command]); tscBuildAndSendRequest(pSql, NULL); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 275042a238..24a6377a73 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1670,7 +1670,7 @@ void tscFetchDatablockForSubquery(SSqlObj* pSql) { pSql1->fp = joinRetrieveFinalResCallback; if (pCmd1->command < TSDB_SQL_LOCAL) { - pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH; + pCmd1->command = (pCmd1->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE_MNODE : TSDB_SQL_FETCH; } tscBuildAndSendRequest(pSql1, NULL);