diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000000..fc9a51af65 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp/.devcontainer/base.Dockerfile + +# [Choice] Debian / Ubuntu version (use Debian 11/9, Ubuntu 18.04/21.04 on local arm64/Apple Silicon): debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 +ARG VARIANT="bullseye" +FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT} + +# [Optional] Uncomment this section to install additional packages. +# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ +# && apt-get -y install --no-install-recommends diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000000..9b752d091d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,32 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.209.3/containers/cpp +{ + "name": "C++", + "build": { + "dockerfile": "Dockerfile", + // Update 'VARIANT' to pick an Debian / Ubuntu OS version: debian-11, debian-10, debian-9, ubuntu-21.04, ubuntu-20.04, ubuntu-18.04 + // Use Debian 11, Debian 9, Ubuntu 18.04 or Ubuntu 21.04 on local arm64/Apple Silicon + "args": { "VARIANT": "ubuntu-21.04" } + }, + "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"], + + // Set *default* container specific settings.json values on container create. + "settings": {}, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.cpptools", + "ms-vscode.cmake-tools", + "austin.code-gnu-global", + "visualstudioexptteam.vscodeintel" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + // "postCreateCommand": "gcc -v", + + // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode" +} diff --git a/include/client/taos.h b/include/client/taos.h index 19d191b84e..eb3332ed18 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -39,7 +39,7 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes #define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes #define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string +#define TSDB_DATA_TYPE_BINARY 8 // string, alias for varchar #define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes #define TSDB_DATA_TYPE_NCHAR 10 // unicode string #define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte @@ -47,10 +47,10 @@ typedef void **TAOS_ROW; #define TSDB_DATA_TYPE_UINT 13 // 4 bytes #define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes #define TSDB_DATA_TYPE_VARCHAR 15 // string -#define TSDB_DATA_TYPE_JSON 16 // json -#define TSDB_DATA_TYPE_DECIMAL 17 // decimal -#define TSDB_DATA_TYPE_BLOB 18 // binary string -#define TSDB_DATA_TYPE_LONGBLOB 19 // long binary string +#define TSDB_DATA_TYPE_VARBINARY 16 // binary +#define TSDB_DATA_TYPE_JSON 17 // json +#define TSDB_DATA_TYPE_DECIMAL 18 // decimal +#define TSDB_DATA_TYPE_BLOB 19 // binary typedef enum { TSDB_OPTION_LOCALE, @@ -61,6 +61,23 @@ typedef enum { TSDB_MAX_OPTIONS } TSDB_OPTION; +typedef enum { + TSDB_SML_UNKNOWN_PROTOCOL = 0, + TSDB_SML_LINE_PROTOCOL = 1, + TSDB_SML_TELNET_PROTOCOL = 2, + TSDB_SML_JSON_PROTOCOL = 3, +} TSDB_SML_PROTOCOL_TYPE; + +typedef enum { + TSDB_SML_TIMESTAMP_NOT_CONFIGURED = 0, + TSDB_SML_TIMESTAMP_HOURS, + TSDB_SML_TIMESTAMP_MINUTES, + TSDB_SML_TIMESTAMP_SECONDS, + TSDB_SML_TIMESTAMP_MILLI_SECONDS, + TSDB_SML_TIMESTAMP_MICRO_SECONDS, + TSDB_SML_TIMESTAMP_NANO_SECONDS, +} TSDB_SML_TIMESTAMP_TYPE; + typedef struct taosField { char name[65]; uint8_t type; @@ -136,37 +153,36 @@ DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); + DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); -DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); -DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); +DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); DLL_EXPORT void taos_stop_query(TAOS_RES *res); DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); -// TAOS_RES *taos_list_tables(TAOS *mysql, const char *wild); -// TAOS_RES *taos_list_dbs(TAOS *mysql, const char *wild); - -// TODO: the return value should be `const` DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); -DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); -DLL_EXPORT int taos_errno(TAOS_RES *tres); +DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); +DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); -typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); -DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval); +typedef void (*__taos_sub_fn_t)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); +DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, __taos_sub_fn_t fp, void *param, int interval); DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); @@ -175,8 +191,7 @@ DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp) DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); - -DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines); +DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); #ifdef __cplusplus } diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index b8ad59afee..1ed02ca21e 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -405,8 +405,10 @@ typedef struct { } SDropUserMsg, SDropAcctMsg; typedef struct { + int8_t type; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; + int8_t superUser; // denote if it is a super user or not int32_t reserve[8]; } SCreateUserMsg, SAlterUserMsg; @@ -830,7 +832,7 @@ typedef struct { } SVgroupsMsg, SVgroupsInfo; typedef struct { - char tbFname[TSDB_TABLE_FNAME_LEN]; // table id + char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name char stbFname[TSDB_TABLE_FNAME_LEN]; int32_t numOfTags; int32_t numOfColumns; diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 747d97b7a0..d6f7b46870 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -16,9 +16,9 @@ #ifndef _TD_TQ_H_ #define _TD_TQ_H_ +#include "mallocator.h" #include "os.h" #include "tutil.h" -#include "mallocator.h" #ifdef __cplusplus extern "C" { @@ -97,128 +97,125 @@ typedef struct TmqHeartbeatReq { typedef struct TmqHeartbeatRsp { } TmqHeartbeatRsp; -typedef struct TqTopicVhandle { +typedef struct STqTopicVhandle { int64_t topicId; // executor for filter void* filterExec; // callback for mnode // trigger when vnode list associated topic change void* (*mCallback)(void*, void*); -} TqTopicVhandle; - +} STqTopicVhandle; #define TQ_BUFFER_SIZE 8 -typedef struct TqBufferItem { +typedef struct STqBufferItem { int64_t offset; // executors are identical but not concurrent // so there must be a copy in each item void* executor; int64_t size; void* content; -} TqBufferItem; +} STqBufferItem; -typedef struct TqBufferHandle { +typedef struct STqBufferHandle { // char* topic; //c style, end with '\0' // int64_t cgId; // void* ahandle; - int64_t nextConsumeOffset; - int64_t topicId; - int32_t head; - int32_t tail; - TqBufferItem buffer[TQ_BUFFER_SIZE]; -} TqBufferHandle; + int64_t nextConsumeOffset; + int64_t floatingCursor; + int64_t topicId; + int32_t head; + int32_t tail; + STqBufferItem buffer[TQ_BUFFER_SIZE]; +} STqBufferHandle; -typedef struct TqListHandle { - TqBufferHandle bufHandle; - struct TqListHandle* next; -} TqListHandle; +typedef struct STqListHandle { + STqBufferHandle bufHandle; + struct STqListHandle* next; +} STqListHandle; -typedef struct TqGroupHandle { - int64_t cId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - TqListHandle* head; -} TqGroupHandle; +typedef struct STqGroupHandle { + int64_t cId; + int64_t cgId; + void* ahandle; + int32_t topicNum; + STqListHandle* head; +} STqGroupHandle; -typedef struct TqQueryExec { - void* src; - TqBufferItem* dest; - void* executor; -} TqQueryExec; +typedef struct STqQueryExec { + void* src; + STqBufferItem* dest; + void* executor; +} STqQueryExec; -typedef struct TqQueryMsg { - TqQueryExec* exec; - struct TqQueryMsg* next; -} TqQueryMsg; +typedef struct STqQueryMsg { + STqQueryExec* exec; + struct STqQueryMsg* next; +} STqQueryMsg; -typedef struct TqLogReader { +typedef struct STqLogReader { void* logHandle; int32_t (*logRead)(void* logHandle, void** data, int64_t ver); int64_t (*logGetFirstVer)(void* logHandle); int64_t (*logGetSnapshotVer)(void* logHandle); int64_t (*logGetLastVer)(void* logHandle); -} TqLogReader; +} STqLogReader; typedef struct STqCfg { // TODO } STqCfg; -typedef struct TqMemRef { - SMemAllocatorFactory *pAlloctorFactory; - SMemAllocator *pAllocator; -} TqMemRef; +typedef struct STqMemRef { + SMemAllocatorFactory* pAlloctorFactory; + SMemAllocator* pAllocator; +} STqMemRef; -typedef struct TqSerializedHead { +typedef struct STqSerializedHead { int16_t ver; int16_t action; int32_t checksum; int64_t ssize; char content[]; -} TqSerializedHead; +} STqSerializedHead; -typedef int (*TqSerializeFun)(const void* pObj, TqSerializedHead** ppHead); -typedef const void* (*TqDeserializeFun)(const TqSerializedHead* pHead, void** ppObj); -typedef void (*TqDeleteFun)(void*); +typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); +typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj); +typedef void (*FTqDelete)(void*); #define TQ_BUCKET_MASK 0xFF #define TQ_BUCKET_SIZE 256 #define TQ_PAGE_SIZE 4096 -//key + offset + size +// key + offset + size #define TQ_IDX_SIZE 24 -//4096 / 24 +// 4096 / 24 #define TQ_MAX_IDX_ONE_PAGE 170 -//24 * 170 +// 24 * 170 #define TQ_IDX_PAGE_BODY_SIZE 4080 -//4096 - 4080 +// 4096 - 4080 #define TQ_IDX_PAGE_HEAD_SIZE 16 -#define TQ_ACTION_CONST 0 -#define TQ_ACTION_INUSE 1 +#define TQ_ACTION_CONST 0 +#define TQ_ACTION_INUSE 1 #define TQ_ACTION_INUSE_CONT 2 -#define TQ_ACTION_INTXN 3 +#define TQ_ACTION_INTXN 3 -#define TQ_SVER 0 +#define TQ_SVER 0 -//TODO: inplace mode is not implemented -#define TQ_UPDATE_INPLACE 0 -#define TQ_UPDATE_APPEND 1 +// TODO: inplace mode is not implemented +#define TQ_UPDATE_INPLACE 0 +#define TQ_UPDATE_APPEND 1 #define TQ_DUP_INTXN_REWRITE 0 -#define TQ_DUP_INTXN_REJECT 2 +#define TQ_DUP_INTXN_REJECT 2 -static inline bool TqUpdateAppend(int32_t tqConfigFlag) { - return tqConfigFlag & TQ_UPDATE_APPEND; -} +static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } -static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { - return tqConfigFlag & TQ_DUP_INTXN_REJECT; -} +static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; -#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE + +#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE typedef struct TqMetaHandle { int64_t key; @@ -229,41 +226,41 @@ typedef struct TqMetaHandle { } STqMetaHandle; typedef struct TqMetaList { - STqMetaHandle handle; + STqMetaHandle handle; struct TqMetaList* next; - //struct TqMetaList* inTxnPrev; - //struct TqMetaList* inTxnNext; + // struct TqMetaList* inTxnPrev; + // struct TqMetaList* inTxnNext; struct TqMetaList* unpersistPrev; struct TqMetaList* unpersistNext; } STqMetaList; typedef struct TqMetaStore { - STqMetaList* bucket[TQ_BUCKET_SIZE]; - //a table head - STqMetaList* unpersistHead; - //TODO:temporaral use, to be replaced by unified tfile - int fileFd; - //TODO:temporaral use, to be replaced by unified tfile - int idxFd; - char* dirPath; - int32_t tqConfigFlag; - TqSerializeFun pSerializer; - TqDeserializeFun pDeserializer; - TqDeleteFun pDeleter; + STqMetaList* bucket[TQ_BUCKET_SIZE]; + // a table head + STqMetaList* unpersistHead; + // TODO:temporaral use, to be replaced by unified tfile + int fileFd; + // TODO:temporaral use, to be replaced by unified tfile + int idxFd; + char* dirPath; + int32_t tqConfigFlag; + FTqSerialize pSerializer; + FTqDeserialize pDeserializer; + FTqDelete pDeleter; } STqMetaStore; typedef struct STQ { // the collection of group handle // the handle of kvstore - char* path; - STqCfg* tqConfig; - TqLogReader* tqLogReader; - TqMemRef tqMemRef; + char* path; + STqCfg* tqConfig; + STqLogReader* tqLogReader; + STqMemRef tqMemRef; STqMetaStore* tqMeta; } STQ; // open in each vnode -STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac); +STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac); void tqDestroy(STQ*); // void* will be replace by a msg type @@ -272,19 +269,19 @@ int tqCommit(STQ*); int tqConsume(STQ*, TmqConsumeReq*); -TqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); +STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); -TqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqMoveOffsetToNext(TqGroupHandle*); -int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); -int tqRegisterContext(TqGroupHandle*, void* ahandle); -int tqLaunchQuery(TqGroupHandle*); -int tqSendLaunchQuery(TqGroupHandle*); +STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqMoveOffsetToNext(STqGroupHandle*); +int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); +int tqRegisterContext(STqGroupHandle*, void* ahandle); +int tqLaunchQuery(STqGroupHandle*); +int tqSendLaunchQuery(STqGroupHandle*); -int tqSerializeGroupHandle(const TqGroupHandle* gHandle, TqSerializedHead** ppHead); +int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead); -const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle** gHandle); +const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle); #ifdef __cplusplus } diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index d65b5ab570..49adaecfdd 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -153,7 +153,7 @@ typedef struct SParseContext { * @param msg extended error message if exists. * @return error code */ -int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen); +int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen); typedef enum { PAYLOAD_TYPE_KV = 0, diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 844757eeb5..a7d418d45e 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -84,8 +84,8 @@ typedef struct SProjectPhyNode { typedef struct SExchangePhyNode { SPhyNode node; - uint64_t templateId; - SArray *pSourceEpSet; // SEpSet + uint64_t srcTemplateId; // template id of datasource suplans + SArray *pSourceEpSet; // SEpSet, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; typedef struct SSubplanId { @@ -113,6 +113,8 @@ typedef struct SQueryDag { */ int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag); +int32_t qSetSuplanExecutionNode(SArray* subplans, SArray* nodes); + int32_t qExplainQuery(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, char** str); @@ -126,7 +128,7 @@ int32_t qSubPlanToString(struct SSubplan *pPhyNode, char** str); * @param pQueryPhyNode * @return */ -void* qDestroyQueryDag(struct SQueryDag* pDag); +void qDestroyQueryDag(struct SQueryDag* pDag); #ifdef __cplusplus } diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 27c7c534a2..9f51969dc1 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -27,7 +27,7 @@ OP_ENUM_MACRO(TableScan) OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) -OP_ENUM_MACRO(TableBlockInfoScan) +OP_ENUM_MACRO(SystemTableScan) OP_ENUM_MACRO(Aggregate) OP_ENUM_MACRO(Project) OP_ENUM_MACRO(Groupby) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index bafdc6c082..89f24cf3a4 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -16,49 +16,75 @@ #define _TD_WAL_H_ #include "os.h" +#include "tarray.h" #include "tdef.h" #include "tlog.h" -#include "tarray.h" #ifdef __cplusplus extern "C" { #endif extern int32_t wDebugFlag; -#define wFatal(...) { if (wDebugFlag & DEBUG_FATAL) { taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); }} -#define wError(...) { if (wDebugFlag & DEBUG_ERROR) { taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); }} -#define wWarn(...) { if (wDebugFlag & DEBUG_WARN) { taosPrintLog("WAL WARN ", 255, __VA_ARGS__); }} -#define wInfo(...) { if (wDebugFlag & DEBUG_INFO) { taosPrintLog("WAL ", 255, __VA_ARGS__); }} -#define wDebug(...) { if (wDebugFlag & DEBUG_DEBUG) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} -#define wTrace(...) { if (wDebugFlag & DEBUG_TRACE) { taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); }} +#define wFatal(...) \ + { \ + if (wDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("WAL FATAL ", 255, __VA_ARGS__); \ + } \ + } +#define wError(...) \ + { \ + if (wDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("WAL ERROR ", 255, __VA_ARGS__); \ + } \ + } +#define wWarn(...) \ + { \ + if (wDebugFlag & DEBUG_WARN) { \ + taosPrintLog("WAL WARN ", 255, __VA_ARGS__); \ + } \ + } +#define wInfo(...) \ + { \ + if (wDebugFlag & DEBUG_INFO) { \ + taosPrintLog("WAL ", 255, __VA_ARGS__); \ + } \ + } +#define wDebug(...) \ + { \ + if (wDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ + } \ + } +#define wTrace(...) \ + { \ + if (wDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("WAL ", wDebugFlag, __VA_ARGS__); \ + } \ + } -#define WAL_HEAD_VER 0 +#define WAL_HEAD_VER 0 #define WAL_NOSUFFIX_LEN 20 -#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN+1) -#define WAL_LOG_SUFFIX "log" +#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1) +#define WAL_LOG_SUFFIX "log" #define WAL_INDEX_SUFFIX "idx" -#define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) -#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) -#define WAL_FILE_LEN (WAL_PATH_LEN + 32) +#define WAL_REFRESH_MS 1000 +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead)) +#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) +#define WAL_FILE_LEN (WAL_PATH_LEN + 32) -#define WAL_CUR_FAILED 1 +#define WAL_CUR_FAILED 1 #pragma pack(push, 1) -typedef enum { - TAOS_WAL_NOLOG = 0, - TAOS_WAL_WRITE = 1, - TAOS_WAL_FSYNC = 2 -} EWalType; +typedef enum { TAOS_WAL_NOLOG = 0, TAOS_WAL_WRITE = 1, TAOS_WAL_FSYNC = 2 } EWalType; typedef struct SWalReadHead { - int8_t headVer; - uint8_t msgType; - int8_t reserved[2]; - int32_t len; - int64_t ingestTs; //not implemented - int64_t version; - char body[]; + int8_t headVer; + uint8_t msgType; + int8_t reserved[2]; + int32_t len; + int64_t ingestTs; // not implemented + int64_t version; + char body[]; } SWalReadHead; typedef struct { @@ -68,12 +94,12 @@ typedef struct { int32_t rollPeriod; // secs int64_t retentionSize; int64_t segSize; - EWalType level; // wal level + EWalType level; // wal level } SWalCfg; typedef struct { - uint32_t cksumHead; - uint32_t cksumBody; + uint32_t cksumHead; + uint32_t cksumBody; SWalReadHead head; } SWalHead; @@ -89,37 +115,37 @@ typedef struct SWal { // cfg SWalCfg cfg; int32_t fsyncSeq; - //meta + // meta SWalVer vers; int64_t writeLogTfd; int64_t writeIdxTfd; int32_t writeCur; - SArray* fileInfoSet; - //status + SArray *fileInfoSet; + // status int64_t totSize; int64_t lastRollSeq; - //ctl - int64_t refId; + // ctl + int64_t refId; pthread_mutex_t mutex; - //path + // path char path[WAL_PATH_LEN]; - //reusable write head + // reusable write head SWalHead writeHead; } SWal; // WAL HANDLE typedef struct SWalReadHandle { - SWal* pWal; - int64_t readLogTfd; - int64_t readIdxTfd; - int64_t curFileFirstVer; - int64_t curVersion; - int64_t capacity; - int64_t status; //if cursor valid - SWalHead* pHead; + SWal *pWal; + int64_t readLogTfd; + int64_t readIdxTfd; + int64_t curFileFirstVer; + int64_t curVersion; + int64_t capacity; + int64_t status; // if cursor valid + SWalHead *pHead; } SWalReadHandle; #pragma pack(pop) -//typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); +// typedef int32_t (*FWalWrite)(void *ahandle, void *pHead); // module initialization int32_t walInit(); @@ -141,15 +167,15 @@ int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely int32_t walBeginSnapshot(SWal *, int64_t ver); int32_t walEndSnapshot(SWal *); -//int32_t walDataCorrupted(SWal*); +// int32_t walDataCorrupted(SWal*); // read -SWalReadHandle* walOpenReadHandle(SWal *); -void walCloseReadHandle(SWalReadHandle *); -int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); +SWalReadHandle *walOpenReadHandle(SWal *); +void walCloseReadHandle(SWalReadHandle *); +int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); int32_t walRead(SWal *, SWalHead **, int64_t ver); -//int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); +// int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); // lifecycle check int64_t walGetFirstVer(SWal *); diff --git a/include/util/tdef.h b/include/util/tdef.h index a315a493f0..dfb53de58f 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -196,6 +196,7 @@ do { \ #define TSDB_AUTH_LEN 16 #define TSDB_PASSWORD_LEN 32 +#define TSDB_USET_PASSWORD_LEN 129 #define TSDB_VERSION_LEN 12 #define TSDB_LABEL_LEN 8 diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt index b46afa31b9..61b60e5ba6 100644 --- a/source/client/CMakeLists.txt +++ b/source/client/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries( taos PRIVATE common INTERFACE api - PRIVATE os util common transport parser + PRIVATE os util common transport parser catalog function query ) ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 3180923aff..87951d7944 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,14 +20,15 @@ extern "C" { #endif +#include #include "taos.h" #include "taosmsg.h" +#include "tdef.h" +#include "tep.h" #include "thash.h" #include "tlist.h" -#include "trpc.h" -#include "tdef.h" #include "tmsgtype.h" -#include "tep.h" +#include "trpc.h" typedef struct SQueryExecMetric { int64_t start; // start timestamp @@ -86,12 +87,22 @@ typedef struct STscObj { SAppInstInfo *pAppInfo; } STscObj; +typedef struct SClientResultInfo { + SSDataBlock *pData; + TAOS_FIELD *resultFields; + int32_t current; +} SClientResultInfo; + typedef struct SReqBody { tsem_t rspSem; // not used now void* fp; void* param; + int32_t paramLen; + SClientResultInfo* pResInfo; } SRequestBody; +#define ERROR_MSG_BUF_DEFAULT_SIZE 512 + typedef struct SRequestObj { uint64_t requestId; int32_t type; // request type @@ -118,7 +129,7 @@ extern int32_t tscReqRef; extern void *tscQhandle; extern int32_t tscConnRef; -extern int (*tscBuildMsg[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsg); +extern int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody); extern int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); int taos_init(); @@ -129,8 +140,6 @@ void destroyTscObj(void*pObj); void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t type); void destroyRequest(SRequestObj* pRequest); -TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); - void taos_init_imp(void); int taos_options_imp(TSDB_OPTION option, const char *str); @@ -139,6 +148,11 @@ void* openTransporter(const char *user, const char *auth); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); void initMsgHandleFp(); +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); +TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); + +void* doFetchRow(SRequestObj* pRequest); + #ifdef __cplusplus } #endif diff --git a/source/client/inc/tscLog.h b/source/client/inc/clientLog.h similarity index 96% rename from source/client/inc/tscLog.h rename to source/client/inc/clientLog.h index f205a50227..bfa2c0319b 100644 --- a/source/client/inc/tscLog.h +++ b/source/client/inc/clientLog.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TSCLOG_H -#define TDENGINE_TSCLOG_H +#ifndef TDENGINE_CLIENTLOG_H +#define TDENGINE_CLIENTLOG_H #ifdef __cplusplus extern "C" { diff --git a/source/client/src/client.c b/source/client/src/client.c index 863e96aa74..e69de29bb2 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tdef.h" -#include "tglobal.h" -#include "clientInt.h" -#include "tscLog.h" - -TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - int32_t p = (port != 0)? port:tsServerPort; - - tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); - if (user == NULL) { - user = TSDB_DEFAULT_USER; - } - - if (pass == NULL) { - pass = TSDB_DEFAULT_PASS; - } - - return taos_connect_internal(ip, user, pass, NULL, db, p); -} - -TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { - tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); - if (user == NULL) { - user = TSDB_DEFAULT_USER; - } - - if (auth == NULL) { - tscError("No auth info is given, failed to connect to server"); - return NULL; - } - - return taos_connect_internal(ip, user, NULL, auth, db, port); -} - -TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { - char ipStr[TSDB_EP_LEN] = {0}; - char dbStr[TSDB_DB_NAME_LEN] = {0}; - char userStr[TSDB_USER_LEN] = {0}; - char passStr[TSDB_PASSWORD_LEN] = {0}; - - strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); - strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipStr, userStr, passStr, dbStr, port); -} - diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2ae79f1947..fb98a7da46 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,11 +1,14 @@ -#include + #include "clientInt.h" +#include "clientLog.h" #include "tdef.h" #include "tep.h" #include "tglobal.h" #include "tmsgtype.h" +#include "tnote.h" +#include "tpagedfile.h" #include "tref.h" -#include "tscLog.h" +#include "parser.h" static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody); @@ -109,6 +112,71 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, return taosConnectImpl(ip, user, &secretEncrypt[0], db, port, NULL, NULL, pInst); } +TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { + STscObj *pTscObj = (STscObj *)taos; + if (sqlLen > (size_t) tsMaxSQLStringLen) { + tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + return NULL; + } + + nPrintTsc("%s", sql) + + SRequestObj* pRequest = createRequest(pTscObj, NULL, NULL, TSDB_SQL_SELECT); + if (pRequest == NULL) { + tscError("failed to malloc sqlObj"); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + pRequest->sqlstr = malloc(sqlLen + 1); + if (pRequest->sqlstr == NULL) { + tscError("0x%"PRIx64" failed to prepare sql string buffer", pRequest->self); + + pRequest->msgBuf = strdup("failed to prepare sql string buffer"); + terrno = pRequest->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRequest; + } + + strntolower(pRequest->sqlstr, sql, (int32_t)sqlLen); + pRequest->sqlstr[sqlLen] = 0; + + tscDebugL("0x%"PRIx64" SQL: %s", pRequest->requestId, pRequest->sqlstr); + + int32_t code = 0; + if (qIsInsertSql(pRequest->sqlstr, sqlLen)) { + // todo add + } else { + int32_t type = 0; + void* output = NULL; + int32_t outputLen = 0; + code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW) { + pRequest->type = type; + pRequest->body.param = output; + pRequest->body.paramLen = outputLen; + + SRequestMsgBody body = {0}; + buildRequestMsgFp[type](pRequest, &body); + + int64_t transporterId = 0; + sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId); + + tsem_wait(&pRequest->body.rspSem); + destroyConnectMsg(&body); + } else { + assert(0); + } + } + + if (code != TSDB_CODE_SUCCESS) { + pRequest->code = code; + return pRequest; + } + + return pRequest; +} + int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet) { pEpSet->version = 0; @@ -272,8 +340,8 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { pMsg->code = (*handleRequestRspFp[pRequest->type])(pRequest, pMsg->pCont, pMsg->contLen); } } else { - tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d", pRequest->requestId, taosMsg[pMsg->msgType], - tstrerror(pMsg->code), pMsg->contLen); + tscError("0x%" PRIx64 " SQL cmd:%s, code:%s rspLen:%d, elapsed time:%"PRId64" ms", pRequest->requestId, taosMsg[pMsg->msgType], + tstrerror(pMsg->code), pMsg->contLen, pRequest->metric.rsp - pRequest->metric.start); } taosReleaseRef(tscReqRef, requestRefId); @@ -281,3 +349,47 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { sem_post(&pRequest->body.rspSem); } + +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to connect to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (auth == NULL) { + tscError("No auth info is given, failed to connect to server"); + return NULL; + } + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port) { + char ipStr[TSDB_EP_LEN] = {0}; + char dbStr[TSDB_DB_NAME_LEN] = {0}; + char userStr[TSDB_USER_LEN] = {0}; + char passStr[TSDB_PASSWORD_LEN] = {0}; + + strncpy(ipStr, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userStr, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passStr, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbStr, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipStr, userStr, passStr, dbStr, port); +} + +void* doFetchRow(SRequestObj* pRequest) { + assert(pRequest != NULL); + SClientResultInfo* pResultInfo = pRequest->body.pResInfo; + + if (pResultInfo == NULL || pResultInfo->current >= pResultInfo->pData->info.rows) { + if (pResultInfo == NULL) { + pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo)); +// pRequest->body.pResInfo. + } + // current data set are exhausted, fetch more result from node +// if (pRes->row >= pRes->numOfRows && needToFetchNewBlock(pSql)) { +// taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); +// tsem_wait(&pSql->rspSem); +// } + } +} \ No newline at end of file diff --git a/source/client/src/clientmain.c b/source/client/src/clientMain.c similarity index 53% rename from source/client/src/clientmain.c rename to source/client/src/clientMain.c index ba80135850..5a1f55e573 100644 --- a/source/client/src/clientmain.c +++ b/source/client/src/clientMain.c @@ -1,5 +1,5 @@ #include "clientInt.h" -#include "trpc.h" +#include "clientLog.h" #include "os.h" #include "taosmsg.h" #include "tcache.h" @@ -7,7 +7,7 @@ #include "tglobal.h" #include "tnote.h" #include "tref.h" -#include "tscLog.h" +#include "trpc.h" #include "tsched.h" #include "ttime.h" #include "ttimezone.h" @@ -16,9 +16,6 @@ #define TSC_VAR_RELEASED 0 static int32_t sentinel = TSC_VAR_NOT_RELEASE; -static pthread_once_t tscinit = PTHREAD_ONCE_INIT; - -extern int32_t tscInitRes; int taos_options(TSDB_OPTION option, const void *arg, ...) { static int32_t lock = 0; @@ -36,11 +33,6 @@ int taos_options(TSDB_OPTION option, const void *arg, ...) { return ret; } -int taos_init() { - pthread_once(&tscinit, taos_init_imp); - return tscInitRes; -} - // this function may be called by user or system, or by both simultaneously. void taos_cleanup(void) { tscDebug("start to cleanup client environment"); @@ -65,6 +57,21 @@ void taos_cleanup(void) { taosCloseLog(); } +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + int32_t p = (port != 0)? port:tsServerPort; + + tscDebug("try to connect to %s:%u, user:%s db:%s", ip, p, user, db); + if (user == NULL) { + user = TSDB_DEFAULT_USER; + } + + if (pass == NULL) { + pass = TSDB_DEFAULT_PASS; + } + + return taos_connect_internal(ip, user, pass, NULL, db, p); +} + void taos_close(TAOS* taos) { if (taos == NULL) { return; @@ -76,10 +83,51 @@ void taos_close(TAOS* taos) { taosRemoveRef(tscConnRef, pTscObj->id); } -const char *taos_errstr(TAOS_RES *res) { +int taos_errno(TAOS_RES *tres) { + if (tres == NULL) { + return terrno; + } + return ((SRequestObj*) tres)->code; +} + +const char *taos_errstr(TAOS_RES *res) { + SRequestObj *pRequest = (SRequestObj *) res; + + if (pRequest == NULL) { + return (const char*) tstrerror(terrno); + } + + if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { + return pRequest->msgBuf; + } else { + return (const char*)tstrerror(pRequest->code); + } } void taos_free_result(TAOS_RES *res) { + SRequestObj* pRequest = (SRequestObj*) res; + destroyRequest(pRequest); +} -} \ No newline at end of file +TAOS_RES *taos_query(TAOS *taos, const char *sql) { + if (taos == NULL || sql == NULL) { + return NULL; + } + + return taos_query_l(taos, sql, strlen(sql)); +} + +TAOS_ROW taos_fetch_row(TAOS_RES *pRes) { + if (pRes == NULL) { + return NULL; + } + + SRequestObj *pRequest = (SRequestObj *) pRes; + if (pRequest->type == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + pRequest->type == TSDB_SQL_INSERT) { + return NULL; + } + + return doFetchRow(pRequest); +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 62466a096d..d860d56f75 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ -#include "os.h" +#include #include "clientInt.h" +#include "clientLog.h" +#include "os.h" #include "tmsgtype.h" #include "trpc.h" -#include "tscLog.h" int (*buildRequestMsgFp[TSDB_SQL_MAX])(SRequestObj *pRequest, SRequestMsgBody *pMsgBody) = {0}; int (*handleRequestRspFp[TSDB_SQL_MAX])(SRequestObj *pRequest, const char* pMsg, int32_t msgLen); @@ -3091,6 +3092,36 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) { #endif +int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT; + pMsgBody->msgLen = sizeof(SConnectMsg); + pMsgBody->requestObjRefId = pRequest->self; + + SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg)); + if (pConnect == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return -1; + } + + // TODO refactor full_name + char *db; // ugly code to move the space + + STscObj *pObj = pRequest->pTscObj; + pthread_mutex_lock(&pObj->mutex); + db = strstr(pObj->db, TS_PATH_DELIMITER); + + db = (db == NULL) ? pObj->db : db + 1; + tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + pthread_mutex_unlock(&pObj->mutex); + + pConnect->pid = htonl(appInfo.pid); + pConnect->startTime = htobe64(appInfo.startTime); + tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); + + pMsgBody->pData = pConnect; + return 0; +} + int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { STscObj *pTscObj = pRequest->pTscObj; @@ -3099,6 +3130,11 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { pConnect->connId = htonl(pConnect->connId); pConnect->clusterId = htonl(pConnect->clusterId); + assert(pConnect->epSet.numOfEps > 0); + for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) { + pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]); + } + // TODO refactor pthread_mutex_lock(&pTscObj->mutex); char temp[TSDB_TABLE_FNAME_LEN * 2] = {0}; @@ -3108,13 +3144,12 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { tstrncpy(pTscObj->db, temp, sizeof(pTscObj->db)); pthread_mutex_unlock(&pTscObj->mutex); - assert(pConnect->epSet.numOfEps > 0); if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); } for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%" PRIx64 " epSet.fqdn[%d]: %s, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pTscObj->id); + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id); } pTscObj->connId = pConnect->connId; @@ -3124,10 +3159,81 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); - // createHbObj(pTscObj); + return 0; +} + +int32_t buildCreateUserMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; + pMsgBody->msgLen = sizeof(SCreateUserMsg); + pMsgBody->requestObjRefId = pRequest->self; + pMsgBody->pData = pRequest->body.param; + return 0; +} + +int32_t buildShowMsg(SRequestObj* pRequest, SRequestMsgBody* pMsgBody) { + pMsgBody->msgType = TSDB_MSG_TYPE_SHOW; + pMsgBody->msgLen = pRequest->body.paramLen; + pMsgBody->requestObjRefId = pRequest->self; + pMsgBody->pData = pRequest->body.param; +} + +STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) { + assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2); + + size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema); + STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize); + + pTableMeta->tableType = pTableMetaMsg->tableType; + pTableMeta->vgId = pTableMetaMsg->vgId; + pTableMeta->suid = pTableMetaMsg->suid; + pTableMeta->uid = pTableMetaMsg->tuid; + + pTableMeta->tableInfo = (STableComInfo) { + .numOfTags = pTableMetaMsg->numOfTags, + .precision = pTableMetaMsg->precision, + .numOfColumns = pTableMetaMsg->numOfColumns, + }; + + pTableMeta->sversion = pTableMetaMsg->sversion; + pTableMeta->tversion = pTableMetaMsg->tversion; + + memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize); + + int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns; + for(int32_t i = 0; i < numOfTotalCols; ++i) { + pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; + } + + return pTableMeta; +} + +int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) { + SShowRsp* pShow = (SShowRsp *)pMsg; + pShow->showId = htonl(pShow->showId); + + STableMetaMsg *pMetaMsg = &(pShow->tableMeta); + pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); + + SSchema* pSchema = pMetaMsg->pSchema; + pMetaMsg->tuid = htobe64(pMetaMsg->tuid); + for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { + pSchema->bytes = htons(pSchema->bytes); + pSchema++; + } + + STableMeta* pTableMeta = createTableMetaFromMsg(pMetaMsg); + SSchema *pTableSchema = pTableMeta->schema; + + TAOS_FIELD* pFields = calloc(1, pTableMeta->tableInfo.numOfColumns); + for (int16_t i = 0; i < pTableMeta->tableInfo.numOfColumns; ++i, ++pSchema) { + tstrncpy(pFields[i].name, pTableSchema[i].name, tListLen(pFields[i].name)); + pFields[i].type = pTableSchema[i].type; + pFields[i].bytes = pTableSchema[i].bytes; + } + +// pRequest->body.resultFields = pFields; +// pRequest->body.numOfFields = pTableMeta->tableInfo.numOfColumns; - // launch a timer to send heartbeat to maintain the connection and send status to mnode - // taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pTscObj->rid, tscTmr, &pTscObj->pTimer); return 0; } @@ -3207,6 +3313,12 @@ void initMsgHandleFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; #endif -// buildRequestMsgFp[TSDB_SQL_CONNECT] = tscBuildConnectMsg; + buildRequestMsgFp[TSDB_SQL_CONNECT] = buildConnectMsg; handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; + + buildRequestMsgFp[TSDB_SQL_CREATE_USER] = buildCreateUserMsg; + + buildRequestMsgFp[TSDB_SQL_SHOW] = buildShowMsg; + handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; + } \ No newline at end of file diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 43d73bf3db..e378ebede4 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include "clientInt.h" +#include "clientLog.h" #include "os.h" #include "taosmsg.h" #include "tcache.h" @@ -20,12 +22,10 @@ #include "tglobal.h" #include "tnote.h" #include "tref.h" -#include "tscLog.h" +#include "trpc.h" #include "tsched.h" #include "ttime.h" -#include "trpc.h" #include "ttimezone.h" -#include "clientInt.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -35,6 +35,7 @@ int32_t tscReqRef = -1; int32_t tscConnRef = -1; void *tscQhandle = NULL; +static pthread_once_t tscinit = PTHREAD_ONCE_INIT; int32_t tsNumOfThreads = 1; volatile int32_t tscInitRes = 0; @@ -64,7 +65,7 @@ static void deregisterRequest(SRequestObj* pRequest) { SInstanceActivity* pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); - int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); tscDebug("0x%"PRIx64" free Request from connObj: 0x%"PRIx64", current:%d, app current:%d", pRequest->self, pTscObj->id, num, currentInst); taosReleaseRef(tscConnRef, pTscObj->id); @@ -172,6 +173,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty pRequest->pTscObj = pObj; pRequest->body.fp = fp; pRequest->body.param = param; + pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE); tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); @@ -250,6 +252,11 @@ void taos_init_imp(void) { tscDebug("client is initialized successfully"); } +int taos_init() { + pthread_once(&tscinit, taos_init_imp); + return tscInitRes; +} + int taos_options_imp(TSDB_OPTION option, const char *str) { SGlobalCfg *cfg = NULL; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 8da4caf7eb..62d2cb5b18 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -14,6 +14,7 @@ */ #include +#include #include #include "tglobal.h" #pragma GCC diagnostic ignored "-Wwrite-strings" @@ -36,5 +37,16 @@ TEST(testCase, driverInit_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); + + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = taos_fetch_row(pRes); + assert(pRow != NULL); + taos_close(pConn); } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 7219342d59..5fa4fe5359 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -431,7 +431,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *p pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; - pMeta->numOfColumns = htons(cols); + pMeta->numOfColumns = htonl(cols); pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/tq/inc/tqMetaStore.h index 4190125e44..253852b00f 100644 --- a/source/dnode/vnode/tq/inc/tqMetaStore.h +++ b/source/dnode/vnode/tq/inc/tqMetaStore.h @@ -25,9 +25,9 @@ extern "C" { STqMetaStore* tqStoreOpen(const char* path, - TqSerializeFun pSerializer, - TqDeserializeFun pDeserializer, - TqDeleteFun pDeleter, + FTqSerialize pSerializer, + FTqDeserialize pDeserializer, + FTqDelete pDeleter, int32_t tqConfigFlag ); int32_t tqStoreClose(STqMetaStore*); diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 5888141c58..1326666857 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -16,75 +16,70 @@ #include "tqInt.h" #include "tqMetaStore.h" -//static -//read next version data +// static +// read next version data // -//send to fetch queue +// send to fetch queue // -//handle management message +// handle management message // -int tqGetgHandleSSize(const TqGroupHandle *gHandle); +int tqGetgHandleSSize(const STqGroupHandle* gHandle); int tqBufHandleSSize(); int tqBufItemSSize(); -TqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - TqGroupHandle* gHandle; +STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + STqGroupHandle* gHandle; return NULL; } -void* tqSerializeListHandle(TqListHandle* listHandle, void* ptr); -void* tqSerializeBufHandle(TqBufferHandle* bufHandle, void* ptr); -void* tqSerializeBufItem(TqBufferItem* bufItem, void* ptr); +void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr); +void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr); +void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr); -const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle* bufHandle); -const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem* bufItem); +const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle); +const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem); -STQ* tqOpen(const char* path, STqCfg* tqConfig, TqLogReader* tqLogReader, SMemAllocatorFactory *allocFac) { +STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); - if(pTq == NULL) { - //TODO: memory error + if (pTq == NULL) { + // TODO: memory error return NULL; } pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; - pTq->tqMemRef.pAlloctorFactory = allocFac; - pTq->tqMemRef.pAllocator = allocFac->create(allocFac); - if(pTq->tqMemRef.pAllocator == NULL) { - //TODO + pTq->tqMemRef.pAlloctorFactory = allocFac; + pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + if (pTq->tqMemRef.pAllocator == NULL) { + // TODO } - pTq->tqMeta = tqStoreOpen(path, - (TqSerializeFun)tqSerializeGroupHandle, - (TqDeserializeFun)tqDeserializeGroupHandle, - free, - 0); - if(pTq->tqMeta == NULL) { - //TODO: free STQ + pTq->tqMeta = + tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0); + if (pTq->tqMeta == NULL) { + // TODO: free STQ return NULL; } return pTq; } -static int tqProtoCheck(TmqMsgHead *pMsg) { - return pMsg->protoVer == 0; -} +static int tqProtoCheck(TmqMsgHead* pMsg) { return pMsg->protoVer == 0; } -static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** ppQuery) { - //clean old item and move forward +static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) { + // clean old item and move forward int32_t consumeOffset = pAck->consumeOffset; - int idx = consumeOffset % TQ_BUFFER_SIZE; + int idx = consumeOffset % TQ_BUFFER_SIZE; ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); tfree(bHandle->buffer[idx].content); - if( 1 /* TODO: need to launch new query */) { - TqQueryMsg* pNewQuery = malloc(sizeof(TqQueryMsg)); - if(pNewQuery == NULL) { - //TODO: memory insufficient + if (1 /* TODO: need to launch new query */) { + STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); + if (pNewQuery == NULL) { + // TODO: memory insufficient return -1; } - //TODO: lock executor + // TODO: lock executor pNewQuery->exec->executor = bHandle->buffer[idx].executor; - //TODO: read from wal and assign to src + // TODO: read from wal and assign to src pNewQuery->exec->src = 0; pNewQuery->exec->dest = &bHandle->buffer[idx]; pNewQuery->next = *ppQuery; @@ -93,98 +88,94 @@ static int tqAckOneTopic(TqBufferHandle *bHandle, TmqOneAck *pAck, TqQueryMsg** return 0; } -static int tqAck(TqGroupHandle* gHandle, TmqAcks* pAcks) { - int32_t ackNum = pAcks->ackNum; - TmqOneAck *acks = pAcks->acks; - //double ptr for acks and list - int i = 0; - TqListHandle* node = gHandle->head; - int ackCnt = 0; - TqQueryMsg *pQuery = NULL; - while(i < ackNum && node->next) { - if(acks[i].topicId == node->next->bufHandle.topicId) { +static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { + int32_t ackNum = pAcks->ackNum; + TmqOneAck* acks = pAcks->acks; + // double ptr for acks and list + int i = 0; + STqListHandle* node = gHandle->head; + int ackCnt = 0; + STqQueryMsg* pQuery = NULL; + while (i < ackNum && node->next) { + if (acks[i].topicId == node->next->bufHandle.topicId) { ackCnt++; tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); - } else if(acks[i].topicId < node->next->bufHandle.topicId) { + } else if (acks[i].topicId < node->next->bufHandle.topicId) { i++; } else { node = node->next; } } - if(pQuery) { - //post message + if (pQuery) { + // post message } return ackCnt; } -static int tqCommitTCGroup(TqGroupHandle* handle) { - //persist modification into disk +static int tqCommitTCGroup(STqGroupHandle* handle) { + // persist modification into disk return 0; } -int tqCreateTCGroup(STQ *pTq, int64_t topicId, int64_t cgId, int64_t cId, TqGroupHandle** handle) { - //create in disk - TqGroupHandle* gHandle = (TqGroupHandle*)malloc(sizeof(TqGroupHandle)); - if(gHandle == NULL) { - //TODO +int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) { + // create in disk + STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle)); + if (gHandle == NULL) { + // TODO return -1; } - memset(gHandle, 0, sizeof(TqGroupHandle)); + memset(gHandle, 0, sizeof(STqGroupHandle)); return 0; } -TqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - TqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); - if(gHandle == NULL) { +STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); + if (gHandle == NULL) { int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return NULL; } } - //create - //open + // create + // open return gHandle; } -int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - return 0; -} +int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; } int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - //delete from disk + // delete from disk return 0; } -static int tqFetch(TqGroupHandle* gHandle, void** msg) { - TqListHandle* head = gHandle->head; - TqListHandle* node = head; - int totSize = 0; - //TODO: make it a macro - int sizeLimit = 4 * 1024; +static int tqFetch(STqGroupHandle* gHandle, void** msg) { + STqListHandle* head = gHandle->head; + STqListHandle* node = head; + int totSize = 0; + // TODO: make it a macro + int sizeLimit = 4 * 1024; TmqMsgContent* buffer = malloc(sizeLimit); - if(buffer == NULL) { - //TODO:memory insufficient + if (buffer == NULL) { + // TODO:memory insufficient return -1; } - //iterate the list to get msgs of all topics - //until all topic iterated or msgs over sizeLimit - while(node->next) { + // iterate the list to get msgs of all topics + // until all topic iterated or msgs over sizeLimit + while (node->next) { node = node->next; - TqBufferHandle* bufHandle = &node->bufHandle; - int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; - if(bufHandle->buffer[idx].content != NULL && - bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset - ) { + STqBufferHandle* bufHandle = &node->bufHandle; + int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; + if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) { totSize += bufHandle->buffer[idx].size; - if(totSize > sizeLimit) { - void *ptr = realloc(buffer, totSize); - if(ptr == NULL) { + if (totSize > sizeLimit) { + void* ptr = realloc(buffer, totSize); + if (ptr == NULL) { totSize -= bufHandle->buffer[idx].size; - //TODO:memory insufficient - //return msgs already copied + // TODO:memory insufficient + // return msgs already copied break; } } @@ -194,7 +185,7 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) { buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); - if(totSize > sizeLimit) { + if (totSize > sizeLimit) { break; } } @@ -202,104 +193,98 @@ static int tqFetch(TqGroupHandle* gHandle, void** msg) { return totSize; } -TqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { - return NULL; -} +STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; } -int tqLaunchQuery(TqGroupHandle* gHandle) { - return 0; -} +int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; } -int tqSendLaunchQuery(TqGroupHandle* gHandle) { - return 0; -} +int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; } /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ - /*return 0;*/ +/*return 0;*/ /*}*/ -int tqPushMsg(STQ* pTq , void* p, int64_t version) { - //add reference - //judge and launch new query +int tqPushMsg(STQ* pTq, void* p, int64_t version) { + // add reference + // judge and launch new query return 0; } int tqCommit(STQ* pTq) { - //do nothing + // do nothing return 0; } int tqConsume(STQ* pTq, TmqConsumeReq* pMsg) { - if(!tqProtoCheck((TmqMsgHead *)pMsg)) { - //proto version invalid + if (!tqProtoCheck((TmqMsgHead*)pMsg)) { + // proto version invalid return -1; } - int64_t clientId = pMsg->head.clientId; - TqGroupHandle *gHandle = tqGetGroupHandle(pTq, clientId); - if(gHandle == NULL) { - //client not connect + int64_t clientId = pMsg->head.clientId; + STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId); + if (gHandle == NULL) { + // client not connect return -1; } - if(pMsg->acks.ackNum != 0) { - if(tqAck(gHandle, &pMsg->acks) != 0) { - //ack not success + if (pMsg->acks.ackNum != 0) { + if (tqAck(gHandle, &pMsg->acks) != 0) { + // ack not success return -1; } } - TmqConsumeRsp *pRsp = (TmqConsumeRsp*) pMsg; + TmqConsumeRsp* pRsp = (TmqConsumeRsp*)pMsg; - if(tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { - //fetch error + if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { + // fetch error return -1; } - //judge and launch new query - if(tqLaunchQuery(gHandle)) { - //launch query error + // judge and launch new query + if (tqLaunchQuery(gHandle)) { + // launch query error return -1; } return 0; } -int tqSerializeGroupHandle(const TqGroupHandle *gHandle, TqSerializedHead** ppHead) { - //calculate size - int sz = tqGetgHandleSSize(gHandle) + sizeof(TqSerializedHead); - if(sz > (*ppHead)->ssize) { +int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) { + // calculate size + int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead); + if (sz > (*ppHead)->ssize) { void* tmpPtr = realloc(*ppHead, sz); - if(tmpPtr == NULL) { + if (tmpPtr == NULL) { free(*ppHead); - //TODO: memory err + // TODO: memory err return -1; } *ppHead = tmpPtr; (*ppHead)->ssize = sz; } void* ptr = (*ppHead)->content; - //do serialization + // do serialization *(int64_t*)ptr = gHandle->cId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); *(int64_t*)ptr = gHandle->cgId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); *(int32_t*)ptr = gHandle->topicNum; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if(gHandle->topicNum > 0) { + if (gHandle->topicNum > 0) { tqSerializeListHandle(gHandle->head, ptr); } return 0; } -void* tqSerializeListHandle(TqListHandle *listHandle, void* ptr) { - TqListHandle *node = listHandle; +void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) { + STqListHandle* node = listHandle; ASSERT(node != NULL); - while(node) { + while (node) { ptr = tqSerializeBufHandle(&node->bufHandle, ptr); node = node->next; } return ptr; } -void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) { +void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) { *(int64_t*)ptr = bufHandle->nextConsumeOffset; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); *(int64_t*)ptr = bufHandle->topicId; @@ -308,21 +293,21 @@ void* tqSerializeBufHandle(TqBufferHandle *bufHandle, void* ptr) { ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); *(int32_t*)ptr = bufHandle->tail; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - for(int i = 0; i < TQ_BUFFER_SIZE; i++) { + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); } return ptr; } -void* tqSerializeBufItem(TqBufferItem *bufItem, void* ptr) { - //TODO: do we need serialize this? - //mainly for executor +void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) { + // TODO: do we need serialize this? + // mainly for executor return ptr; } -const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandle **ppGHandle) { - TqGroupHandle *gHandle = *ppGHandle; - const void* ptr = pHead->content; +const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) { + STqGroupHandle* gHandle = *ppGHandle; + const void* ptr = pHead->content; gHandle->cId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); gHandle->cgId = *(int64_t*)ptr; @@ -331,20 +316,20 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl gHandle->topicNum = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); gHandle->head = NULL; - TqListHandle *node = gHandle->head; - for(int i = 0; i < gHandle->topicNum; i++) { - if(gHandle->head == NULL) { - if((node = malloc(sizeof(TqListHandle))) == NULL) { - //TODO: error + STqListHandle* node = gHandle->head; + for (int i = 0; i < gHandle->topicNum; i++) { + if (gHandle->head == NULL) { + if ((node = malloc(sizeof(STqListHandle))) == NULL) { + // TODO: error return NULL; } - node->next= NULL; - ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); + node->next = NULL; + ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); gHandle->head = node; } else { - node->next = malloc(sizeof(TqListHandle)); - if(node->next == NULL) { - //TODO: error + node->next = malloc(sizeof(STqListHandle)); + if (node->next == NULL) { + // TODO: error return NULL; } node->next->next = NULL; @@ -355,7 +340,7 @@ const void* tqDeserializeGroupHandle(const TqSerializedHead* pHead, TqGroupHandl return ptr; } -const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle) { +const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) { const void* ptr = pBytes; bufHandle->nextConsumeOffset = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); @@ -365,32 +350,30 @@ const void* tqDeserializeBufHandle(const void* pBytes, TqBufferHandle *bufHandle ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); bufHandle->tail = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - for(int i = 0; i < TQ_BUFFER_SIZE; i++) { + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); } return ptr; } -const void* tqDeserializeBufItem(const void* pBytes, TqBufferItem *bufItem) { - return pBytes; +const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; } + +// TODO: make this a macro +int tqGetgHandleSSize(const STqGroupHandle* gHandle) { + return sizeof(int64_t) * 2 // cId + cgId + + sizeof(int32_t) // topicNum + + gHandle->topicNum * tqBufHandleSSize(); } -//TODO: make this a macro -int tqGetgHandleSSize(const TqGroupHandle *gHandle) { - return sizeof(int64_t) * 2 //cId + cgId - + sizeof(int32_t) //topicNum - + gHandle->topicNum * tqBufHandleSSize(); -} - -//TODO: make this a macro +// TODO: make this a macro int tqBufHandleSSize() { - return sizeof(int64_t) * 2 // nextConsumeOffset + topicId - + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqBufItemSSize(); + return sizeof(int64_t) * 2 // nextConsumeOffset + topicId + + sizeof(int32_t) * 2 // head + tail + + TQ_BUFFER_SIZE * tqBufItemSSize(); } int tqBufItemSSize() { - //TODO: do this need serialization? - //mainly for executor + // TODO: do this need serialization? + // mainly for executor return 0; } diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/tq/src/tqMetaStore.c index 3f40c94f24..082f0ad28e 100644 --- a/source/dnode/vnode/tq/src/tqMetaStore.c +++ b/source/dnode/vnode/tq/src/tqMetaStore.c @@ -13,20 +13,20 @@ * along with this program. If not, see . */ #include "tqMetaStore.h" -//TODO:replace by an abstract file layer -#include "osDir.h" +// TODO:replace by an abstract file layer #include #include #include +#include "osDir.h" #define TQ_META_NAME "tq.meta" -#define TQ_IDX_NAME "tq.idx" +#define TQ_IDX_NAME "tq.idx" static int32_t tqHandlePutCommitted(STqMetaStore*, int64_t key, void* value); static void* tqHandleGetUncommitted(STqMetaStore*, int64_t key); -static inline void tqLinkUnpersist(STqMetaStore *pMeta, STqMetaList* pNode) { - if(pNode->unpersistNext == NULL) { +static inline void tqLinkUnpersist(STqMetaStore* pMeta, STqMetaList* pNode) { + if (pNode->unpersistNext == NULL) { pNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pNode->unpersistPrev = pMeta->unpersistHead; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNode; @@ -41,24 +41,24 @@ static inline int tqSeekLastPage(int fd) { return lseek(fd, curPageOffset, SEEK_SET); } -//TODO: the struct is tightly coupled with index entry -typedef struct TqIdxPageHead { +// TODO: the struct is tightly coupled with index entry +typedef struct STqIdxPageHead { int16_t writeOffset; int8_t unused[14]; -} TqIdxPageHead; +} STqIdxPageHead; -typedef struct TqIdxPageBuf { - TqIdxPageHead head; - char buffer[TQ_IDX_PAGE_BODY_SIZE]; -} TqIdxPageBuf; +typedef struct STqIdxPageBuf { + STqIdxPageHead head; + char buffer[TQ_IDX_PAGE_BODY_SIZE]; +} STqIdxPageBuf; -static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) { +static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { int offset = tqSeekLastPage(fd); int nBytes; - if((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { return -1; } - if(nBytes == 0) { + if (nBytes == 0) { memset(pBuf, 0, TQ_PAGE_SIZE); pBuf->head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; } @@ -67,28 +67,24 @@ static inline int tqReadLastPage(int fd, TqIdxPageBuf* pBuf) { return lseek(fd, offset, SEEK_SET); } -STqMetaStore* tqStoreOpen(const char* path, - TqSerializeFun serializer, - TqDeserializeFun deserializer, - TqDeleteFun deleter, - int32_t tqConfigFlag - ) { - STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); - if(pMeta == NULL) { - //close +STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter, + int32_t tqConfigFlag) { + STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); + if (pMeta == NULL) { + // close return NULL; } memset(pMeta, 0, sizeof(STqMetaStore)); - //concat data file name and index file name + // concat data file name and index file name size_t pathLen = strlen(path); - pMeta->dirPath = malloc(pathLen+1); - if(pMeta->dirPath != NULL) { - //TODO: memory insufficient + pMeta->dirPath = malloc(pathLen + 1); + if (pMeta->dirPath != NULL) { + // TODO: memory insufficient } strcpy(pMeta->dirPath, path); - - char name[pathLen+10]; + + char name[pathLen + 10]; strcpy(name, path); if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { @@ -96,98 +92,96 @@ STqMetaStore* tqStoreOpen(const char* path, } strcat(name, "/" TQ_IDX_NAME); int idxFd = open(name, O_RDWR | O_CREAT, 0755); - if(idxFd < 0) { + if (idxFd < 0) { ASSERT(false); - //close file - //free memory + // close file + // free memory return NULL; } pMeta->idxFd = idxFd; pMeta->unpersistHead = malloc(sizeof(STqMetaList)); - if(pMeta->unpersistHead == NULL) { + if (pMeta->unpersistHead == NULL) { ASSERT(false); - //close file - //free memory + // close file + // free memory return NULL; } memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); - pMeta->unpersistHead->unpersistNext - = pMeta->unpersistHead->unpersistPrev - = pMeta->unpersistHead; + pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; strcpy(name, path); strcat(name, "/" TQ_META_NAME); int fileFd = open(name, O_RDWR | O_CREAT, 0755); - if(fileFd < 0){ + if (fileFd < 0) { ASSERT(false); return NULL; } pMeta->fileFd = fileFd; - + pMeta->pSerializer = serializer; pMeta->pDeserializer = deserializer; pMeta->pDeleter = deleter; pMeta->tqConfigFlag = tqConfigFlag; - //read idx file and load into memory - TqIdxPageBuf idxBuf; - TqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); - if(serializedObj == NULL) { - //TODO:memory insufficient + // read idx file and load into memory + STqIdxPageBuf idxBuf; + STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); + if (serializedObj == NULL) { + // TODO:memory insufficient } - int idxRead; - int allocated = TQ_PAGE_SIZE; + int idxRead; + int allocated = TQ_PAGE_SIZE; bool readEnd = false; - while((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { - if(idxRead == -1) { - //TODO: handle error + while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { + if (idxRead == -1) { + // TODO: handle error ASSERT(false); } ASSERT(idxBuf.head.writeOffset == idxRead); - //loop read every entry - for(int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList *pNode = malloc(sizeof(STqMetaList)); - if(pNode == NULL) { - //TODO: free memory and return error + // loop read every entry + for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { + STqMetaList* pNode = malloc(sizeof(STqMetaList)); + if (pNode == NULL) { + // TODO: free memory and return error } memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); lseek(fileFd, pNode->handle.offset, SEEK_SET); - if(allocated < pNode->handle.serializedSize) { - void *ptr = realloc(serializedObj, pNode->handle.serializedSize); - if(ptr == NULL) { - //TODO: memory insufficient + if (allocated < pNode->handle.serializedSize) { + void* ptr = realloc(serializedObj, pNode->handle.serializedSize); + if (ptr == NULL) { + // TODO: memory insufficient } serializedObj = ptr; allocated = pNode->handle.serializedSize; } serializedObj->ssize = pNode->handle.serializedSize; - if(read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { - //TODO: read error + if (read(fileFd, serializedObj, pNode->handle.serializedSize) != pNode->handle.serializedSize) { + // TODO: read error } - if(serializedObj->action == TQ_ACTION_INUSE) { - if(serializedObj->ssize != sizeof(TqSerializedHead)) { + if (serializedObj->action == TQ_ACTION_INUSE) { + if (serializedObj->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } - } else if(serializedObj->action == TQ_ACTION_INTXN) { - if(serializedObj->ssize != sizeof(TqSerializedHead)) { + } else if (serializedObj->action == TQ_ACTION_INTXN) { + if (serializedObj->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; } - } else if(serializedObj->action == TQ_ACTION_INUSE_CONT) { - if(serializedObj->ssize != sizeof(TqSerializedHead)) { + } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) { + if (serializedObj->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } - TqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); - if(ptr->ssize != sizeof(TqSerializedHead)) { + STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); + if (ptr->ssize != sizeof(STqSerializedHead)) { pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; @@ -196,22 +190,21 @@ STqMetaStore* tqStoreOpen(const char* path, ASSERT(0); } - //put into list - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; + // put into list + int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; STqMetaList* pBucketNode = pMeta->bucket[bucketKey]; - if(pBucketNode == NULL) { + if (pBucketNode == NULL) { pMeta->bucket[bucketKey] = pNode; - } else if(pBucketNode->handle.key == pNode->handle.key) { + } else if (pBucketNode->handle.key == pNode->handle.key) { pNode->next = pBucketNode->next; pMeta->bucket[bucketKey] = pNode; } else { - while(pBucketNode->next && - pBucketNode->next->handle.key != pNode->handle.key) { - pBucketNode = pBucketNode->next; + while (pBucketNode->next && pBucketNode->next->handle.key != pNode->handle.key) { + pBucketNode = pBucketNode->next; } - if(pBucketNode->next) { + if (pBucketNode->next) { ASSERT(pBucketNode->next->handle.key == pNode->handle.key); - STqMetaList *pNodeFound = pBucketNode->next; + STqMetaList* pNodeFound = pBucketNode->next; pNode->next = pNodeFound->next; pBucketNode->next = pNode; pBucketNode = pNodeFound; @@ -221,13 +214,11 @@ STqMetaStore* tqStoreOpen(const char* path, pBucketNode = NULL; } } - if(pBucketNode) { - if(pBucketNode->handle.valueInUse - && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pBucketNode) { + if (pBucketNode->handle.valueInUse && pBucketNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pBucketNode->handle.valueInUse); } - if(pBucketNode->handle.valueInTxn - && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pBucketNode->handle.valueInTxn && pBucketNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pBucketNode->handle.valueInTxn); } free(pBucketNode); @@ -239,23 +230,21 @@ STqMetaStore* tqStoreOpen(const char* path, } int32_t tqStoreClose(STqMetaStore* pMeta) { - //commit data and idx + // commit data and idx tqStorePersist(pMeta); - ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next==NULL); + ASSERT(pMeta->unpersistHead && pMeta->unpersistHead->next == NULL); close(pMeta->fileFd); close(pMeta->idxFd); - //free memory - for(int i = 0; i < TQ_BUCKET_SIZE; i++) { + // free memory + for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; - while(pNode) { + while (pNode) { ASSERT(pNode->unpersistNext == NULL); ASSERT(pNode->unpersistPrev == NULL); - if(pNode->handle.valueInTxn - && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; @@ -272,17 +261,15 @@ int32_t tqStoreClose(STqMetaStore* pMeta) { int32_t tqStoreDelete(STqMetaStore* pMeta) { close(pMeta->fileFd); close(pMeta->idxFd); - //free memory - for(int i = 0; i < TQ_BUCKET_SIZE; i++) { + // free memory + for (int i = 0; i < TQ_BUCKET_SIZE; i++) { STqMetaList* pNode = pMeta->bucket[i]; pMeta->bucket[i] = NULL; - while(pNode) { - if(pNode->handle.valueInTxn - && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.valueInTxn && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } STqMetaList* next = pNode->next; @@ -297,26 +284,26 @@ int32_t tqStoreDelete(STqMetaStore* pMeta) { return 0; } -//TODO: wrap in tfile +// TODO: wrap in tfile int32_t tqStorePersist(STqMetaStore* pMeta) { - TqIdxPageBuf idxBuf; - int64_t* bufPtr = (int64_t*)idxBuf.buffer; - STqMetaList *pHead = pMeta->unpersistHead; - STqMetaList *pNode = pHead->unpersistNext; - TqSerializedHead *pSHead = malloc(sizeof(TqSerializedHead)); - if(pSHead == NULL) { - //TODO: memory error + STqIdxPageBuf idxBuf; + int64_t* bufPtr = (int64_t*)idxBuf.buffer; + STqMetaList* pHead = pMeta->unpersistHead; + STqMetaList* pNode = pHead->unpersistNext; + STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); + if (pSHead == NULL) { + // TODO: memory error return -1; } pSHead->ver = TQ_SVER; pSHead->checksum = 0; - pSHead->ssize = sizeof(TqSerializedHead); - int allocatedSize = sizeof(TqSerializedHead); + pSHead->ssize = sizeof(STqSerializedHead); + int allocatedSize = sizeof(STqSerializedHead); int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); tqReadLastPage(pMeta->idxFd, &idxBuf); - if(idxBuf.head.writeOffset == TQ_PAGE_SIZE) { + if (idxBuf.head.writeOffset == TQ_PAGE_SIZE) { lseek(pMeta->idxFd, 0, SEEK_END); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; @@ -324,18 +311,18 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { bufPtr = POINTER_SHIFT(&idxBuf, idxBuf.head.writeOffset); } - while(pHead != pNode) { + while (pHead != pNode) { int nBytes = 0; - if(pNode->handle.valueInUse) { - if(pNode->handle.valueInTxn) { + if (pNode->handle.valueInUse) { + if (pNode->handle.valueInTxn) { pSHead->action = TQ_ACTION_INUSE_CONT; } else { pSHead->action = TQ_ACTION_INUSE; } - if(pNode->handle.valueInUse == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(TqSerializedHead); + if (pNode->handle.valueInUse == TQ_DELETE_TOKEN) { + pSHead->ssize = sizeof(STqSerializedHead); } else { pMeta->pSerializer(pNode->handle.valueInUse, &pSHead); } @@ -343,10 +330,10 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { ASSERT(nBytes == pSHead->ssize); } - if(pNode->handle.valueInTxn) { + if (pNode->handle.valueInTxn) { pSHead->action = TQ_ACTION_INTXN; - if(pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { - pSHead->ssize = sizeof(TqSerializedHead); + if (pNode->handle.valueInTxn == TQ_DELETE_TOKEN) { + pSHead->ssize = sizeof(STqSerializedHead); } else { pMeta->pSerializer(pNode->handle.valueInTxn, &pSHead); } @@ -357,42 +344,39 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pNode->handle.offset = offset; offset += nBytes; - //write idx file - //TODO: endian check and convert + // write idx file + // TODO: endian check and convert *(bufPtr++) = pNode->handle.key; *(bufPtr++) = pNode->handle.offset; *(bufPtr++) = (int64_t)nBytes; idxBuf.head.writeOffset += TQ_IDX_SIZE; - if(idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { + if (idxBuf.head.writeOffset >= TQ_PAGE_SIZE) { nBytes = write(pMeta->idxFd, &idxBuf, TQ_PAGE_SIZE); - //TODO: handle error with tfile + // TODO: handle error with tfile ASSERT(nBytes == TQ_PAGE_SIZE); memset(&idxBuf, 0, TQ_PAGE_SIZE); idxBuf.head.writeOffset = TQ_IDX_PAGE_HEAD_SIZE; bufPtr = (int64_t*)&idxBuf.buffer; } - //remove from unpersist list + // remove from unpersist list pHead->unpersistNext = pNode->unpersistNext; pHead->unpersistNext->unpersistPrev = pHead; pNode->unpersistPrev = pNode->unpersistNext = NULL; pNode = pHead->unpersistNext; - //remove from bucket - if(pNode->handle.valueInUse == TQ_DELETE_TOKEN && - pNode->handle.valueInTxn == NULL - ) { - int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; + // remove from bucket + if (pNode->handle.valueInUse == TQ_DELETE_TOKEN && pNode->handle.valueInTxn == NULL) { + int bucketKey = pNode->handle.key & TQ_BUCKET_MASK; STqMetaList* pBucketHead = pMeta->bucket[bucketKey]; - if(pBucketHead == pNode) { + if (pBucketHead == pNode) { pMeta->bucket[bucketKey] = pNode->next; } else { STqMetaList* pBucketNode = pBucketHead; - while(pBucketNode->next != NULL - && pBucketNode->next != pNode) { - pBucketNode = pBucketNode->next; + while (pBucketNode->next != NULL && pBucketNode->next != pNode) { + pBucketNode = pBucketNode->next; } - //impossible for pBucket->next == NULL + // impossible for pBucket->next == NULL ASSERT(pBucketNode->next == pNode); pBucketNode->next = pNode->next; } @@ -400,46 +384,45 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { } } - //write left bytes + // write left bytes free(pSHead); - //TODO: write new version in tfile - if((char*)bufPtr != idxBuf.buffer) { + // TODO: write new version in tfile + if ((char*)bufPtr != idxBuf.buffer) { int nBytes = write(pMeta->idxFd, &idxBuf, idxBuf.head.writeOffset); - //TODO: handle error in tfile + // TODO: handle error in tfile ASSERT(nBytes == idxBuf.head.writeOffset); } - //TODO: using fsync in tfile + // TODO: using fsync in tfile fsync(pMeta->idxFd); fsync(pMeta->fileFd); return 0; } static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - //TODO: think about thread safety - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + // TODO: think about thread safety + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } - //change pointer ownership + // change pointer ownership pNode->handle.valueInUse = value; return 0; } else { pNode = pNode->next; } } - STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); - if(pNewNode == NULL) { - //TODO: memory error + STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + if (pNewNode == NULL) { + // TODO: memory error return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInUse = value; - //put into unpersist list + // put into unpersist list pNewNode->unpersistPrev = pMeta->unpersistHead; pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; pMeta->unpersistHead->unpersistNext->unpersistPrev = pNewNode; @@ -448,12 +431,11 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu } void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInUse != NULL - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { return pNode->handle.valueInUse; } else { return NULL; @@ -466,12 +448,11 @@ void* tqHandleGet(STqMetaStore* pMeta, int64_t key) { } void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInUse != NULL - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInUse != NULL && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { tqLinkUnpersist(pMeta, pNode); return pNode->handle.valueInUse; } else { @@ -485,16 +466,16 @@ void* tqHandleTouchGet(STqMetaStore* pMeta, int64_t key) { } static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* value) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - //TODO: think about thread safety - if(pNode->handle.valueInTxn) { - if(TqDupIntxnReject(pMeta->tqConfigFlag)) { + while (pNode) { + if (pNode->handle.key == key) { + // TODO: think about thread safety + if (pNode->handle.valueInTxn) { + if (TqDupIntxnReject(pMeta->tqConfigFlag)) { return -2; } - if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } } @@ -505,9 +486,9 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va pNode = pNode->next; } } - STqMetaList *pNewNode = malloc(sizeof(STqMetaList)); - if(pNewNode == NULL) { - //TODO: memory error + STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + if (pNewNode == NULL) { + // TODO: memory error return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -519,14 +500,12 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va return 0; } -int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { - return tqHandlePutImpl(pMeta, key, value); -} +int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return tqHandlePutImpl(pMeta, key, value); } int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { - void *vmem = malloc(vsize); - if(vmem == NULL) { - //TODO: memory error + void* vmem = malloc(vsize); + if (vmem == NULL) { + // TODO: memory error return -1; } memcpy(vmem, value, vsize); @@ -534,12 +513,11 @@ int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vs } static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInTxn != NULL - && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn != NULL && pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { return pNode->handle.valueInTxn; } else { return NULL; @@ -552,15 +530,14 @@ static void* tqHandleGetUncommitted(STqMetaStore* pMeta, int64_t key) { } int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInTxn == NULL) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn == NULL) { return -1; } - if(pNode->handle.valueInUse - && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } pNode->handle.valueInUse = pNode->handle.valueInTxn; @@ -575,12 +552,12 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { } int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.key == key) { - if(pNode->handle.valueInTxn) { - if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + while (pNode) { + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); } pNode->handle.valueInTxn = NULL; @@ -596,13 +573,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { } int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { - int64_t bucketKey = key & TQ_BUCKET_MASK; + int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; - while(pNode) { - if(pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - if(pNode->handle.valueInTxn) { + while (pNode) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn) { pMeta->pDeleter(pNode->handle.valueInTxn); } + pNode->handle.valueInTxn = TQ_DELETE_TOKEN; tqLinkUnpersist(pMeta, pNode); return 0; @@ -610,11 +588,9 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - //no such key + // no such key return -1; } -//TODO: clean deleted idx and data from persistent file -int32_t tqStoreCompact(STqMetaStore *pMeta) { - return 0; -} +// TODO: clean deleted idx and data from persistent file +int32_t tqStoreCompact(STqMetaStore* pMeta) { return 0; } diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/tq/test/tqMetaTest.cpp index bbd436ab1a..58263efa71 100644 --- a/source/dnode/vnode/tq/test/tqMetaTest.cpp +++ b/source/dnode/vnode/tq/test/tqMetaTest.cpp @@ -9,17 +9,17 @@ struct Foo { int32_t a; }; -int FooSerializer(const void* pObj, TqSerializedHead** ppHead) { +int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { Foo* foo = (Foo*) pObj; - if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(TqSerializedHead) + sizeof(int32_t)) { - *ppHead = (TqSerializedHead*)realloc(*ppHead, sizeof(TqSerializedHead) + sizeof(int32_t)); - (*ppHead)->ssize = sizeof(TqSerializedHead) + sizeof(int32_t); + if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { + *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); + (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } *(int32_t*)(*ppHead)->content = foo->a; return (*ppHead)->ssize; } -const void* FooDeserializer(const TqSerializedHead* pHead, void** ppObj) { +const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { if(*ppObj == NULL) { *ppObj = realloc(*ppObj, sizeof(int32_t)); } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h new file mode 100644 index 0000000000..223d5a5768 --- /dev/null +++ b/source/libs/parser/inc/astToMsg.h @@ -0,0 +1,9 @@ +#ifndef TDENGINE_ASTTOMSG_H +#define TDENGINE_ASTTOMSG_H + +#include "parserInt.h" +#include "taosmsg.h" + +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); + +#endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index e3925d3446..4f506e631f 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -53,6 +53,15 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t */ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen); +/** + * validate the ddl ast, and convert the ast to the corresponding message format + * @param pSqlInfo + * @param output + * @param type + * @return + */ +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen); + /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. * @param pNode diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 6c95c4327b..b402621903 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -45,6 +45,8 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo); SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index); int32_t parserValidateIdToken(SToken* pToken); +int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf); + int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg); int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr); @@ -63,6 +65,8 @@ int32_t getExprFunctionId(SExprInfo *pExprInfo); STableMeta* tableMetaDup(const STableMeta* pTableMeta); +bool isDclSqlStatement(SSqlInfo* pSqlInfo); + #ifdef __cplusplus } #endif diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c new file mode 100644 index 0000000000..6bfbd5ebed --- /dev/null +++ b/source/libs/parser/src/astToMsg.c @@ -0,0 +1,23 @@ +#include "parserInt.h" + +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { + SCreateUserMsg *pMsg = (SCreateUserMsg *)calloc(1, sizeof(SCreateUserMsg)); + if (pMsg == NULL) { +// tscError("0x%" PRIx64 " failed to malloc for query msg", id); + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } + + SUserInfo *pUser = &pInfo->pMiscInfo->user; + strncpy(pMsg->user, pUser->user.z, pUser->user.n); + pMsg->type = pUser->type; + pMsg->superUser = (int8_t)pUser->type; + + if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { +// pMsg->privilege = (char)pCmd->count; + } else { + strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); + } + + return pMsg; +} \ No newline at end of file diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 9f1363a9b4..2ee95732d0 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "astGenerator.h" #include "function.h" #include "parserInt.h" @@ -23,6 +22,7 @@ #include "tglobal.h" #include "tmsgtype.h" #include "ttime.h" +#include "astToMsg.h" #define TSQL_TBNAME_L "tbname" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -3636,11 +3636,14 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) } int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msgBuf, int32_t msgBufLen) { - //1. if it is a query, get the meta info and continue. assert(pCatalog != NULL && pInfo != NULL); int32_t code = 0; -#if 0 + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf *pMsgBuf = &m; + switch (pInfo->type) { +#if 0 case TSDB_SQL_DROP_TABLE: case TSDB_SQL_DROP_USER: case TSDB_SQL_DROP_ACCT: @@ -3651,14 +3654,14 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (parserValidateIdToken(pzName) != TSDB_CODE_SUCCESS)) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } if (pInfo->type == TSDB_SQL_DROP_DB) { assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } } else if (pInfo->type == TSDB_SQL_DROP_TABLE) { @@ -3675,7 +3678,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer strncpy(pCmd->payload, pzName->z, pzName->n); } else { // drop user/account if (pzName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } strncpy(pCmd->payload, pzName->z, pzName->n); @@ -3689,12 +3692,12 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); if (ret != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } break; @@ -3729,19 +3732,19 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SCreateDbInfo* pCreateDB = &(pInfo->pMiscInfo->dbOpt); if (pCreateDB->dbname.n >= TSDB_DB_NAME_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } char buf[TSDB_DB_NAME_LEN] = {0}; SToken token = taosTokenDup(&pCreateDB->dbname, buf, tListLen(buf)); if (tscValidateName(&token) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), &token); if (ret != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } if (parseCreateDBOptions(pCmd, pCreateDB) != TSDB_CODE_SUCCESS) { @@ -3755,7 +3758,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer const char* msg = "invalid host name (ip address)"; if (taosArrayGetSize(pInfo->pMiscInfo->a) > 1) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -3779,11 +3782,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } if (pName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; @@ -3793,7 +3796,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } else if (strncmp(pAcctOpt->stat.z, "all", 3) == 0 && pAcctOpt->stat.n == 3) { } else if (strncmp(pAcctOpt->stat.z, "no", 2) == 0 && pAcctOpt->stat.n == 2) { } else { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } } @@ -3805,7 +3808,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } // additional msg has been attached already code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); @@ -3821,7 +3824,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } code = tscSetTableFullName(&pTableMetaInfo->name, pToken, pSql); @@ -3836,11 +3839,11 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer SToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } if (pToken->n > TSDB_DB_NAME_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } return tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pToken); } @@ -3853,7 +3856,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer /* validate the parameter names and options */ if (validateDNodeConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); + return buildInvalidOperationMsg(pMsgBuf, msg2); } char* pMsg = pCmd->payload; @@ -3867,7 +3870,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer strncpy(pCfg->ep, t0->z, t0->n); if (validateEp(pCfg->ep) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg3); + return buildInvalidOperationMsg(pMsgBuf, msg3); } strncpy(pCfg->config, t1->z, t1->n); @@ -3882,65 +3885,13 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer break; } - case TSDB_SQL_CREATE_USER: - case TSDB_SQL_ALTER_USER: { - const char* msg2 = "invalid user/account name"; - const char* msg3 = "name too long"; - const char* msg5 = "invalid user rights"; - const char* msg7 = "not support options"; - - pCmd->command = pInfo->type; - - SUserInfo* pUser = &pInfo->pMiscInfo->user; - SToken* pName = &pUser->user; - SToken* pPwd = &pUser->passwd; - - if (pName->n >= TSDB_USER_LEN) { - return setInvalidOperatorMsg(pMsgBuf, msg3); - } - - if (tscValidateName(pName) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg2); - } - - if (pCmd->command == TSDB_SQL_CREATE_USER) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else { - if (pUser->type == TSDB_ALTER_USER_PASSWD) { - if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - assert(pPwd->type == TSDB_DATA_TYPE_NULL); - - SToken* pPrivilege = &pUser->privilege; - - if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 1; - } else if (strncasecmp(pPrivilege->z, "read", 4) == 0 && pPrivilege->n == 4) { - pCmd->count = 2; - } else if (strncasecmp(pPrivilege->z, "write", 5) == 0 && pPrivilege->n == 5) { - pCmd->count = 3; - } else { - return setInvalidOperatorMsg(pMsgBuf, msg5); - } - } else { - return setInvalidOperatorMsg(pMsgBuf, msg7); - } - } - - break; - } - case TSDB_SQL_CFG_LOCAL: { SMiscInfo *pMiscInfo = pInfo->pMiscInfo; const char *msg = "invalid configure options or values"; // validate the parameter names and options if (validateLocalConfig(pMiscInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } int32_t numOfToken = (int32_t) taosArrayGetSize(pMiscInfo->a); @@ -3995,7 +3946,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer tscTrace("0x%"PRIx64" start to parse the %dth subclause, total:%"PRIzu, pSql->self, i, size); if (size > 1 && pSqlNode->from && pSqlNode->from->type == SQL_FROM_NODE_SUBQUERY) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } // normalizeSqlNode(pSqlNode); // normalize the column name in each function @@ -4061,21 +4012,22 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer assert(taosArrayGetSize(pInfo->pMiscInfo->a) == 1); code = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pSql), pzName); if (code != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg1); + return buildInvalidOperationMsg(pMsgBuf, msg1); } break; } case TSDB_SQL_COMPACT_VNODE:{ const char* msg = "invalid compact"; if (setCompactVnodeInfo(pSql, pInfo) != TSDB_CODE_SUCCESS) { - return setInvalidOperatorMsg(pMsgBuf, msg); + return buildInvalidOperationMsg(pMsgBuf, msg); } break; } + #endif default: - return setInvalidOperatorMsg(pMsgBuf, "not support sql expression"); + return buildInvalidOperationMsg(pMsgBuf, "not support sql expression"); } -#endif + SCatalogReq req = {0}; SMetaData data = {0}; @@ -4114,5 +4066,159 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQuer } + return code; +} + +// todo remove it +static int32_t setShowInfo(struct SSqlInfo* pInfo, void** output, int32_t* msgLen, SMsgBuf* pMsgBuf) { + const char* msg1 = "invalid name"; + const char* msg2 = "wildcard string should be less than %d characters"; + const char* msg3 = "database name too long"; + const char* msg4 = "pattern is invalid"; + const char* msg5 = "database name is empty"; + const char* msg6 = "pattern string is empty"; + + /* + * database prefix in pInfo->pMiscInfo->a[0] + * wildcard in like clause in pInfo->pMiscInfo->a[1] + */ + SShowInfo* pShowInfo = &pInfo->pMiscInfo->showOpt; + int16_t showType = pShowInfo->showType; + if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_VGROUP) { + SToken* pDbPrefixToken = &pShowInfo->prefix; + if (pDbPrefixToken->type != 0) { + if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (pDbPrefixToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg5); + } + + if (parserValidateIdToken(pDbPrefixToken) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + // int32_t ret = tNameSetDbName(&pTableMetaInfo->name, getAccountId(pRequest->pTsc), pDbPrefixToken); + // if (ret != TSDB_CODE_SUCCESS) { + // return buildInvalidOperationMsg(pMsgBuf, msg1); + // } + } + + // show table/stable like 'xxxx', set the like pattern for show tables + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type != 0) { + if (pPattern->type == TK_ID && pPattern->z[0] == TS_ESCAPE_CHAR) { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + + pPattern->n = strdequote(pPattern->z); + if (pPattern->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg6); + } + + if (pPattern->n > tsMaxWildCardsLen) { + char tmp[64] = {0}; + sprintf(tmp, msg2, tsMaxWildCardsLen); + return buildInvalidOperationMsg(pMsgBuf, tmp); + } + } + } else if (showType == TSDB_MGMT_TABLE_VNODES) { + if (pShowInfo->prefix.type == 0) { + return buildInvalidOperationMsg(pMsgBuf, "No specified dnode ep"); + } + + if (pShowInfo->prefix.type == TK_STRING) { + pShowInfo->prefix.n = strdequote(pShowInfo->prefix.z); + } + } + + SShowMsg* pShowMsg = calloc(1, sizeof(SShowMsg)); + pShowMsg->type = pShowInfo->showType; + + if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { + SToken* pPattern = &pShowInfo->pattern; + if (pPattern->type > 0) { // only show tables support wildcard query + strncpy(pShowMsg->payload, pPattern->z, pPattern->n); + pShowMsg->payloadLen = htons(pPattern->n); + } + } else { + SToken* pEpAddr = &pShowInfo->prefix; + assert(pEpAddr->n > 0 && pEpAddr->type > 0); + + strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); + pShowMsg->payloadLen = htons(pEpAddr->n); + } + + *output = pShowMsg; + *msgLen = sizeof(SShowMsg) + htons(pShowMsg->payloadLen); + return TSDB_CODE_SUCCESS; +} + +int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) { + int32_t code = 0; + + SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; + SMsgBuf *pMsgBuf = &m; + + *type = pInfo->type; + + switch (pInfo->type) { + case TSDB_SQL_CREATE_USER: + case TSDB_SQL_ALTER_USER: { + const char* msg1 = "not support options"; + const char* msg2 = "invalid user/account name"; + const char* msg3 = "name too long"; + const char* msg4 = "invalid user rights"; + + SUserInfo* pUser = &pInfo->pMiscInfo->user; + SToken* pName = &pUser->user; + SToken* pPwd = &pUser->passwd; + + if (pName->n >= TSDB_USER_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + if (parserValidateIdToken(pName) != TSDB_CODE_SUCCESS) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + if (pInfo->type == TSDB_SQL_CREATE_USER) { + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } else { + if (pUser->type == TSDB_ALTER_USER_PASSWD) { + if (parserValidatePassword(pPwd, pMsgBuf) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { + assert(pPwd->type == TSDB_DATA_TYPE_NULL); + + SToken* pPrivilege = &pUser->privilege; + if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { + // pCmd->count = 1; + } else if (strncasecmp(pPrivilege->z, "normal", 4) == 0 && pPrivilege->n == 4) { + // pCmd->count = 2; + } else { + return buildInvalidOperationMsg(pMsgBuf, msg4); + } + } else { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + } + + *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_SHOW: { + code = setShowInfo(pInfo, output, outputLen, pMsgBuf); + break; + } + default: + break; + } + return code; } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index d167f7ad3c..1d883974b8 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -31,25 +31,35 @@ bool qIsInsertSql(const char* pStr, size_t length) { } while (1); } -int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen) { - *pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); - if (*pQueryInfo == NULL) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. +int32_t qParseQuerySql(const char* pStr, size_t length, int64_t id, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) { + SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); + if (pQueryInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code. + return terrno; } SSqlInfo info = doGenerateAST(pStr); if (!info.valid) { strncpy(msg, info.msg, msgLen); - return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + return terrno; } - struct SCatalog* pCatalog = NULL; - int32_t code = catalogGetHandle(NULL, &pCatalog); - if (code) { - return code; + if (isDclSqlStatement(&info)) { + int32_t code = qParserValidateDclSqlNode(&info, id, pOutput, outputLen, type, msg, msgLen); + if (code == TSDB_CODE_SUCCESS) { + // do nothing + } + } else { + struct SCatalog* pCatalog = NULL; + int32_t code = catalogGetHandle(NULL, &pCatalog); + code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, id, msg, msgLen); + if (code == TSDB_CODE_SUCCESS) { + *pOutput = pQueryInfo; + } } - - return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen); + + return TSDB_CODE_SUCCESS; } int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) { diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 7fcb7ea304..3a61f5912b 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "parserUtil.h" + #include "taosmsg.h" #include "parser.h" @@ -22,6 +22,8 @@ #include "thash.h" #include "tbuffer.h" #include "parserInt.h" +#include "parserUtil.h" +#include "tmsgtype.h" #include "queryInfoUtil.h" #include "function.h" @@ -97,12 +99,35 @@ int32_t parserValidateIdToken(SToken* pToken) { return TSDB_CODE_SUCCESS; } +int32_t parserValidatePassword(SToken* pToken, SMsgBuf* pMsgBuf) { + const char* msg1 = "password can not be empty"; + const char* msg2 = "name or password too long"; + const char* msg3 = "password needs single quote marks enclosed"; + + if (pToken->type != TK_STRING) { + return buildInvalidOperationMsg(pMsgBuf, msg3); + } + + strdequote(pToken->z); + + pToken->n = (uint32_t)strtrim(pToken->z); // trim space before and after passwords + if (pToken->n <= 0) { + return buildInvalidOperationMsg(pMsgBuf, msg1); + } + + if (pToken->n >= TSDB_USET_PASSWORD_LEN) { + return buildInvalidOperationMsg(pMsgBuf, msg2); + } + + return TSDB_CODE_SUCCESS; +} + int32_t buildInvalidOperationMsg(SMsgBuf* pBuf, const char* msg) { strncpy(pBuf->buf, msg, pBuf->len); return TSDB_CODE_TSC_INVALID_OPERATION; } -int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { +int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr) { const char* msgFormat1 = "syntax error near \'%s\'"; const char* msgFormat2 = "syntax error near \'%s\' (%s)"; const char* msgFormat3 = "%s"; @@ -584,21 +609,6 @@ int32_t getNumOfFields(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } -int32_t getFirstInvisibleFieldPos(SQueryStmtInfo* pQueryInfo) { - if (pQueryInfo->fieldsInfo.numOfOutput <= 0 || pQueryInfo->fieldsInfo.internalField == NULL) { - return 0; - } - - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - SInternalField* pField = taosArrayGet(pQueryInfo->fieldsInfo.internalField, i); - if (!pField->visible) { - return i; - } - } - - return pQueryInfo->fieldsInfo.numOfOutput; -} - SInternalField* appendFieldInfo(SFieldInfo* pFieldInfo, TAOS_FIELD* pField) { assert(pFieldInfo != NULL); pFieldInfo->numOfOutput++; @@ -1583,6 +1593,10 @@ uint32_t convertRelationalOperator(SToken *pToken) { } } +bool isDclSqlStatement(SSqlInfo* pSqlInfo) { + return (pSqlInfo->type != TSDB_SQL_SELECT); +} + #if 0 int32_t tscCreateQueryFromQueryInfo(SQueryStmtInfo* pQueryInfo, SQueryAttr* pQueryAttr, void* addr) { memset(pQueryAttr, 0, sizeof(SQueryAttr)); diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 6a402cd620..396e8a12fe 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -29,6 +29,7 @@ #include "taos.h" #include "tdef.h" #include "tvariant.h" +#include "parserUtil.h" namespace { void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_t colId) { @@ -700,5 +701,47 @@ TEST(testCase, function_Test6) { destroyQueryInfo(pQueryInfo); qParserClearupMetaRequestInfo(&req); + destroySqlInfo(&info1); +} + +TEST(testCase, show_user_Test) { + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + char sql1[] = "show users"; + SSqlInfo info1 = doGenerateAST(sql1); + ASSERT_EQ(info1.valid, true); + + void* output = NULL; + int32_t type = 0; + int32_t len = 0; + int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + ASSERT_EQ(code, 0); + + // convert the show command to be the select query + // select name, privilege, create_time, account from information_schema.users; +} + +TEST(testCase, create_user_Test) { + char msg[128] = {0}; + SMsgBuf buf; + buf.len = 128; + buf.buf = msg; + + char sql[] = {"create user abc pass 'abc'"}; + + SSqlInfo info1 = doGenerateAST(sql); + ASSERT_EQ(info1.valid, true); + + ASSERT_EQ(isDclSqlStatement(&info1), true); + + void* output = NULL; + int32_t type = 0; + int32_t len = 0; + int32_t code = qParserValidateDclSqlNode(&info1, 1, &output, &len, &type, msg, buf.len); + ASSERT_EQ(code, 0); + destroySqlInfo(&info1); } \ No newline at end of file diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 6c65e4810d..38b399fb0b 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -99,6 +99,8 @@ int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str); */ int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql); +int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag); + /** * Convert to physical plan to string to enable to print it out in the shell. * @param pPhyNode @@ -111,7 +113,7 @@ int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str); * Destroy the query plan object. * @return */ -void* destroyQueryPlan(struct SQueryPlanNode* pQueryNode); +void destroyQueryPlan(struct SQueryPlanNode* pQueryNode); /** * Destroy the physical plan. diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c new file mode 100644 index 0000000000..ce923314bd --- /dev/null +++ b/source/libs/planner/src/logicPlan.c @@ -0,0 +1,604 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "function.h" +#include "os.h" +#include "parser.h" +#include "plannerInt.h" + +typedef struct SFillEssInfo { + int32_t fillType; // fill type + int64_t *val; // fill value +} SFillEssInfo; + +typedef struct SJoinCond { + bool tagExists; // denote if tag condition exists or not + SColumn *tagCond[2]; + SColumn *colCond[2]; +} SJoinCond; + +static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); +static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); + +int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len); +int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { + return 0; +} + +int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { + SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); + assert(taosArrayGetSize(upstream) == 1); + + *pQueryNode = taosArrayGetP(upstream, 0); + + taosArrayDestroy(upstream); + return TSDB_CODE_SUCCESS; +} + +int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { + return 0; +} + +void destroyQueryPlan(SQueryPlanNode* pQueryNode) { + if (pQueryNode == NULL) { + return; + } + + doDestroyQueryNode(pQueryNode); +} + +//====================================================================================================================== + +static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, + SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { + SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); + + pNode->info.type = type; + pNode->info.name = strdup(name); + + pNode->numOfExpr = numOfOutput; + pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); + + for(int32_t i = 0; i < numOfOutput; ++i) { + taosArrayPush(pNode->pExpr, &pExpr[i]); + } + + pNode->pChildren = taosArrayInit(4, POINTER_BYTES); + for(int32_t i = 0; i < numOfPrev; ++i) { + taosArrayPush(pNode->pChildren, &prev[i]); + } + + switch(type) { + case QNODE_TAGSCAN: + case QNODE_TABLESCAN: { + SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); + memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); + info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName); + pNode->pExtInfo = info; + break; + } + + case QNODE_TIMEWINDOW: { + SInterval* pInterval = calloc(1, sizeof(SInterval)); + pNode->pExtInfo = pInterval; + memcpy(pInterval, pExtInfo, sizeof(SInterval)); + break; + } + + case QNODE_STATEWINDOW: { + SColumn* psw = calloc(1, sizeof(SColumn)); + pNode->pExtInfo = psw; + memcpy(psw, pExtInfo, sizeof(SColumn)); + break; + } + + case QNODE_SESSIONWINDOW: { + SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow)); + pNode->pExtInfo = pSessionWindow; + memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow)); + break; + } + + case QNODE_GROUPBY: { + SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; + + SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); + pGroupbyExpr->groupbyTag = p->groupbyTag; + pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); + + pNode->pExtInfo = pGroupbyExpr; + break; + } + + case QNODE_FILL: { // todo !! + pNode->pExtInfo = pExtInfo; + break; + } + + case QNODE_LIMIT: { + pNode->pExtInfo = calloc(1, sizeof(SLimit)); + memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); + break; + } + + case QNODE_SORT: { + pNode->pExtInfo = taosArrayDup(pExtInfo); + break; + } + + default: + break; + } + + return pNode; +} + +static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, + SArray* pExprs, SArray* tableCols) { + if (pQueryInfo->info.onlyTagQuery) { + int32_t num = (int32_t) taosArrayGetSize(pExprs); + SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info); + + if (pQueryInfo->info.distinct) { + pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); + } + + return pNode; + } + + SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); + + if (pQueryInfo->info.projectionQuery) { + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL); + } else { + STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); + + // table source column projection, generate the projection expr + int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); + SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumn* pCol = taosArrayGetP(tableCols, i); + + SSourceParam param = {0}; + addIntoSourceParam(¶m, NULL, pCol); + SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); + SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); + pExpr[i] = p; + } + + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); + tfree(pExpr); + } + + return pNode; +} + +static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { + // group by column not by tag + size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); + + // check for aggregation + int32_t level = getExprFunctionLevel(pQueryInfo); + + for(int32_t i = level - 1; i >= 0; --i) { + SArray* p = pQueryInfo->exprList[i]; + size_t num = taosArrayGetSize(p); + + bool aggregateFunc = false; + for(int32_t j = 0; j < num; ++j) { + SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); + if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { + continue; + } + + aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); + if (aggregateFunc) { + break; + } + } + + if (aggregateFunc) { + if (pQueryInfo->interval.interval > 0) { + pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval); + } else if (pQueryInfo->sessionWindow.gap > 0) { + pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow); + } else if (pQueryInfo->stateWindow.col.info.colId > 0) { + pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow); + } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { + pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr); + } else { + pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL); + } + } else { + pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); + } + } + + if (pQueryInfo->havingFieldNum > 0) { +// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); +// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL); + } + + if (pQueryInfo->fillType != TSDB_FILL_NONE) { + SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); + pInfo->fillType = pQueryInfo->fillType; + pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); + memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); + + SArray* p = pQueryInfo->exprList[0]; // top expression in select clause + pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); + } + + if (pQueryInfo->order != NULL) { + SArray* pList = pQueryInfo->exprList[0]; + pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order); + } + + if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { + pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); + } + + return pNode; +} + +static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, + SArray* tableCols) { + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); + + SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; + + // handle the only tag query + SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); + if (pQueryInfo->info.onlyTagQuery) { + tfree(info.tableName); + return pNode; + } + + SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); + tfree(info.tableName); + return pNode1; +} + +static bool isAllAggExpr(SArray* pList) { + assert(pList != NULL); + + for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { + SExprInfo* p = taosArrayGetP(pList, k); + if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { + return false; + } + } + + return true; +} + +SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { + SArray* upstream = NULL; + + if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause + upstream = taosArrayInit(4, POINTER_BYTES); + + size_t size = taosArrayGetSize(pQueryInfo->pUpstream); + for(int32_t i = 0; i < size; ++i) { + SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); + SArray* p = createQueryPlanImpl(pq); + taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); + } + } + + if (pQueryInfo->numOfTables > 1) { // it is a join query + // 1. separate the select clause according to table + taosArrayDestroy(upstream); + upstream = taosArrayInit(5, POINTER_BYTES); + + for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; + uint64_t uid = pTableMetaInfo->pTableMeta->uid; + + SArray* exprList = taosArrayInit(4, POINTER_BYTES); + if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; +// dropAllExprInfo(exprList); + exit(-1); + } + + // 2. create the query execution node + char name[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(&pTableMetaInfo->name, name); + SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; + + // 3. get the required table column list + SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); + columnListCopy(tableColumnList, pQueryInfo->colList, uid); + + // 4. add the projection query node + SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); + columnListDestroy(tableColumnList); +// dropAllExprInfo(exprList); + taosArrayPush(upstream, &pNode); + } + + // 3. add the join node here + SQueryTableInfo info = {0}; + int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); + SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, + pQueryInfo->exprList[0]->pData, num, NULL); + + // 4. add the aggregation or projection execution node + pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } else { // only one table, normal query process + STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); + upstream = taosArrayInit(5, POINTER_BYTES); + taosArrayPush(upstream, &pNode); + } + + return upstream; +} + +static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { + tfree(pQueryNode->pExtInfo); + tfree(pQueryNode->pSchema); + tfree(pQueryNode->info.name); +// dropAllExprInfo(pQueryNode->pExpr); + + if (pQueryNode->pChildren != NULL) { + int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); + for(int32_t i = 0; i < size; ++i) { + SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i); + doDestroyQueryNode(p); + } + + taosArrayDestroy(pQueryNode->pChildren); + } + + tfree(pQueryNode); +} + +static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { + if (level > 0) { + sprintf(buf + totalLen, "%*c", level, ' '); + totalLen += level; + } + + int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); + int32_t len = len1 + totalLen; + + switch(pQueryNode->info.type) { + case QNODE_TABLESCAN: { + SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid, + pInfo->window.skey, pInfo->window.ekey); + assert(len1 > 0); + len += len1; + + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); + len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); + + assert(len1 > 0); + len += len1; + } + + len1 = sprintf(buf + len, "\n"); + assert(len1 > 0); + + len += len1; + break; + } + + case QNODE_PROJECT: { + len1 = sprintf(buf + len, "cols:"); + assert(len1 > 0); + len += len1; + + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ")"); + len += len1; + + // todo print filter info + len1 = sprintf(buf + len, " filters:(nil)\n"); + len += len1; + break; + } + + case QNODE_AGGREGATE: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ")\n"); + len += len1; + + break; + } + + case QNODE_TIMEWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ") "); + len += len1; + + SInterval* pInterval = pQueryNode->pExtInfo; + + // todo dynamic return the time precision + len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n", + pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, + TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); + len += len1; + + break; + } + + case QNODE_STATEWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + len1 = sprintf(buf + len, ") "); + len += len1; + + SColumn* pCol = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId); + len += len1; + break; + } + + case QNODE_SESSIONWINDOW: { + len = printExprInfo(buf, pQueryNode, len); + + len1 = sprintf(buf + len, ") "); + len += len1; + + struct SSessionWindow* ps = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); + len += len1; + break; + } + + case QNODE_GROUPBY: { + len = printExprInfo(buf, pQueryNode, len); + + SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, ") groupby_col: "); + len += len1; + + for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { + SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); + len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); + len += len1; + } + + len += sprintf(buf + len, "\n"); + break; + } + + case QNODE_FILL: { + SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "%d", pEssInfo->fillType); + len += len1; + + if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { + len1 = sprintf(buf + len, ", val:"); + len += len1; + + // todo get the correct fill data type + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]); + len += len1; + + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + + case QNODE_LIMIT: { + SLimit* pVal = pQueryNode->pExtInfo; + len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset); + len += len1; + break; + } + + case QNODE_DISTINCT: + case QNODE_TAGSCAN: { + len1 = sprintf(buf + len, "cols: "); + len += len1; + + len = printExprInfo(buf, pQueryNode, len); + + len1 = sprintf(buf + len, ")\n"); + len += len1; + + break; + } + + case QNODE_SORT: { + len1 = sprintf(buf + len, "cols:"); + len += len1; + + SArray* pSort = pQueryNode->pExtInfo; + for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) { + SOrder* p = taosArrayGet(pSort, i); + len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC"); + + len += len1; + } + + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + + case QNODE_JOIN: { + // print join condition + len1 = sprintf(buf + len, ")\n"); + len += len1; + break; + } + } + + return len; +} + +int32_t printExprInfo(char* buf, const SQueryPlanNode* pQueryNode, int32_t len) { + int32_t len1 = 0; + + for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); + + SSqlExpr* pExpr = &pExprInfo->base; + len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); + assert(len1 > 0); + + len += len1; + if (i < pQueryNode->numOfExpr - 1) { + len1 = sprintf(buf + len, ", "); + len += len1; + } + } + + return len; +} + +int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { + int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); + + for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) { + SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i); + int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); + len = len1; + } + + return len; +} + +int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { + assert(pQueryNode); + + *str = calloc(1, 4096); + + int32_t len = sprintf(*str, "===== logic plan =====\n"); + queryPlanToStringImpl(*str, pQueryNode, 0, len); + + return TSDB_CODE_SUCCESS; +} + +SQueryPlanNode* queryPlanFromString() { + return NULL; +} diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index e7acb12bc0..277703f5c0 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -16,6 +16,9 @@ #include "plannerInt.h" #include "parser.h" +#define STORE_CURRENT_SUBPLAN(cxt) SSubplan* _ = cxt->pCurrentSubplan +#define RECOVERY_CURRENT_SUBPLAN(cxt) cxt->pCurrentSubplan = _ + static const char* gOpName[] = { "Unknown", #define INCLUDE_AS_NAME @@ -43,8 +46,56 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si toDataBlockSchema(pPlanNode, &(node->targetSchema)); } +static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { + SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size); + node->uid = pTable->pMeta->pTableMeta->uid; + node->tableType = pTable->pMeta->pTableMeta->tableType; + return (SPhyNode*)node; +} + +static SPhyNode* createPseudoScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { + return initScanNode(pPlanNode, pTable, op, sizeof(SScanPhyNode)); +} + static SPhyNode* createTagScanNode(SQueryPlanNode* pPlanNode) { - return initPhyNode(pPlanNode, OP_TagScan, sizeof(STagScanPhyNode)); + SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; + return createPseudoScanNode(pPlanNode, pTable, OP_TagScan); +} + +static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + // todo + return MASTER_SCAN; +} + +static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { + STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode)); + node->scanFlag = getScanFlag(pPlanNode, pTable); + node->window = pTable->window; + // todo tag cond + return (SPhyNode*)node; +} + +static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); +} + +static bool isSystemTable(SQueryTableInfo* pTable) { + // todo + return false; +} + +static bool needSeqScan(SQueryPlanNode* pPlanNode) { + // todo + return false; +} + +static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { + if (isSystemTable(pTable)) { + return createPseudoScanNode(pPlanNode, pTable, OP_SystemTableScan); + } else if (needSeqScan(pPlanNode)) { + return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan); + } + return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan); } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { @@ -58,53 +109,61 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { if (NULL == pCxt->pCurrentSubplan->pChildern) { pCxt->pCurrentSubplan->pChildern = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); } - taosArrayPush(pCxt->pCurrentSubplan->pChildern, subplan); + taosArrayPush(pCxt->pCurrentSubplan->pChildern, &subplan); subplan->pParents = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); - taosArrayPush(subplan->pParents, pCxt->pCurrentSubplan); + taosArrayPush(subplan->pParents, &pCxt->pCurrentSubplan); } + SArray* currentLevel; + if (subplan->level >= taosArrayGetSize(pCxt->pDag->pSubplans)) { + currentLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + taosArrayPush(pCxt->pDag->pSubplans, ¤tLevel); + } else { + currentLevel = taosArrayGetP(pCxt->pDag->pSubplans, subplan->level); + } + taosArrayPush(currentLevel, &subplan); pCxt->pCurrentSubplan = subplan; return subplan; } -static uint8_t getScanFlag(SQueryPlanNode* pPlanNode) { - // todo - return MASTER_SCAN; -} - -static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { - STableScanPhyNode* node = (STableScanPhyNode*)initPhyNode(pPlanNode, OP_TableScan, sizeof(STableScanPhyNode)); - node->scan.uid = pTable->pMeta->pTableMeta->uid; - node->scan.tableType = pTable->pMeta->pTableMeta->tableType; - node->scanFlag = getScanFlag(pPlanNode); - node->window = pTable->window; - // todo tag cond -} - static void vgroupToEpSet(const SVgroupMsg* vg, SEpSet* epSet) { - // todo + epSet->inUse = 0; // todo + epSet->numOfEps = vg->numOfEps; + for (int8_t i = 0; i < vg->numOfEps; ++i) { + epSet->port[i] = vg->epAddr[i].port; + strcpy(epSet->fqdn[i], vg->epAddr[i].fqdn); + } + return; } -static void splitSubplanBySTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { +static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { SVgroupsInfo* vgroupList = pTable->pMeta->vgroupList; for (int32_t i = 0; i < pTable->pMeta->vgroupList->numOfVgroups; ++i) { + STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); vgroupToEpSet(&(pTable->pMeta->vgroupList->vgroups[i]), &subplan->execEpSet); - subplan->pNode = createTableScanNode(pCxt, pPlanNode, pTable); - // todo reset pCxt->pCurrentSubplan + subplan->pNode = createMultiTableScanNode(pPlanNode, pTable); + RECOVERY_CURRENT_SUBPLAN(pCxt); } + return pCxt->nextId.templateId++; } -static SPhyNode* createExchangeNode() { - +static SPhyNode* createExchangeNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, uint64_t srcTemplateId) { + SExchangePhyNode* node = (SExchangePhyNode*)initPhyNode(pPlanNode, OP_Exchange, sizeof(SExchangePhyNode)); + node->srcTemplateId = srcTemplateId; + return (SPhyNode*)node; } -static SPhyNode* createScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { +static bool needMultiNodeScan(SQueryTableInfo* pTable) { + // todo system table, for instance, user_tables + return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType); +} + +static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; - if (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType) { - splitSubplanBySTable(pCxt, pPlanNode, pTable); - return createExchangeNode(pCxt, pTable); + if (needMultiNodeScan(pTable)) { + return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } - return createTableScanNode(pCxt, pPlanNode, pTable); + return createSingleTableScanNode(pPlanNode, pTable); } static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { @@ -114,7 +173,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { node = createTagScanNode(pPlanNode); break; case QNODE_TABLESCAN: - node = createScanNode(pCxt, pPlanNode); + node = createTableScanNode(pCxt, pPlanNode); break; default: assert(false); @@ -133,6 +192,7 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MERGE); + ++(pCxt->nextId.templateId); subplan->pNode = createPhyNode(pCxt, pRoot); SArray* l0 = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); taosArrayPush(l0, &subplan); @@ -144,7 +204,8 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD SPlanContext context = { .pCatalog = pCatalog, .pDag = calloc(1, sizeof(SQueryDag)), - .pCurrentSubplan = NULL + .pCurrentSubplan = NULL, + .nextId = {0} // todo queryid }; if (NULL == context.pDag) { return TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 19aac36e78..744a849e2d 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -13,606 +13,31 @@ * along with this program. If not, see . */ -#include "function.h" -#include "os.h" #include "parser.h" #include "plannerInt.h" -typedef struct SFillEssInfo { - int32_t fillType; // fill type - int64_t *val; // fill value -} SFillEssInfo; - -typedef struct SJoinCond { - bool tagExists; // denote if tag condition exists or not - SColumn *tagCond[2]; - SColumn *colCond[2]; -} SJoinCond; - -static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo); -static void doDestroyQueryNode(SQueryPlanNode* pQueryNode); - -int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len); -int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { - return 0; +void qDestroyQueryDag(struct SQueryDag* pDag) { + // todo } -int32_t createQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode** pQueryNode) { - SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo); - assert(taosArrayGetSize(upstream) == 1); - - *pQueryNode = taosArrayGetP(upstream, 0); - - taosArrayDestroy(upstream); +int32_t qCreateQueryDag(const struct SQueryStmtInfo* pQueryInfo, struct SEpSet* pQnode, struct SQueryDag** pDag) { + SQueryPlanNode* logicPlan; + int32_t code = createQueryPlan(pQueryInfo, &logicPlan); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + return code; + } + code = optimizeQueryPlan(logicPlan); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + return code; + } + code = createDag(logicPlan, NULL, pDag); + if (TSDB_CODE_SUCCESS != code) { + destroyQueryPlan(logicPlan); + qDestroyQueryDag(*pDag); + return code; + } + destroyQueryPlan(logicPlan); return TSDB_CODE_SUCCESS; } - -int32_t queryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) { - return 0; -} - -int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDag** pDag) { - - return 0; -} - -int32_t phyPlanToString(struct SPhyNode *pPhyNode, char** str) { - return 0; -} - -void* destroyQueryPlan(SQueryPlanNode* pQueryNode) { - if (pQueryNode == NULL) { - return NULL; - } - - doDestroyQueryNode(pQueryNode); - return NULL; -} - -void* destroyQueryPhyPlan(struct SPhyNode* pQueryPhyNode) { - return NULL; -} - -//====================================================================================================================== - -static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev, - SExprInfo** pExpr, int32_t numOfOutput, void* pExtInfo) { - SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode)); - - pNode->info.type = type; - pNode->info.name = strdup(name); - - pNode->numOfExpr = numOfOutput; - pNode->pExpr = taosArrayInit(numOfOutput, POINTER_BYTES); - - for(int32_t i = 0; i < numOfOutput; ++i) { - taosArrayPush(pNode->pExpr, &pExpr[i]); - } - - pNode->pChildren = taosArrayInit(4, POINTER_BYTES); - for(int32_t i = 0; i < numOfPrev; ++i) { - taosArrayPush(pNode->pChildren, &prev[i]); - } - - switch(type) { - case QNODE_TAGSCAN: - case QNODE_TABLESCAN: { - SQueryTableInfo* info = calloc(1, sizeof(SQueryTableInfo)); - memcpy(info, pExtInfo, sizeof(SQueryTableInfo)); - info->tableName = strdup(((SQueryTableInfo*) pExtInfo)->tableName); - pNode->pExtInfo = info; - break; - } - - case QNODE_TIMEWINDOW: { - SInterval* pInterval = calloc(1, sizeof(SInterval)); - pNode->pExtInfo = pInterval; - memcpy(pInterval, pExtInfo, sizeof(SInterval)); - break; - } - - case QNODE_STATEWINDOW: { - SColumn* psw = calloc(1, sizeof(SColumn)); - pNode->pExtInfo = psw; - memcpy(psw, pExtInfo, sizeof(SColumn)); - break; - } - - case QNODE_SESSIONWINDOW: { - SSessionWindow *pSessionWindow = calloc(1, sizeof(SSessionWindow)); - pNode->pExtInfo = pSessionWindow; - memcpy(pSessionWindow, pExtInfo, sizeof(struct SSessionWindow)); - break; - } - - case QNODE_GROUPBY: { - SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo; - - SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr)); - pGroupbyExpr->groupbyTag = p->groupbyTag; - pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo); - - pNode->pExtInfo = pGroupbyExpr; - break; - } - - case QNODE_FILL: { // todo !! - pNode->pExtInfo = pExtInfo; - break; - } - - case QNODE_LIMIT: { - pNode->pExtInfo = calloc(1, sizeof(SLimit)); - memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit)); - break; - } - - case QNODE_SORT: { - pNode->pExtInfo = taosArrayDup(pExtInfo); - break; - } - - default: - break; - } - - return pNode; -} - -static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info, - SArray* pExprs, SArray* tableCols) { - if (pQueryInfo->info.onlyTagQuery) { - int32_t num = (int32_t) taosArrayGetSize(pExprs); - SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info); - - if (pQueryInfo->info.distinct) { - pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, NULL); - } - - return pNode; - } - - SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info); - - if (pQueryInfo->info.projectionQuery) { - int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs); - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, NULL); - } else { - STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0); - - // table source column projection, generate the projection expr - int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols); - SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumn* pCol = taosArrayGetP(tableCols, i); - - SSourceParam param = {0}; - addIntoSourceParam(¶m, NULL, pCol); - SSchema s = createSchema(pCol->info.type, pCol->info.bytes, pCol->info.colId, pCol->name); - SExprInfo* p = createExprInfo(pTableMetaInfo1, "project", ¶m, &s, 0); - pExpr[i] = p; - } - - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, NULL); - tfree(pExpr); - } - - return pNode; -} - -static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info) { - // group by column not by tag - size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo); - - // check for aggregation - int32_t level = getExprFunctionLevel(pQueryInfo); - - for(int32_t i = level - 1; i >= 0; --i) { - SArray* p = pQueryInfo->exprList[i]; - size_t num = taosArrayGetSize(p); - - bool aggregateFunc = false; - for(int32_t j = 0; j < num; ++j) { - SExprInfo* pExpr = (SExprInfo*)taosArrayGetP(p, 0); - if (pExpr->pExpr->nodeType != TEXPR_FUNCTION_NODE) { - continue; - } - - aggregateFunc = qIsAggregateFunction(pExpr->pExpr->_function.functionName); - if (aggregateFunc) { - break; - } - } - - if (aggregateFunc) { - if (pQueryInfo->interval.interval > 0) { - pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->interval); - } else if (pQueryInfo->sessionWindow.gap > 0) { - pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->sessionWindow); - } else if (pQueryInfo->stateWindow.col.info.colId > 0) { - pNode = createQueryNode(QNODE_STATEWINDOW, "StateWindowAgg", &pNode, 1, p->pData, num, &pQueryInfo->stateWindow); - } else if (numOfGroupCols != 0 && !pQueryInfo->groupbyExpr.groupbyTag) { - pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, p->pData, num, &pQueryInfo->groupbyExpr); - } else { - pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, p->pData, num, NULL); - } - } else { - pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, p->pData, num, NULL); - } - } - - if (pQueryInfo->havingFieldNum > 0) { -// int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1); -// pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, NULL); - } - - if (pQueryInfo->fillType != TSDB_FILL_NONE) { - SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo)); - pInfo->fillType = pQueryInfo->fillType; - pInfo->val = calloc(pNode->numOfExpr, sizeof(int64_t)); - memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfExpr); - - SArray* p = pQueryInfo->exprList[0]; // top expression in select clause - pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, p, taosArrayGetSize(p), pInfo); - } - - if (pQueryInfo->order != NULL) { - SArray* pList = pQueryInfo->exprList[0]; - pNode = createQueryNode(QNODE_SORT, "Sort", &pNode, 1, pList->pData, taosArrayGetSize(pList), pQueryInfo->order); - } - - if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) { - pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, &pQueryInfo->limit); - } - - return pNode; -} - -static SQueryPlanNode* doCreateQueryPlanForSingleTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs, - SArray* tableCols) { - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tstrncpy(name, pTableMetaInfo->name.tname, TSDB_TABLE_FNAME_LEN); - - SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; - - // handle the only tag query - SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols); - if (pQueryInfo->info.onlyTagQuery) { - tfree(info.tableName); - return pNode; - } - - SQueryPlanNode* pNode1 = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); - tfree(info.tableName); - return pNode1; -} - -static bool isAllAggExpr(SArray* pList) { - assert(pList != NULL); - - for (int32_t k = 0; k < taosArrayGetSize(pList); ++k) { - SExprInfo* p = taosArrayGetP(pList, k); - if (p->pExpr->nodeType != TEXPR_FUNCTION_NODE || !qIsAggregateFunction(p->pExpr->_function.functionName)) { - return false; - } - } - - return true; -} - -SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) { - SArray* upstream = NULL; - - if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause - upstream = taosArrayInit(4, POINTER_BYTES); - - size_t size = taosArrayGetSize(pQueryInfo->pUpstream); - for(int32_t i = 0; i < size; ++i) { - SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i); - SArray* p = createQueryPlanImpl(pq); - taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p)); - } - } - - if (pQueryInfo->numOfTables > 1) { // it is a join query - // 1. separate the select clause according to table - taosArrayDestroy(upstream); - upstream = taosArrayInit(5, POINTER_BYTES); - - for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i]; - uint64_t uid = pTableMetaInfo->pTableMeta->uid; - - SArray* exprList = taosArrayInit(4, POINTER_BYTES); - if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; -// dropAllExprInfo(exprList); - exit(-1); - } - - // 2. create the query execution node - char name[TSDB_TABLE_FNAME_LEN] = {0}; - tNameExtractFullName(&pTableMetaInfo->name, name); - SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,}; - - // 3. get the required table column list - SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn)); - columnListCopy(tableColumnList, pQueryInfo->colList, uid); - - // 4. add the projection query node - SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList); - columnListDestroy(tableColumnList); -// dropAllExprInfo(exprList); - taosArrayPush(upstream, &pNode); - } - - // 3. add the join node here - SQueryTableInfo info = {0}; - int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList[0]); - SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables, - pQueryInfo->exprList[0]->pData, num, NULL); - - // 4. add the aggregation or projection execution node - pNode = doCreateQueryPlanForSingleTableImpl(pQueryInfo, pNode, &info); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } else { // only one table, normal query process - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; - SQueryPlanNode* pNode = doCreateQueryPlanForSingleTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList[0], pQueryInfo->colList); - upstream = taosArrayInit(5, POINTER_BYTES); - taosArrayPush(upstream, &pNode); - } - - return upstream; -} - -static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - tfree(pQueryNode->pExtInfo); - tfree(pQueryNode->pSchema); - tfree(pQueryNode->info.name); -// dropAllExprInfo(pQueryNode->pExpr); - - if (pQueryNode->pChildren != NULL) { - int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); - for(int32_t i = 0; i < size; ++i) { - SQueryPlanNode* p = taosArrayGetP(pQueryNode->pChildren, i); - doDestroyQueryNode(p); - } - - taosArrayDestroy(pQueryNode->pChildren); - } - - tfree(pQueryNode); -} - -static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { - if (level > 0) { - sprintf(buf + totalLen, "%*c", level, ' '); - totalLen += level; - } - - int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name); - int32_t len = len1 + totalLen; - - switch(pQueryNode->info.type) { - case QNODE_TABLESCAN: { - SQueryTableInfo* pInfo = (SQueryTableInfo*)pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64, pInfo->tableName, pInfo->uid, - pInfo->window.skey, pInfo->window.ekey); - assert(len1 > 0); - len += len1; - - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SColumn* pCol = taosArrayGetP(pQueryNode->pExpr, i); - len1 = sprintf(buf + len, " [%s #%d] ", pCol->name, pCol->info.colId); - - assert(len1 > 0); - len += len1; - } - - len1 = sprintf(buf + len, "\n"); - assert(len1 > 0); - - len += len1; - break; - } - - case QNODE_PROJECT: { - len1 = sprintf(buf + len, "cols:"); - assert(len1 > 0); - len += len1; - - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ")"); - len += len1; - - // todo print filter info - len1 = sprintf(buf + len, " filters:(nil)\n"); - len += len1; - break; - } - - case QNODE_AGGREGATE: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ")\n"); - len += len1; - - break; - } - - case QNODE_TIMEWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ") "); - len += len1; - - SInterval* pInterval = pQueryNode->pExtInfo; - - // todo dynamic return the time precision - len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "(%s)\n", - pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, - TSDB_TIME_PRECISION_MILLI_STR, pInterval->offset, TSDB_TIME_PRECISION_MILLI_STR); - len += len1; - - break; - } - - case QNODE_STATEWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - len1 = sprintf(buf + len, ") "); - len += len1; - - SColumn* pCol = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "col:%s #%d\n", pCol->name, pCol->info.colId); - len += len1; - break; - } - - case QNODE_SESSIONWINDOW: { - len = printExprInfo(buf, pQueryNode, len); - - len1 = sprintf(buf + len, ") "); - len += len1; - - struct SSessionWindow* ps = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "col:[%s #%d], gap:%" PRId64 " (ms) \n", ps->col.name, ps->col.info.colId, ps->gap); - len += len1; - break; - } - - case QNODE_GROUPBY: { - len = printExprInfo(buf, pQueryNode, len); - - SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, ") groupby_col: "); - len += len1; - - for (int32_t i = 0; i < taosArrayGetSize(pGroupbyExpr->columnInfo); ++i) { - SColumn* pCol = taosArrayGet(pGroupbyExpr->columnInfo, i); - len1 = sprintf(buf + len, "[%s #%d] ", pCol->name, pCol->info.colId); - len += len1; - } - - len += sprintf(buf + len, "\n"); - break; - } - - case QNODE_FILL: { - SFillEssInfo* pEssInfo = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "%d", pEssInfo->fillType); - len += len1; - - if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) { - len1 = sprintf(buf + len, ", val:"); - len += len1; - - // todo get the correct fill data type - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - len1 = sprintf(buf + len, "%" PRId64, pEssInfo->val[i]); - len += len1; - - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_LIMIT: { - SLimit* pVal = pQueryNode->pExtInfo; - len1 = sprintf(buf + len, "limit: %" PRId64 ", offset: %" PRId64 ")\n", pVal->limit, pVal->offset); - len += len1; - break; - } - - case QNODE_DISTINCT: - case QNODE_TAGSCAN: { - len1 = sprintf(buf + len, "cols: "); - len += len1; - - len = printExprInfo(buf, pQueryNode, len); - - len1 = sprintf(buf + len, ")\n"); - len += len1; - - break; - } - - case QNODE_SORT: { - len1 = sprintf(buf + len, "cols:"); - len += len1; - - SArray* pSort = pQueryNode->pExtInfo; - for (int32_t i = 0; i < taosArrayGetSize(pSort); ++i) { - SOrder* p = taosArrayGet(pSort, i); - len1 = sprintf(buf + len, " [%s #%d %s]", p->col.name, p->col.info.colId, p->order == TSDB_ORDER_ASC? "ASC":"DESC"); - - len += len1; - } - - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - - case QNODE_JOIN: { - // print join condition - len1 = sprintf(buf + len, ")\n"); - len += len1; - break; - } - } - - return len; -} - -int32_t printExprInfo(const char* buf, const SQueryPlanNode* pQueryNode, int32_t len) { - int32_t len1 = 0; - - for (int32_t i = 0; i < pQueryNode->numOfExpr; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i); - - SSqlExpr* pExpr = &pExprInfo->base; - len1 = sprintf(buf + len, "%s [%s #%d]", pExpr->token, pExpr->resSchema.name, pExpr->resSchema.colId); - assert(len1 > 0); - - len += len1; - if (i < pQueryNode->numOfExpr - 1) { - len1 = sprintf(buf + len, ", "); - len += len1; - } - } - - return len; -} - -int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) { - int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen); - - for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pChildren); ++i) { - SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pChildren, i); - int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len); - len = len1; - } - - return len; -} - -int32_t queryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) { - assert(pQueryNode); - - *str = calloc(1, 4096); - - int32_t len = sprintf(*str, "===== logic plan =====\n"); - queryPlanToStringImpl(*str, pQueryNode, 0, len); - - return TSDB_CODE_SUCCESS; -} - -SQueryPlanNode* queryPlanFromString() { - return NULL; -} diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index e2715d51f1..e86a2b6221 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -14,27 +14,21 @@ */ #define _DEFAULT_SOURCE +#include "cJSON.h" #include "os.h" #include "taoserror.h" -#include "tref.h" #include "tfile.h" -#include "cJSON.h" +#include "tref.h" #include "walInt.h" #include #include -int64_t inline walGetFirstVer(SWal *pWal) { - return pWal->vers.firstVer; -} +int64_t inline walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } -int64_t inline walGetSnaphostVer(SWal *pWal) { - return pWal->vers.snapshotVer; -} +int64_t inline walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; } -int64_t inline walGetLastVer(SWal *pWal) { - return pWal->vers.lastVer; -} +int64_t inline walGetLastVer(SWal* pWal) { return pWal->vers.lastVer; } static inline int walBuildMetaName(SWal* pWal, int metaVer, char* buf) { return sprintf(buf, "%s/meta-ver%d", pWal->path, metaVer); @@ -44,31 +38,30 @@ int walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info const char* logPattern = "^[0-9]+.log$"; const char* idxPattern = "^[0-9]+.idx$"; - regex_t logRegPattern; - regex_t idxRegPattern; - SArray* pLogArray = taosArrayInit(8, sizeof(int64_t)); + regex_t logRegPattern; + regex_t idxRegPattern; + SArray* pLogArray = taosArrayInit(8, sizeof(int64_t)); regcomp(&logRegPattern, logPattern, REG_EXTENDED); regcomp(&idxRegPattern, idxPattern, REG_EXTENDED); - - DIR *dir = opendir(pWal->path); - if(dir == NULL) { + + DIR* dir = opendir(pWal->path); + if (dir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } struct dirent* ent; - while((ent = readdir(dir)) != NULL) { - char *name = basename(ent->d_name); - int code = regexec(&logRegPattern, name, 0, NULL, 0); - if(code == 0) { + while ((ent = readdir(dir)) != NULL) { + char* name = basename(ent->d_name); + int code = regexec(&logRegPattern, name, 0, NULL, 0); + if (code == 0) { int64_t firstVer; sscanf(name, "%" PRId64 ".log", &firstVer); taosArrayPush(pLogArray, &firstVer); } } - // load meta // if not match, or meta missing // rebuild meta @@ -85,15 +78,15 @@ int walRollFileInfo(SWal* pWal) { int64_t ts = taosGetTimestampSec(); SArray* pArray = pWal->fileInfoSet; - if(taosArrayGetSize(pArray) != 0) { - WalFileInfo *pInfo = taosArrayGetLast(pArray); + if (taosArrayGetSize(pArray) != 0) { + WalFileInfo* pInfo = taosArrayGetLast(pArray); pInfo->lastVer = pWal->vers.lastVer; pInfo->closeTs = ts; } - //TODO: change to emplace back - WalFileInfo *pNewInfo = malloc(sizeof(WalFileInfo)); - if(pNewInfo == NULL) { + // TODO: change to emplace back + WalFileInfo* pNewInfo = malloc(sizeof(WalFileInfo)); + if (pNewInfo == NULL) { return -1; } pNewInfo->firstVer = pWal->vers.lastVer + 1; @@ -109,13 +102,13 @@ int walRollFileInfo(SWal* pWal) { char* walMetaSerialize(SWal* pWal) { char buf[30]; ASSERT(pWal->fileInfoSet); - int sz = pWal->fileInfoSet->size; + int sz = pWal->fileInfoSet->size; cJSON* pRoot = cJSON_CreateObject(); cJSON* pMeta = cJSON_CreateObject(); cJSON* pFiles = cJSON_CreateArray(); cJSON* pField; - if(pRoot == NULL || pMeta == NULL || pFiles == NULL) { - //TODO + if (pRoot == NULL || pMeta == NULL || pFiles == NULL) { + // TODO return NULL; } cJSON_AddItemToObject(pRoot, "meta", pMeta); @@ -130,15 +123,15 @@ char* walMetaSerialize(SWal* pWal) { cJSON_AddItemToObject(pRoot, "files", pFiles); WalFileInfo* pData = pWal->fileInfoSet->pData; - for(int i = 0; i < sz; i++) { + for (int i = 0; i < sz; i++) { WalFileInfo* pInfo = &pData[i]; cJSON_AddItemToArray(pFiles, pField = cJSON_CreateObject()); - if(pField == NULL) { + if (pField == NULL) { cJSON_Delete(pRoot); return NULL; } - //cjson only support int32_t or double - //string are used to prohibit the loss of precision + // cjson only support int32_t or double + // string are used to prohibit the loss of precision sprintf(buf, "%" PRId64, pInfo->firstVer); cJSON_AddStringToObject(pField, "firstVer", buf); sprintf(buf, "%" PRId64, pInfo->lastVer); @@ -171,12 +164,12 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { pFiles = cJSON_GetObjectItem(pRoot, "files"); int sz = cJSON_GetArraySize(pFiles); - //deserialize + // deserialize SArray* pArray = pWal->fileInfoSet; taosArrayEnsureCap(pArray, sz); - WalFileInfo *pData = pArray->pData; - for(int i = 0; i < sz; i++) { - cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); + WalFileInfo* pData = pArray->pData; + for (int i = 0; i < sz; i++) { + cJSON* pInfoJson = cJSON_GetArrayItem(pFiles, i); WalFileInfo* pInfo = &pData[i]; pField = cJSON_GetObjectItem(pInfoJson, "firstVer"); pInfo->firstVer = atoll(cJSON_GetStringValue(pField)); @@ -196,24 +189,24 @@ int walMetaDeserialize(SWal* pWal, const char* bytes) { } static int walFindCurMetaVer(SWal* pWal) { - const char * pattern = "^meta-ver[0-9]+$"; - regex_t walMetaRegexPattern; + const char* pattern = "^meta-ver[0-9]+$"; + regex_t walMetaRegexPattern; regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED); - DIR *dir = opendir(pWal->path); - if(dir == NULL) { + DIR* dir = opendir(pWal->path); + if (dir == NULL) { wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return -1; } struct dirent* ent; - //find existing meta-ver[x].json + // find existing meta-ver[x].json int metaVer = -1; - while((ent = readdir(dir)) != NULL) { - char *name = basename(ent->d_name); - int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); - if(code == 0) { + while ((ent = readdir(dir)) != NULL) { + char* name = basename(ent->d_name); + int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0); + if (code == 0) { sscanf(name, "meta-ver%d", &metaVer); break; } @@ -224,23 +217,23 @@ static int walFindCurMetaVer(SWal* pWal) { } int walSaveMeta(SWal* pWal) { - int metaVer = walFindCurMetaVer(pWal); + int metaVer = walFindCurMetaVer(pWal); char fnameStr[WAL_FILE_LEN]; - walBuildMetaName(pWal, metaVer+1, fnameStr); + walBuildMetaName(pWal, metaVer + 1, fnameStr); int metaTfd = tfOpenCreateWrite(fnameStr); - if(metaTfd < 0) { + if (metaTfd < 0) { return -1; } char* serialized = walMetaSerialize(pWal); - int len = strlen(serialized); - if(len != tfWrite(metaTfd, serialized, len)) { - //TODO:clean file + int len = strlen(serialized); + if (len != tfWrite(metaTfd, serialized, len)) { + // TODO:clean file return -1; } - + tfClose(metaTfd); - //delete old file - if(metaVer > -1) { + // delete old file + if (metaVer > -1) { walBuildMetaName(pWal, metaVer, fnameStr); remove(fnameStr); } @@ -250,30 +243,30 @@ int walSaveMeta(SWal* pWal) { int walLoadMeta(SWal* pWal) { ASSERT(pWal->fileInfoSet->size == 0); - //find existing meta file + // find existing meta file int metaVer = walFindCurMetaVer(pWal); - if(metaVer == -1) { + if (metaVer == -1) { return 0; } char fnameStr[WAL_FILE_LEN]; walBuildMetaName(pWal, metaVer, fnameStr); - //read metafile + // read metafile struct stat statbuf; stat(fnameStr, &statbuf); - int size = statbuf.st_size; + int size = statbuf.st_size; char* buf = malloc(size + 5); - if(buf == NULL) { + if (buf == NULL) { return -1; } - memset(buf, 0, size+5); + memset(buf, 0, size + 5); int tfd = tfOpenRead(fnameStr); - if(tfRead(tfd, buf, size) != size) { + if (tfRead(tfd, buf, size) != size) { free(buf); return -1; } - //load into fileInfoSet + // load into fileInfoSet int code = walMetaDeserialize(pWal, buf); - if(code != 0) { + if (code != 0) { free(buf); return -1; } diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index 06df3eb3be..5b6a8c6b29 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -14,11 +14,11 @@ */ #define _DEFAULT_SOURCE +#include "compare.h" #include "os.h" #include "taoserror.h" -#include "tref.h" #include "tfile.h" -#include "compare.h" +#include "tref.h" #include "walInt.h" typedef struct { @@ -34,16 +34,14 @@ static int32_t walCreateThread(); static void walStopThread(); static void walFreeObj(void *pWal); -int64_t walGetSeq() { - return (int64_t)atomic_load_32(&tsWal.seq); -} +int64_t walGetSeq() { return (int64_t)atomic_load_32(&tsWal.seq); } int32_t walInit() { int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); - if(old == 1) return 0; + if (old == 1) return 0; int code = tfInit(); - if(code != 0) { + if (code != 0) { wError("failed to init tfile since %s", tstrerror(code)); atomic_store_8(&tsWal.inited, 0); return code; @@ -63,7 +61,7 @@ int32_t walInit() { void walCleanUp() { int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 1, 0); - if(old == 0) { + if (old == 0) { return; } walStopThread(); @@ -78,52 +76,52 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - //set config + // set config memcpy(&pWal->cfg, pCfg, sizeof(SWalCfg)); pWal->fsyncSeq = pCfg->fsyncPeriod / 1000; - if(pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; + if (pWal->fsyncSeq <= 0) pWal->fsyncSeq = 1; tstrncpy(pWal->path, path, sizeof(pWal->path)); - if(taosMkDir(pWal->path) != 0) { + if (taosMkDir(pWal->path) != 0) { wError("vgId:%d, path:%s, failed to create directory since %s", pWal->cfg.vgId, pWal->path, strerror(errno)); return NULL; } - //open meta + // open meta walResetVer(&pWal->vers); pWal->writeLogTfd = -1; pWal->writeIdxTfd = -1; pWal->writeCur = -1; pWal->fileInfoSet = taosArrayInit(8, sizeof(WalFileInfo)); - if(pWal->fileInfoSet == NULL) { + if (pWal->fileInfoSet == NULL) { wError("vgId:%d, path:%s, failed to init taosArray %s", pWal->cfg.vgId, pWal->path, strerror(errno)); free(pWal); return NULL; } - //init status + // init status pWal->totSize = 0; pWal->lastRollSeq = -1; - //init write buffer + // init write buffer memset(&pWal->writeHead, 0, sizeof(SWalHead)); pWal->writeHead.head.headVer = WAL_HEAD_VER; - if(pthread_mutex_init(&pWal->mutex, NULL) < 0) { + if (pthread_mutex_init(&pWal->mutex, NULL) < 0) { taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; } pWal->refId = taosAddRef(tsWal.refSetId, pWal); - if(pWal->refId < 0) { + if (pWal->refId < 0) { pthread_mutex_destroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); free(pWal); return NULL; } - if(walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { + if (walLoadMeta(pWal) < 0 && walCheckAndRepairMeta(pWal) < 0) { taosRemoveRef(tsWal.refSetId, pWal->refId); pthread_mutex_destroy(&pWal->mutex); taosArrayDestroy(pWal->fileInfoSet); @@ -131,11 +129,11 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { return NULL; } - if(walCheckAndRepairIdx(pWal) < 0) { - + if (walCheckAndRepairIdx(pWal) < 0) { } - wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, pWal->cfg.fsyncPeriod); + wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->cfg.vgId, pWal, pWal->cfg.level, + pWal->cfg.fsyncPeriod); return pWal; } @@ -203,10 +201,12 @@ static void walFsyncAll() { SWal *pWal = taosIterateRef(tsWal.refSetId, 0); while (pWal) { if (walNeedFsync(pWal)) { - wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->cfg.vgId, pWal->cfg.level, pWal->fsyncSeq, + atomic_load_32(&tsWal.seq)); int32_t code = tfFsync(pWal->writeLogTfd); if (code != 0) { - wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(code)); + wError("vgId:%d, file:%" PRId64 ".log, failed to fsync since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(code)); } } pWal = taosIterateRef(tsWal.refSetId, pWal->refId); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 42fcb8c375..b6e232fa5c 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ -#include "walInt.h" #include "tfile.h" +#include "walInt.h" -SWalReadHandle* walOpenReadHandle(SWal* pWal) { +SWalReadHandle *walOpenReadHandle(SWal *pWal) { SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); - if(pRead == NULL) { + if (pRead == NULL) { return NULL; } pRead->pWal = pWal; @@ -29,7 +29,7 @@ SWalReadHandle* walOpenReadHandle(SWal* pWal) { pRead->capacity = 0; pRead->status = 0; pRead->pHead = malloc(sizeof(SWalHead)); - if(pRead->pHead == NULL) { + if (pRead->pHead == NULL) { free(pRead); return NULL; } @@ -43,27 +43,25 @@ void walCloseReadHandle(SWalReadHandle *pRead) { free(pRead); } -int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { - return 0; -} +int32_t walRegisterRead(SWalReadHandle *pRead, int64_t ver) { return 0; } static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, int64_t ver) { int code = 0; int64_t idxTfd = pRead->readIdxTfd; int64_t logTfd = pRead->readLogTfd; - - //seek position + + // seek position int64_t offset = (ver - fileFirstVer) * sizeof(WalIdxEntry); code = tfLseek(idxTfd, offset, SEEK_SET); - if(code < 0) { + if (code < 0) { return -1; } WalIdxEntry entry; - if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { return -1; } - //TODO:deserialize + // TODO:deserialize ASSERT(entry.ver == ver); code = tfLseek(logTfd, entry.offset, SEEK_SET); if (code < 0) { @@ -80,13 +78,13 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); int64_t logTfd = tfOpenRead(fnameStr); - if(logTfd < 0) { + if (logTfd < 0) { return -1; } walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); int64_t idxTfd = tfOpenRead(fnameStr); - if(idxTfd < 0) { + if (idxTfd < 0) { return -1; } @@ -96,76 +94,75 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { } static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { - int code; + int code; SWal *pWal = pRead->pWal; - if(ver == pRead->curVersion) { + if (ver == pRead->curVersion) { return 0; } - if(ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { return -1; } - if(ver < pWal->vers.snapshotVer) { - + if (ver < pWal->vers.snapshotVer) { } WalFileInfo tmpInfo; tmpInfo.firstVer = ver; - //bsearch in fileSet - WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); + // bsearch in fileSet + WalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); - if(pRead->curFileFirstVer != pRet->firstVer) { + if (pRead->curFileFirstVer != pRet->firstVer) { code = walReadChangeFile(pRead, pRet->firstVer); - if(code < 0) { - //TODO: set error flag + if (code < 0) { + // TODO: set error flag return -1; } } code = walReadSeekFilePos(pRead, pRet->firstVer, ver); - if(code < 0) { + if (code < 0) { return -1; } pRead->curVersion = ver; - + return 0; } int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int code; - //TODO: check wal life - if(pRead->curVersion != ver) { - code = walReadSeekVer(pRead, ver); - if(code != 0) { + // TODO: check wal life + if (pRead->curVersion != ver) { + code = walReadSeekVer(pRead, ver); + if (code != 0) { return -1; } } - if(!tfValid(pRead->readLogTfd)) return -1; + if (!tfValid(pRead->readLogTfd)) return -1; code = tfRead(pRead->readLogTfd, pRead->pHead, sizeof(SWalHead)); - if(code != sizeof(SWalHead)) { + if (code != sizeof(SWalHead)) { return -1; } code = walValidHeadCksum(pRead->pHead); - if(code != 0) { + if (code != 0) { return -1; } - if(pRead->capacity < pRead->pHead->head.len) { - void* ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); - if(ptr == NULL) { + if (pRead->capacity < pRead->pHead->head.len) { + void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); + if (ptr == NULL) { return -1; } pRead->pHead = ptr; pRead->capacity = pRead->pHead->head.len; } - if(pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { + if (pRead->pHead->head.len != tfRead(pRead->readLogTfd, pRead->pHead->head.body, pRead->pHead->head.len)) { return -1; } /*code = walValidBodyCksum(pRead->pHead);*/ ASSERT(pRead->pHead->head.version == ver); - if(code != 0) { + if (code != 0) { return -1; } pRead->curVersion++; @@ -176,40 +173,40 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int32_t walRead(SWal *pWal, SWalHead **ppHead, int64_t ver) { int code; code = walSeekVer(pWal, ver); - if(code != 0) { + if (code != 0) { return code; } - if(*ppHead == NULL) { - void* ptr = realloc(*ppHead, sizeof(SWalHead)); - if(ptr == NULL) { + if (*ppHead == NULL) { + void *ptr = realloc(*ppHead, sizeof(SWalHead)); + if (ptr == NULL) { return -1; } *ppHead = ptr; } - if(tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { + if (tfRead(pWal->writeLogTfd, *ppHead, sizeof(SWalHead)) != sizeof(SWalHead)) { return -1; } - //TODO: endian compatibility processing after read - if(walValidHeadCksum(*ppHead) != 0) { + // TODO: endian compatibility processing after read + if (walValidHeadCksum(*ppHead) != 0) { return -1; } - void* ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); - if(ptr == NULL) { + void *ptr = realloc(*ppHead, sizeof(SWalHead) + (*ppHead)->head.len); + if (ptr == NULL) { free(*ppHead); *ppHead = NULL; return -1; } - if(tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { + if (tfRead(pWal->writeLogTfd, (*ppHead)->head.body, (*ppHead)->head.len) != (*ppHead)->head.len) { return -1; } - //TODO: endian compatibility processing after read - if(walValidBodyCksum(*ppHead) != 0) { + // TODO: endian compatibility processing after read + if (walValidBodyCksum(*ppHead) != 0) { return -1; } - + return 0; } /*int32_t walReadWithFp(SWal *pWal, FWalWrite writeFp, int64_t verStart, int32_t readNum) {*/ - /*return 0;*/ +/*return 0;*/ /*}*/ diff --git a/source/libs/wal/src/walSeek.c b/source/libs/wal/src/walSeek.c index 769b2fd2e4..82c596d225 100644 --- a/source/libs/wal/src/walSeek.c +++ b/source/libs/wal/src/walSeek.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "tref.h" #include "tfile.h" +#include "tref.h" #include "walInt.h" static int walSeekFilePos(SWal* pWal, int64_t ver) { @@ -25,17 +25,17 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { int64_t idxTfd = pWal->writeIdxTfd; int64_t logTfd = pWal->writeLogTfd; - - //seek position + + // seek position int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = tfLseek(idxTfd, idxOff, SEEK_SET); - if(code != 0) { + if (code != 0) { return -1; } WalIdxEntry entry; - //TODO:deserialize + // TODO:deserialize code = tfRead(idxTfd, &entry, sizeof(WalIdxEntry)); - if(code != 0) { + if (code != 0) { return -1; } ASSERT(entry.ver == ver); @@ -46,8 +46,8 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) { return code; } -int walChangeFileToLast(SWal *pWal) { - int64_t idxTfd, logTfd; +int walChangeFileToLast(SWal* pWal) { + int64_t idxTfd, logTfd; WalFileInfo* pRet = taosArrayGetLast(pWal->fileInfoSet); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; @@ -55,42 +55,42 @@ int walChangeFileToLast(SWal *pWal) { char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenReadWrite(fnameStr); - if(idxTfd < 0) { + if (idxTfd < 0) { return -1; } walBuildLogName(pWal, fileFirstVer, fnameStr); logTfd = tfOpenReadWrite(fnameStr); - if(logTfd < 0) { + if (logTfd < 0) { return -1; } - //switch file + // switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; return 0; } -int walChangeFile(SWal *pWal, int64_t ver) { - int code = 0; +int walChangeFile(SWal* pWal, int64_t ver) { + int code = 0; int64_t idxTfd, logTfd; - char fnameStr[WAL_FILE_LEN]; + char fnameStr[WAL_FILE_LEN]; code = tfClose(pWal->writeLogTfd); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return -1; } code = tfClose(pWal->writeIdxTfd); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return -1; } WalFileInfo tmpInfo; tmpInfo.firstVer = ver; - //bsearch in fileSet + // bsearch in fileSet WalFileInfo* pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE); ASSERT(pRet != NULL); int64_t fileFirstVer = pRet->firstVer; - //closed - if(taosArrayGetLast(pWal->fileInfoSet) != pRet) { + // closed + if (taosArrayGetLast(pWal->fileInfoSet) != pRet) { walBuildIdxName(pWal, fileFirstVer, fnameStr); idxTfd = tfOpenRead(fnameStr); walBuildLogName(pWal, fileFirstVer, fnameStr); @@ -107,27 +107,26 @@ int walChangeFile(SWal *pWal, int64_t ver) { return fileFirstVer; } -int walSeekVer(SWal *pWal, int64_t ver) { +int walSeekVer(SWal* pWal, int64_t ver) { int code; - if(ver == pWal->vers.lastVer) { + if (ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->vers.lastVer|| ver < pWal->vers.firstVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { return -1; } - if(ver < pWal->vers.snapshotVer) { - + if (ver < pWal->vers.snapshotVer) { } - if(ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { + if (ver < walGetCurFileFirstVer(pWal) || (ver > walGetCurFileLastVer(pWal))) { code = walChangeFile(pWal, ver); - if(code != 0) { + if (code != 0) { return -1; } } code = walSeekFilePos(pWal, ver); - if(code != 0) { + if (code != 0) { return -1; } - + return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 67b70f249d..c3a7ca5f4d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -24,7 +24,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) { ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); - if(ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { + if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { return -1; } pWal->vers.commitVer = ver; @@ -32,57 +32,57 @@ int32_t walCommit(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { - int code; + int code; char fnameStr[WAL_FILE_LEN]; - if(ver == pWal->vers.lastVer) { + if (ver == pWal->vers.lastVer) { return 0; } - if(ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { return -1; } pthread_mutex_lock(&pWal->mutex); - //find correct file - if(ver < walGetLastFileFirstVer(pWal)) { - //close current files + // find correct file + if (ver < walGetLastFileFirstVer(pWal)) { + // close current files tfClose(pWal->writeIdxTfd); tfClose(pWal->writeLogTfd); - //open old files + // open old files code = walChangeFile(pWal, ver); - if(code != 0) { + if (code != 0) { return -1; } - //delete files + // delete files int fileSetSize = taosArrayGetSize(pWal->fileInfoSet); - for(int i = pWal->writeCur; i < fileSetSize; i++) { - walBuildLogName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + for (int i = pWal->writeCur; i < fileSetSize; i++) { + walBuildLogName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); remove(fnameStr); - walBuildIdxName(pWal, ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); + walBuildIdxName(pWal, ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, i))->firstVer, fnameStr); remove(fnameStr); } - //pop from fileInfoSet + // pop from fileInfoSet taosArraySetSize(pWal->fileInfoSet, pWal->writeCur + 1); } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); int64_t idxTfd = tfOpenReadWrite(fnameStr); - //TODO:change to deserialize function - if(idxTfd < 0) { + // TODO:change to deserialize function + if (idxTfd < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; } int64_t idxOff = walGetVerIdxOffset(pWal, ver); code = tfLseek(idxTfd, idxOff, SEEK_SET); - if(code < 0) { + if (code < 0) { pthread_mutex_unlock(&pWal->mutex); return -1; } - //read idx file and get log file pos - //TODO:change to deserialize function + // read idx file and get log file pos + // TODO:change to deserialize function WalIdxEntry entry; - if(tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { + if (tfRead(idxTfd, &entry, sizeof(WalIdxEntry)) != sizeof(WalIdxEntry)) { pthread_mutex_unlock(&pWal->mutex); return -1; } @@ -90,56 +90,56 @@ int32_t walRollback(SWal *pWal, int64_t ver) { walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); int64_t logTfd = tfOpenReadWrite(fnameStr); - if(logTfd < 0) { - //TODO + if (logTfd < 0) { + // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } code = tfLseek(logTfd, entry.offset, SEEK_SET); - if(code < 0) { - //TODO + if (code < 0) { + // TODO pthread_mutex_unlock(&pWal->mutex); return -1; } - //validate offset + // validate offset SWalHead head; ASSERT(tfValid(logTfd)); int size = tfRead(logTfd, &head, sizeof(SWalHead)); - if(size != sizeof(SWalHead)) { + if (size != sizeof(SWalHead)) { return -1; } code = walValidHeadCksum(&head); ASSERT(code == 0); - if(code != 0) { + if (code != 0) { return -1; } - if(head.head.version != ver) { - //TODO + if (head.head.version != ver) { + // TODO return -1; } - //truncate old files + // truncate old files code = tfFtruncate(logTfd, entry.offset); - if(code < 0) { + if (code < 0) { return -1; } code = tfFtruncate(idxTfd, idxOff); - if(code < 0) { + if (code < 0) { return -1; } pWal->vers.lastVer = ver - 1; - ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; - ((WalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; + ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; + ((WalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; - //unlock + // unlock pthread_mutex_unlock(&pWal->mutex); return 0; } -int32_t walBeginSnapshot(SWal* pWal, int64_t ver) { +int32_t walBeginSnapshot(SWal *pWal, int64_t ver) { pWal->vers.verInSnapshotting = ver; - //check file rolling - if(pWal->cfg.retentionPeriod == 0) { + // check file rolling + if (pWal->cfg.retentionPeriod == 0) { walRoll(pWal); } @@ -148,54 +148,54 @@ int32_t walBeginSnapshot(SWal* pWal, int64_t ver) { int32_t walEndSnapshot(SWal *pWal) { int64_t ver = pWal->vers.verInSnapshotting; - if(ver == -1) return -1; + if (ver == -1) return -1; pWal->vers.snapshotVer = ver; int ts = taosGetTimestampSec(); - int deleteCnt = 0; - int64_t newTotSize = pWal->totSize; + int deleteCnt = 0; + int64_t newTotSize = pWal->totSize; WalFileInfo tmp; tmp.firstVer = ver; - //find files safe to delete - WalFileInfo* pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); - if(ver >= pInfo->lastVer) { + // find files safe to delete + WalFileInfo *pInfo = taosArraySearch(pWal->fileInfoSet, &tmp, compareWalFileInfo, TD_LE); + if (ver >= pInfo->lastVer) { pInfo++; } - //iterate files, until the searched result - for(WalFileInfo* iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { - if(pWal->totSize > pWal->cfg.retentionSize || - iter->closeTs + pWal->cfg.retentionPeriod > ts) { - //delete according to file size or close time + // iterate files, until the searched result + for (WalFileInfo *iter = pWal->fileInfoSet->pData; iter < pInfo; iter++) { + if (pWal->totSize > pWal->cfg.retentionSize || iter->closeTs + pWal->cfg.retentionPeriod > ts) { + // delete according to file size or close time deleteCnt++; newTotSize -= iter->fileSize; } } char fnameStr[WAL_FILE_LEN]; - //remove file - for(int i = 0; i < deleteCnt; i++) { - WalFileInfo* pInfo = taosArrayGet(pWal->fileInfoSet, i); - walBuildLogName(pWal, pInfo->firstVer, fnameStr); + // remove file + for (int i = 0; i < deleteCnt; i++) { + WalFileInfo *pInfo = taosArrayGet(pWal->fileInfoSet, i); + walBuildLogName(pWal, pInfo->firstVer, fnameStr); remove(fnameStr); - walBuildIdxName(pWal, pInfo->firstVer, fnameStr); + walBuildIdxName(pWal, pInfo->firstVer, fnameStr); remove(fnameStr); } - //make new array, remove files - taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); - if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + // make new array, remove files + taosArrayPopFrontBatch(pWal->fileInfoSet, deleteCnt); + if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->writeCur = -1; pWal->vers.firstVer = -1; } else { - pWal->vers.firstVer = ((WalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + pWal->vers.firstVer = ((WalFileInfo *)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; } - pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1;; + pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; + ; pWal->totSize = newTotSize; pWal->vers.verInSnapshotting = -1; - //save snapshot ver, commit ver + // save snapshot ver, commit ver int code = walSaveMeta(pWal); - if(code != 0) { + if (code != 0) { return -1; } @@ -204,41 +204,41 @@ int32_t walEndSnapshot(SWal *pWal) { int walRoll(SWal *pWal) { int code = 0; - if(pWal->writeIdxTfd != -1) { + if (pWal->writeIdxTfd != -1) { code = tfClose(pWal->writeIdxTfd); - if(code != 0) { + if (code != 0) { return -1; } } - if(pWal->writeLogTfd != -1) { + if (pWal->writeLogTfd != -1) { code = tfClose(pWal->writeLogTfd); - if(code != 0) { + if (code != 0) { return -1; } } int64_t idxTfd, logTfd; - //create new file + // create new file int64_t newFileFirstVersion = pWal->vers.lastVer + 1; - char fnameStr[WAL_FILE_LEN]; + char fnameStr[WAL_FILE_LEN]; walBuildIdxName(pWal, newFileFirstVersion, fnameStr); idxTfd = tfOpenCreateWriteAppend(fnameStr); - if(idxTfd < 0) { + if (idxTfd < 0) { ASSERT(0); return -1; } walBuildLogName(pWal, newFileFirstVersion, fnameStr); logTfd = tfOpenCreateWriteAppend(fnameStr); - if(logTfd < 0) { + if (logTfd < 0) { ASSERT(0); return -1; } code = walRollFileInfo(pWal); - if(code != 0) { + if (code != 0) { ASSERT(0); return -1; } - //switch file + // switch file pWal->writeIdxTfd = idxTfd; pWal->writeLogTfd = logTfd; pWal->writeCur = taosArrayGetSize(pWal->fileInfoSet) - 1; @@ -248,10 +248,10 @@ int walRoll(SWal *pWal) { } static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - WalIdxEntry entry = { .ver = ver, .offset = offset }; - int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); - if(size != sizeof(WalIdxEntry)) { - //TODO truncate + WalIdxEntry entry = {.ver = ver, .offset = offset}; + int size = tfWrite(pWal->writeIdxTfd, &entry, sizeof(WalIdxEntry)); + if (size != sizeof(WalIdxEntry)) { + // TODO truncate return -1; } return 0; @@ -265,21 +265,21 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i if (pWal->cfg.level == TAOS_WAL_NOLOG) return 0; if (index == pWal->vers.lastVer + 1) { - if(taosArrayGetSize(pWal->fileInfoSet) == 0) { + if (taosArrayGetSize(pWal->fileInfoSet) == 0) { pWal->vers.firstVer = index; code = walRoll(pWal); ASSERT(code == 0); } else { int64_t passed = walGetSeq() - pWal->lastRollSeq; - if(pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { + if (pWal->cfg.rollPeriod != -1 && pWal->cfg.rollPeriod != 0 && passed > pWal->cfg.rollPeriod) { walRoll(pWal); - } else if(pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { + } else if (pWal->cfg.segSize != -1 && pWal->cfg.segSize != 0 && walGetLastFileSize(pWal) > pWal->cfg.segSize) { walRoll(pWal); } } } else { - //reject skip log or rewrite log - //must truncate explicitly first + // reject skip log or rewrite log + // must truncate explicitly first return -1; } /*if (!tfValid(pWal->writeLogTfd)) return -1;*/ @@ -294,28 +294,30 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); if (tfWrite(pWal->writeLogTfd, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { - //ftruncate + // ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(errno)); } - if (tfWrite(pWal->writeLogTfd, (char*)body, bodyLen) != bodyLen) { - //ftruncate + if (tfWrite(pWal->writeLogTfd, (char *)body, bodyLen) != bodyLen) { + // ftruncate code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), + strerror(errno)); } code = walWriteIndex(pWal, index, offset); - if(code != 0) { - //TODO + if (code != 0) { + // TODO return -1; } - //set status + // set status pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalHead) + bodyLen; walGetCurFileInfo(pWal)->lastVer = index; walGetCurFileInfo(pWal)->fileSize += sizeof(SWalHead) + bodyLen; - + pthread_mutex_unlock(&pWal->mutex); return code; @@ -323,33 +325,34 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, i void walFsync(SWal *pWal, bool forceFsync) { if (forceFsync || (pWal->cfg.level == TAOS_WAL_FSYNC && pWal->cfg.fsyncPeriod == 0)) { - wTrace("vgId:%d, fileId:%"PRId64".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); + wTrace("vgId:%d, fileId:%" PRId64 ".log, do fsync", pWal->cfg.vgId, walGetCurFileFirstVer(pWal)); if (tfFsync(pWal->writeLogTfd) < 0) { - wError("vgId:%d, file:%"PRId64".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), strerror(errno)); + wError("vgId:%d, file:%" PRId64 ".log, fsync failed since %s", pWal->cfg.vgId, walGetCurFileFirstVer(pWal), + strerror(errno)); } } } /*static int walValidateOffset(SWal* pWal, int64_t ver) {*/ - /*int code = 0;*/ - /*SWalHead *pHead = NULL;*/ - /*code = (int)walRead(pWal, &pHead, ver);*/ - /*if(pHead->head.version != ver) {*/ - /*return -1;*/ - /*}*/ - /*return 0;*/ +/*int code = 0;*/ +/*SWalHead *pHead = NULL;*/ +/*code = (int)walRead(pWal, &pHead, ver);*/ +/*if(pHead->head.version != ver) {*/ +/*return -1;*/ +/*}*/ +/*return 0;*/ /*}*/ /*static int64_t walGetOffset(SWal* pWal, int64_t ver) {*/ - /*int code = walSeekVer(pWal, ver);*/ - /*if(code != 0) {*/ - /*return -1;*/ - /*}*/ - - /*code = walValidateOffset(pWal, ver);*/ - /*if(code != 0) {*/ - /*return -1;*/ - /*}*/ - - /*return 0;*/ +/*int code = walSeekVer(pWal, ver);*/ +/*if(code != 0) {*/ +/*return -1;*/ +/*}*/ + +/*code = walValidateOffset(pWal, ver);*/ +/*if(code != 0) {*/ +/*return -1;*/ +/*}*/ + +/*return 0;*/ /*}*/ diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index d06388201e..cd082a3a43 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -6,111 +6,105 @@ #include "walInt.h" const char* ranStr = "tvapq02tcp"; -const int ranStrLen = strlen(ranStr); +const int ranStrLen = strlen(ranStr); class WalCleanEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(); - ASSERT(code == 0); - } + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } - static void TearDownTestCase() { - walCleanUp(); - } + static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - free(pCfg); - ASSERT(pWal != NULL); - } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + free(pCfg); + ASSERT(pWal != NULL); + } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } - SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; }; class WalCleanDeleteEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(); - ASSERT(code == 0); - } + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } - static void TearDownTestCase() { - walCleanUp(); - } + static void TearDownTestCase() { walCleanUp(); } - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - free(pCfg); - ASSERT(pWal != NULL); - } + void SetUp() override { + taosRemoveDir(pathName); + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + free(pCfg); + ASSERT(pWal != NULL); + } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } - SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; }; class WalKeepEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(); - ASSERT(code == 0); - } + protected: + static void SetUpTestCase() { + int code = walInit(); + ASSERT(code == 0); + } - static void TearDownTestCase() { - walCleanUp(); - } + static void TearDownTestCase() { walCleanUp(); } - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } + void walResetEnv() { + TearDown(); + taosRemoveDir(pathName); + SetUp(); + } - void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - free(pCfg); - ASSERT(pWal != NULL); - } + void SetUp() override { + SWalCfg* pCfg = (SWalCfg*)malloc(sizeof(SWalCfg)); + memset(pCfg, 0, sizeof(SWalCfg)); + pCfg->rollPeriod = -1; + pCfg->segSize = -1; + pCfg->retentionPeriod = 0; + pCfg->retentionSize = 0; + pCfg->level = TAOS_WAL_FSYNC; + pWal = walOpen(pathName, pCfg); + free(pCfg); + ASSERT(pWal != NULL); + } - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + void TearDown() override { + walClose(pWal); + pWal = NULL; + } - SWal* pWal = NULL; - const char* pathName = "/tmp/wal_test"; + SWal* pWal = NULL; + const char* pathName = "/tmp/wal_test"; }; TEST_F(WalCleanEnv, createNew) { @@ -139,7 +133,7 @@ TEST_F(WalCleanEnv, serialize) { ASSERT(code == 0); code = walRollFileInfo(pWal); ASSERT(code == 0); - char*ss = walMetaSerialize(pWal); + char* ss = walMetaSerialize(pWal); printf("%s\n", ss); free(ss); code = walSaveMeta(pWal); @@ -162,11 +156,11 @@ TEST_F(WalKeepEnv, readOldMeta) { walResetEnv(); int code; - for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + for (int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); + code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -182,7 +176,7 @@ TEST_F(WalKeepEnv, readOldMeta) { int len = strlen(oldss); ASSERT_EQ(len, strlen(newss)); - for(int i = 0; i < len; i++) { + for (int i = 0; i < len; i++) { EXPECT_EQ(oldss[i], newss[i]); } free(oldss); @@ -191,11 +185,11 @@ TEST_F(WalKeepEnv, readOldMeta) { TEST_F(WalCleanEnv, write) { int code; - for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + for (int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); - code = walWrite(pWal, i+2, i, (void*)ranStr, ranStrLen); + code = walWrite(pWal, i + 2, i, (void*)ranStr, ranStrLen); ASSERT_EQ(code, -1); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -205,8 +199,8 @@ TEST_F(WalCleanEnv, write) { TEST_F(WalCleanEnv, rollback) { int code; - for(int i = 0; i < 10; i++) { - code = walWrite(pWal, i, i+1, (void*)ranStr, ranStrLen); + for (int i = 0; i < 10; i++) { + code = walWrite(pWal, i, i + 1, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); } @@ -223,7 +217,7 @@ TEST_F(WalCleanEnv, rollback) { TEST_F(WalCleanDeleteEnv, roll) { int code; int i; - for(i = 0; i < 100; i++) { + for (i = 0; i < 100; i++) { code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, i); @@ -231,16 +225,16 @@ TEST_F(WalCleanDeleteEnv, roll) { ASSERT_EQ(pWal->vers.commitVer, i); } - walBeginSnapshot(pWal, i-1); - ASSERT_EQ(pWal->vers.verInSnapshotting, i-1); + walBeginSnapshot(pWal, i - 1); + ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); walEndSnapshot(pWal); - ASSERT_EQ(pWal->vers.snapshotVer, i-1); + ASSERT_EQ(pWal->vers.snapshotVer, i - 1); ASSERT_EQ(pWal->vers.verInSnapshotting, -1); code = walWrite(pWal, 5, 0, (void*)ranStr, ranStrLen); ASSERT_NE(code, 0); - for(; i < 200; i++) { + for (; i < 200; i++) { code = walWrite(pWal, i, 0, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); code = walCommit(pWal, i); @@ -255,36 +249,36 @@ TEST_F(WalCleanDeleteEnv, roll) { TEST_F(WalKeepEnv, readHandleRead) { walResetEnv(); - int code; + int code; SWalReadHandle* pRead = walOpenReadHandle(pWal); ASSERT(pRead != NULL); - int i ; - for(i = 0; i < 100; i++) { + int i; + for (i = 0; i < 100; i++) { char newStr[100]; sprintf(newStr, "%s-%d", ranStr, i); int len = strlen(newStr); code = walWrite(pWal, i, 0, newStr, len); ASSERT_EQ(code, 0); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ver = rand() % 100; code = walReadWithHandle(pRead, ver); ASSERT_EQ(code, 0); - //printf("rrbody: \n"); - //for(int i = 0; i < pRead->pHead->head.len; i++) { - //printf("%d ", pRead->pHead->head.body[i]); + // printf("rrbody: \n"); + // for(int i = 0; i < pRead->pHead->head.len; i++) { + // printf("%d ", pRead->pHead->head.body[i]); //} - //printf("\n"); + // printf("\n"); ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver+1); + ASSERT_EQ(pRead->curVersion, ver + 1); char newStr[100]; sprintf(newStr, "%s-%d", ranStr, ver); int len = strlen(newStr); ASSERT_EQ(pRead->pHead->head.len, len); - for(int j = 0; j < len; j++) { + for (int j = 0; j < len; j++) { EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); } }