diff --git a/.gitignore b/.gitignore index 2308ea7896..b9b5341b06 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ build/ compile_commands.json .cache .ycm_extra_conf.py +.tasks +.vimspector.json .vscode/ .idea/ cmake-build-debug/ @@ -86,6 +88,7 @@ tests/comparisonTest/opentsdb/opentsdbtest/.settings/ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ +source/libs/parser/inc/new_sql.* # Emacs # -*- mode: gitignore; -*- @@ -101,4 +104,4 @@ TAGS contrib/* !contrib/CMakeLists.txt -!contrib/test \ No newline at end of file +!contrib/test diff --git a/CMakeLists.txt b/CMakeLists.txt index 0383cc8aed..fd542966cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) add_subdirectory(tests) +add_subdirectory(example) # docs add_subdirectory(docs) diff --git a/cmake/cmake.options b/cmake/cmake.options index faa45256fb..e84d02800c 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -50,6 +50,12 @@ option( OFF ) +option( + BUILD_WITH_UV_TRANS + "If build with libuv_trans " + OFF +) + option( BUILD_WITH_CRAFT "If build with canonical-raft" diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt new file mode 100644 index 0000000000..71a9f9f4c5 --- /dev/null +++ b/example/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src TMQ_DEMO_SRC) + +add_executable(tmq ${TMQ_DEMO_SRC}) +target_link_libraries( + tmq + PUBLIC taos + #PUBLIC util + #PUBLIC common + #PUBLIC os +) +target_include_directories( + tmq + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) diff --git a/example/src/tmq.c b/example/src/tmq.c new file mode 100644 index 0000000000..64b631159b --- /dev/null +++ b/example/src/tmq.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "taos.h" + +static int running = 1; +static void msg_process(tmq_message_t* message) { + tmqShowMsg(message); +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu2 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + + const char* sql = "select * from st1"; + pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); + /*if (taos_errno(pRes) != 0) {*/ + /*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/ + /*return -1;*/ + /*}*/ + /*taos_free_result(pRes);*/ + taos_close(pConn); + return 0; +} + +tmq_t* build_consumer() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); + return tmq; + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + tmq_subscribe(tmq, topic_list); + return NULL; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + return topic_list; +} + +void basic_consume_loop(tmq_t *tmq, + tmq_list_t *topics) { + tmq_resp_err_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + clock_t startTime = clock(); + while (running) { + tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 0); + if (tmqmessage) { + cnt++; + /*msg_process(tmqmessage);*/ + tmq_message_destroy(tmqmessage); + } else { + break; + } + } + clock_t endTime = clock(); + printf("log cnt: %d %f s\n", cnt, (double)(endTime - startTime) / CLOCKS_PER_SEC); + + err = tmq_consumer_close(tmq); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void sync_consume_loop(tmq_t *tmq, + tmq_list_t *topics) { + static const int MIN_COMMIT_COUNT = 1000; + + int msg_count = 0; + tmq_resp_err_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + return; + } + + while (running) { + tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500); + if (tmqmessage) { + msg_process(tmqmessage); + tmq_message_destroy(tmqmessage); + + if ((++msg_count % MIN_COMMIT_COUNT) == 0) + tmq_commit(tmq, NULL, 0); + } + } + + err = tmq_consumer_close(tmq); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main() { + int code; + code = init_env(); + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + /*sync_consume_loop(tmq, topic_list);*/ +} diff --git a/include/client/taos.h b/include/client/taos.h index 40772e9d2c..a8627a43da 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -92,15 +92,6 @@ typedef struct taosField { typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); -DLL_EXPORT void taos_cleanup(void); -DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); -DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); -DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); -DLL_EXPORT void taos_close(TAOS *taos); - -const char *taos_data_type(int type); - typedef struct TAOS_BIND { int buffer_type; void * buffer; @@ -134,6 +125,15 @@ typedef struct TAOS_MULTI_BIND { int num; } TAOS_MULTI_BIND; +DLL_EXPORT void taos_cleanup(void); +DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); +DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +DLL_EXPORT void taos_close(TAOS *taos); + +const char *taos_data_type(int type); + DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags); @@ -192,37 +192,67 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); -typedef struct tmq_t tmq_t; -typedef struct tmq_conf_t tmq_conf_t; -typedef struct tmq_list_t tmq_list_t; -typedef struct tmq_message_t tmq_message_t; -typedef struct tmq_message_topic_t tmq_message_topic_t; -typedef struct tmq_message_tb_t tmq_message_tb_t; -typedef struct tmq_tb_iter_t tmq_tb_iter_t; -typedef struct tmq_message_col_t tmq_message_col_t; -typedef struct tmq_col_iter_t tmq_col_iter_t; +/* --------------------------TMQ INTERFACE------------------------------- */ -DLL_EXPORT tmq_list_t* tmq_list_new(); -DLL_EXPORT int32_t tmq_list_append(tmq_list_t*, char*); +enum tmq_resp_err_t { + TMQ_RESP_ERR__SUCCESS = 0, + TMQ_RESP_ERR__FAIL = 1, +}; -DLL_EXPORT tmq_conf_t* tmq_conf_new(); +typedef enum tmq_resp_err_t tmq_resp_err_t; -DLL_EXPORT int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value); +typedef struct tmq_t tmq_t; +typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; +typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; -DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); +typedef struct tmq_conf_t tmq_conf_t; +typedef struct tmq_list_t tmq_list_t; +typedef struct tmq_message_t tmq_message_t; -DLL_EXPORT tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen); +typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); -DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); +DLL_EXPORT tmq_list_t *tmq_list_new(); +DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); -DLL_EXPORT tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time); +DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); +DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT void tmq_message_destroy(tmq_message_t* tmq_message); +DLL_EXPORT const char* tmq_err2str(tmq_resp_err_t); -DLL_EXPORT int32_t tmq_topic_num(tmq_message_t* msg); -DLL_EXPORT char* tmq_get_topic(tmq_message_topic_t* msg); -DLL_EXPORT int32_t tmq_get_vgId(tmq_message_topic_t* msg); -DLL_EXPORT tmq_message_tb_t* tmq_get_next_tb(tmq_message_topic_t* msg, tmq_tb_iter_t* iter); -DLL_EXPORT tmq_message_col_t* tmq_get_next_col(tmq_message_tb_t* msg, tmq_col_iter_t* iter); +/* ------------------------TMQ CONSUMER INTERFACE------------------------ */ +DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +#endif +DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); +DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t* tmq); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups); +DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups); +#endif +DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); +#endif +/* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ + +enum tmq_conf_res_t { + TMQ_CONF_UNKNOWN = -2, + TMQ_CONF_INVALID = -1, + TMQ_CONF_OK = 0, +}; + +typedef enum tmq_conf_res_t tmq_conf_res_t; + +DLL_EXPORT tmq_conf_t *tmq_conf_new(); +DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); +DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); +DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); + +//temporary used function for demo only +void tmqShowMsg(tmq_message_t* tmq_message); #ifdef __cplusplus } diff --git a/include/common/common.h b/include/common/common.h index 9b8a465442..31f905d47f 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -38,6 +38,16 @@ // int16_t bytes; //} SSchema; +#define TMQ_REQ_TYPE_COMMIT_ONLY 0 +#define TMQ_REQ_TYPE_CONSUME_ONLY 1 +#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 + +typedef struct { + uint32_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid +} STableGroupInfo; + typedef struct SColumnDataAgg { int16_t colId; int64_t sum; @@ -57,17 +67,12 @@ typedef struct SDataBlockInfo { typedef struct SConstantItem { SColumnInfo info; - int32_t startIndex; // run-length-encoding to save the space for multiple rows - int32_t endIndex; + int32_t startRow; // run-length-encoding to save the space for multiple rows + int32_t endRow; SVariant value; } SConstantItem; -typedef struct { - uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid -} STableGroupInfo; - +// info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray @@ -75,11 +80,133 @@ typedef struct SSDataBlock { SDataBlockInfo info; } SSDataBlock; +// pBlockAgg->numOfNull == info.rows, all data are null +// pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed - char *pData; // the corresponding block data in memory + SColumnInfo info; // TODO filter info needs to be removed + char *nullbitmap;// + char *pData; // the corresponding block data in memory } SColumnInfoData; +static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { + int64_t tbUid = pBlock->info.uid; + int32_t numOfCols = pBlock->info.numOfCols; + int32_t rows = pBlock->info.rows; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, tbUid); + tlen += taosEncodeFixedI32(buf, numOfCols); + tlen += taosEncodeFixedI32(buf, rows); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tlen += taosEncodeFixedI16(buf, pColData->info.colId); + tlen += taosEncodeFixedI16(buf, pColData->info.type); + tlen += taosEncodeFixedI16(buf, pColData->info.bytes); + int32_t colSz = rows * pColData->info.bytes; + tlen += taosEncodeBinary(buf, pColData->pData, colSz); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { + int32_t sz; + + buf = taosDecodeFixedI64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI32(buf, &pBlock->info.numOfCols); + buf = taosDecodeFixedI32(buf, &pBlock->info.rows); + buf = taosDecodeFixedI32(buf, &sz); + pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); + for (int32_t i = 0; i < sz; i++) { + SColumnInfoData data = {0}; + buf = taosDecodeFixedI16(buf, &data.info.colId); + buf = taosDecodeFixedI16(buf, &data.info.type); + buf = taosDecodeFixedI16(buf, &data.info.bytes); + int32_t colSz = pBlock->info.rows * data.info.bytes; + buf = taosDecodeBinary(buf, (void**)&data.pData, colSz); + taosArrayPush(pBlock->pDataBlock, &data); + } + return buf; +} + +static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) { + int32_t tlen = 0; + int32_t sz = 0; + tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); + if (pRsp->numOfTopics == 0) return tlen; + tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas); + if (pRsp->pBlockData) { + sz = taosArrayGetSize(pRsp->pBlockData); + } + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pBlock = (SSDataBlock*) taosArrayGet(pRsp->pBlockData, i); + tlen += tEncodeDataBlock(buf, pBlock); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { + int32_t sz; + buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); + if (pRsp->numOfTopics == 0) return buf; + pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); + if (pRsp->schemas == NULL) return NULL; + buf = tDecodeSSchemaWrapper(buf, pRsp->schemas); + buf = taosDecodeFixedI32(buf, &sz); + pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock block = {0}; + tDecodeDataBlock(buf, &block); + taosArrayPush(pRsp->pBlockData, &block); + } + return buf; +} + +static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return; + } + + //int32_t numOfOutput = pBlock->info.numOfCols; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + for(int32_t i = 0; i < sz; ++i) { + SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); + } + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); + //tfree(pBlock); +} + + +static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { + if (pRsp->schemas) { + if (pRsp->schemas->nCols) { + tfree(pRsp->schemas->pSchema); + } + free(pRsp->schemas); + } + taosArrayDestroyEx(pRsp->pBlockData, (void(*)(void*))tDeleteSSDataBlock); + pRsp->pBlockData = NULL; + //for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + //SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); + //tDeleteSSDataBlock(pDataBlock); + //} +} + //====================================================================================================================== // the following structure shared by parser and executor typedef struct SColumn { diff --git a/include/common/tep.h b/include/common/tep.h index c8f45e4c82..69dd385a37 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -1,6 +1,10 @@ #ifndef TDENGINE_TEP_H #define TDENGINE_TEP_H +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" #include "tmsg.h" @@ -9,10 +13,16 @@ typedef struct SCorEpSet { SEpSet epSet; } SCorEpSet; -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); +int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); +void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); + bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); -void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); +void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); SEpSet getEpSet_s(SCorEpSet *pEpSet); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_TEP_H diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1b96cbdc78..4bc8c9fb86 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -64,7 +64,6 @@ extern int8_t tsKeepOriginalColumnName; extern int8_t tsDeadLockKillQuery; // client -extern int32_t tsMaxSQLStringLen; extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; extern int8_t tsTscEnableRecordSql; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 398e46e6a4..c641fbb1a3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -76,6 +76,13 @@ typedef enum { HEARTBEAT_TYPE_MAX } EHbType; +enum { + HEARTBEAT_KEY_DBINFO = 1, + HEARTBEAT_KEY_STBINFO, + HEARTBEAT_KEY_MQ_TMP, +}; + + typedef enum _mgmt_table { TSDB_MGMT_TABLE_START, TSDB_MGMT_TABLE_ACCT, @@ -140,24 +147,29 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) + +#define TD_SUPER_TABLE TSDB_SUPER_TABLE +#define TD_CHILD_TABLE TSDB_CHILD_TABLE +#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE + typedef struct { int32_t vgId; - char* dbName; - char* tableFullName; + char* dbFName; + char* tbName; } SBuildTableMetaInput; typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; int32_t vgVersion; } SBuildUseDBInput; #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta -typedef struct { +typedef struct SEp { char fqdn[TSDB_FQDN_LEN]; uint16_t port; -} SEpAddr; +} SEp; typedef struct { int32_t contLen; @@ -266,8 +278,7 @@ typedef struct { typedef struct SEpSet { int8_t inUse; int8_t numOfEps; - uint16_t port[TSDB_MAX_REPLICA]; - char fqdn[TSDB_MAX_REPLICA][TSDB_FQDN_LEN]; + SEp eps[TSDB_MAX_REPLICA]; } SEpSet; static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { @@ -275,8 +286,8 @@ static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { tlen += taosEncodeFixedI8(buf, pEp->inUse); tlen += taosEncodeFixedI8(buf, pEp->numOfEps); for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - tlen += taosEncodeFixedU16(buf, pEp->port[i]); - tlen += taosEncodeString(buf, pEp->fqdn[i]); + tlen += taosEncodeFixedU16(buf, pEp->eps[i].port); + tlen += taosEncodeString(buf, pEp->eps[i].fqdn); } return tlen; } @@ -285,11 +296,12 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { buf = taosDecodeFixedI8(buf, &pEp->inUse); buf = taosDecodeFixedI8(buf, &pEp->numOfEps); for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - buf = taosDecodeFixedU16(buf, &pEp->port[i]); - buf = taosDecodeStringTo(buf, pEp->fqdn[i]); + buf = taosDecodeFixedU16(buf, &pEp->eps[i].port); + buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn); } return buf; } + typedef struct { int32_t acctId; int64_t clusterId; @@ -297,6 +309,7 @@ typedef struct { int8_t superUser; int8_t align[3]; SEpSet epSet; + char sVersion[128]; } SConnectRsp; typedef struct { @@ -507,21 +520,26 @@ typedef struct { } SAlterDbReq; typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; int8_t ignoreNotExists; } SDropDbReq; typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + uint64_t uid; +} SDropDbRsp; + +typedef struct { + char db[TSDB_DB_FNAME_LEN]; int32_t vgVersion; } SUseDbReq; typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; } SSyncDbReq; typedef struct { - char db[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; typedef struct { @@ -617,8 +635,7 @@ typedef struct { int32_t id; int8_t isMnode; int8_t align; - uint16_t port; - char fqdn[TSDB_FQDN_LEN]; + SEp ep; } SDnodeEp; typedef struct { @@ -679,8 +696,8 @@ typedef struct { typedef struct { SMsgHead header; - char dbFname[TSDB_DB_FNAME_LEN]; - char tableFname[TSDB_TABLE_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; } STableInfoReq; typedef struct { @@ -691,30 +708,23 @@ typedef struct { char tableNames[]; } SMultiTableInfoReq; +// todo refactor typedef struct SVgroupInfo { int32_t vgId; uint32_t hashBegin; uint32_t hashEnd; - int8_t inUse; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpSet epset; } SVgroupInfo; typedef struct { - int32_t vgId; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; -} SVgroupMsg; - -typedef struct { - int32_t numOfVgroups; - SVgroupMsg vgroups[]; + int32_t numOfVgroups; + SVgroupInfo vgroups[]; } SVgroupsInfo; typedef struct { - char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name - char stbFname[TSDB_TABLE_FNAME_LEN]; - char dbFname[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; int32_t numOfTags; int32_t numOfColumns; int8_t precision; @@ -747,7 +757,7 @@ typedef struct { typedef struct { char db[TSDB_DB_FNAME_LEN]; - int64_t uid; + uint64_t uid; int32_t vgVersion; int32_t vgNum; int8_t hashMethod; @@ -871,13 +881,21 @@ typedef struct { char desc[TSDB_STEP_DESC_LEN]; } SStartupReq; +/** + * The layout of the query message payload is as following: + * +--------------------+---------------------------------+ + * |Sql statement | Physical plan | + * |(denoted by sqlLen) |(In JSON, denoted by contentLen) | + * +--------------------+---------------------------------+ + */ typedef struct SSubQueryMsg { SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; int8_t taskType; - uint32_t contentLen; + uint32_t sqlLen; // the query sql, + uint32_t phyLen; char msg[]; } SSubQueryMsg; @@ -1141,10 +1159,7 @@ typedef struct SVCreateTbReq { char* name; uint32_t ttl; uint32_t keep; -#define TD_SUPER_TABLE TSDB_SUPER_TABLE -#define TD_CHILD_TABLE TSDB_CHILD_TABLE -#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE - uint8_t type; + uint8_t type; union { struct { tb_uid_t suid; @@ -1189,15 +1204,21 @@ typedef struct { } SVAlterTbRsp; typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - int64_t suid; + uint64_t ver; + char* name; + uint8_t type; + tb_uid_t suid; } SVDropTbReq; typedef struct { - SMsgHead head; + uint64_t ver; } SVDropTbRsp; +int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq); +void* tDeserializeSVDropTbReq(void* buf, SVDropTbReq* pReq); +int32_t tSerializeSVDropTbRsp(void** buf, SVDropTbRsp* pRsp); +void* tDeserializeSVDropTbRsp(void* buf, SVDropTbRsp* pRsp); + typedef struct { SMsgHead head; int64_t uid; @@ -1342,9 +1363,8 @@ static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBat } typedef struct { - int32_t keyLen; + int32_t key; int32_t valueLen; - void* key; void* value; } SKv; @@ -1366,8 +1386,7 @@ typedef struct { typedef struct { SClientHbKey connKey; int32_t status; - int32_t bodyLen; - void* body; + SArray* info; // Array } SClientHbRsp; typedef struct { @@ -1386,9 +1405,26 @@ void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); + +static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { + void *pIter = taosHashIterate(info, NULL); + while (pIter != NULL) { + SKv* kv = (SKv*)pIter; + + tfree(kv->value); + + pIter = taosHashIterate(info, pIter); + } +} + + static FORCE_INLINE void tFreeClientHbReq(void *pReq) { SClientHbReq* req = (SClientHbReq*)pReq; - if (req->info) taosHashCleanup(req->info); + if (req->info) { + tFreeReqKvHash(req->info); + + taosHashCleanup(req->info); + } } int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); @@ -1404,22 +1440,39 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { free(pReq); } +static FORCE_INLINE void tFreeClientKv(void *pKv) { + SKv *kv = (SKv *)pKv; + if (kv) { + tfree(kv->value); + } +} + +static FORCE_INLINE void tFreeClientHbRsp(void *pRsp) { + SClientHbRsp* rsp = (SClientHbRsp*)pRsp; + if (rsp->info) taosArrayDestroyEx(rsp->info, tFreeClientKv); +} + + +static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { + SClientHbBatchRsp *rsp = (SClientHbBatchRsp*)pRsp; + taosArrayDestroyEx(rsp->rsps, tFreeClientHbRsp); +} + + int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKv->keyLen); + tlen += taosEncodeFixedI32(buf, pKv->key); tlen += taosEncodeFixedI32(buf, pKv->valueLen); - tlen += taosEncodeBinary(buf, pKv->key, pKv->keyLen); tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen); return tlen; } static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) { - buf = taosDecodeFixedI32(buf, &pKv->keyLen); + buf = taosDecodeFixedI32(buf, &pKv->key); buf = taosDecodeFixedI32(buf, &pKv->valueLen); - buf = taosDecodeBinary(buf, &pKv->key, pKv->keyLen); buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen); return buf; } @@ -1525,6 +1578,7 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { } typedef struct SMqSetCVgReq { + int64_t leftForVer; int32_t vgId; int64_t oldConsumerId; int64_t newConsumerId; @@ -1533,9 +1587,7 @@ typedef struct SMqSetCVgReq { char* sql; char* logicalPlan; char* physicalPlan; - uint32_t qmsgLen; - void* qmsg; - //SSubQueryMsg msg; + char* qmsg; } SMqSetCVgReq; static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { @@ -1543,7 +1595,8 @@ static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* tlen += taosEncodeFixedU64(buf, pMsg->sId); tlen += taosEncodeFixedU64(buf, pMsg->queryId); tlen += taosEncodeFixedU64(buf, pMsg->taskId); - tlen += taosEncodeFixedU32(buf, pMsg->contentLen); + tlen += taosEncodeFixedU32(buf, pMsg->sqlLen); + tlen += taosEncodeFixedU32(buf, pMsg->phyLen); //tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); return tlen; } @@ -1552,13 +1605,15 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { buf = taosDecodeFixedU64(buf, &pMsg->sId); buf = taosDecodeFixedU64(buf, &pMsg->queryId); buf = taosDecodeFixedU64(buf, &pMsg->taskId); - buf = taosDecodeFixedU32(buf, &pMsg->contentLen); + buf = taosDecodeFixedU32(buf, &pMsg->sqlLen); + buf = taosDecodeFixedU32(buf, &pMsg->phyLen); //buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); return buf; } static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->leftForVer); tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); @@ -1567,13 +1622,13 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += taosEncodeFixedU32(buf, pReq->qmsgLen); - tlen += taosEncodeString(buf, (char*)pReq->qmsg); + tlen += taosEncodeString(buf, pReq->qmsg); //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->leftForVer); buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); @@ -1582,8 +1637,7 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = taosDecodeFixedU32(buf, &pReq->qmsgLen); - buf = taosDecodeString(buf, (char**)&pReq->qmsg); + buf = taosDecodeString(buf, &pReq->qmsg); //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } @@ -1596,41 +1650,80 @@ typedef struct SMqSetCVgRsp { char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; -typedef struct SMqColData { - int16_t colId; - int16_t type; - int16_t bytes; - char data[]; -} SMqColData; +typedef struct { + uint32_t nCols; + SSchema *pSchema; +} SSchemaWrapper; + +static FORCE_INLINE int32_t tEncodeSSchema(void** buf, const SSchema* pSchema) { + int32_t tlen = 0; + tlen += taosEncodeFixedI8(buf, pSchema->type); + tlen += taosEncodeFixedI32(buf, pSchema->bytes); + tlen += taosEncodeFixedI32(buf, pSchema->colId); + tlen += taosEncodeString(buf, pSchema->name); + return tlen; +} + +static FORCE_INLINE void* tDecodeSSchema(void* buf, SSchema* pSchema) { + buf = taosDecodeFixedI8(buf, &pSchema->type); + buf = taosDecodeFixedI32(buf, &pSchema->bytes); + buf = taosDecodeFixedI32(buf, &pSchema->colId); + buf = taosDecodeStringTo(buf, pSchema->name); + return buf; +} + +static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { + int32_t tlen = 0; + tlen += taosEncodeFixedU32(buf, pSW->nCols); + for (int32_t i = 0; i < pSW->nCols; i ++) { + tlen += tEncodeSSchema(buf, &pSW->pSchema[i]); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { + buf = taosDecodeFixedU32(buf, &pSW->nCols); + pSW->pSchema = (SSchema*) calloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + return NULL; + } + for (int32_t i = 0; i < pSW->nCols; i ++) { + buf = tDecodeSSchema(buf, &pSW->pSchema[i]); + } + return buf; +} typedef struct SMqTbData { int64_t uid; - int32_t numOfCols; int32_t numOfRows; - SMqColData colData[]; + char* colData; } SMqTbData; typedef struct SMqTopicBlk { - char topicName[TSDB_TOPIC_FNAME_LEN]; - int64_t committedOffset; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t bodyLen; - int32_t numOfTb; - SMqTbData tbData[]; + char topicName[TSDB_TOPIC_FNAME_LEN]; + int64_t committedOffset; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t bodyLen; + int32_t numOfTb; + SMqTbData* tbData; } SMqTopicData; typedef struct SMqConsumeRsp { - int64_t reqId; - int64_t consumerId; - int32_t bodyLen; - int32_t numOfTopics; - SMqTopicData data[]; + int64_t consumerId; + SSchemaWrapper* schemas; + int64_t committedOffset; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t numOfTopics; + SArray* pBlockData; //SArray } SMqConsumeRsp; // one req for one vg+topic typedef struct SMqConsumeReq { + SMsgHead head; //0: commit only, current offset //1: consume only, poll next offset //2: commit current and consume next offset @@ -1657,13 +1750,18 @@ typedef struct SMqSubTopicEp { typedef struct SMqCMGetSubEpRsp { int64_t consumerId; + int64_t epoch; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray } SMqCMGetSubEpRsp; +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + taosArrayDestroy(pSubTopicEp->vgs); +} + static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; - tlen += taosEncodeFixedI16(buf, pVgEp->vgId); + tlen += taosEncodeFixedI32(buf, pVgEp->vgId); tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } @@ -1674,6 +1772,10 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { return buf; } +static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) { + taosArrayDestroyEx(pRsp->topics, (void (*)(void*)) tDeleteSMqSubTopicEp); +} + static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { int32_t tlen = 0; tlen += taosEncodeString(buf, pTopicEp->topic); @@ -1705,6 +1807,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += taosEncodeFixedI64(buf, pRsp->epoch); tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -1717,6 +1820,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + buf = taosDecodeFixedI64(buf, &pRsp->epoch); buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 90926da120..98903f5617 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -207,6 +207,77 @@ #define TK_INTO 189 #define TK_VALUES 190 +#define NEW_TK_OR 1 +#define NEW_TK_AND 2 +#define NEW_TK_UNION 3 +#define NEW_TK_ALL 4 +#define NEW_TK_MINUS 5 +#define NEW_TK_EXCEPT 6 +#define NEW_TK_INTERSECT 7 +#define NEW_TK_NK_PLUS 8 +#define NEW_TK_NK_MINUS 9 +#define NEW_TK_NK_STAR 10 +#define NEW_TK_NK_SLASH 11 +#define NEW_TK_NK_REM 12 +#define NEW_TK_SHOW 13 +#define NEW_TK_DATABASES 14 +#define NEW_TK_NK_INTEGER 15 +#define NEW_TK_NK_FLOAT 16 +#define NEW_TK_NK_STRING 17 +#define NEW_TK_NK_BOOL 18 +#define NEW_TK_TIMESTAMP 19 +#define NEW_TK_NK_VARIABLE 20 +#define NEW_TK_NK_COMMA 21 +#define NEW_TK_NK_ID 22 +#define NEW_TK_NK_LP 23 +#define NEW_TK_NK_RP 24 +#define NEW_TK_NK_DOT 25 +#define NEW_TK_BETWEEN 26 +#define NEW_TK_NOT 27 +#define NEW_TK_IS 28 +#define NEW_TK_NULL 29 +#define NEW_TK_NK_LT 30 +#define NEW_TK_NK_GT 31 +#define NEW_TK_NK_LE 32 +#define NEW_TK_NK_GE 33 +#define NEW_TK_NK_NE 34 +#define NEW_TK_NK_EQ 35 +#define NEW_TK_LIKE 36 +#define NEW_TK_MATCH 37 +#define NEW_TK_NMATCH 38 +#define NEW_TK_IN 39 +#define NEW_TK_FROM 40 +#define NEW_TK_AS 41 +#define NEW_TK_JOIN 42 +#define NEW_TK_ON 43 +#define NEW_TK_INNER 44 +#define NEW_TK_SELECT 45 +#define NEW_TK_DISTINCT 46 +#define NEW_TK_WHERE 47 +#define NEW_TK_PARTITION 48 +#define NEW_TK_BY 49 +#define NEW_TK_SESSION 50 +#define NEW_TK_STATE_WINDOW 51 +#define NEW_TK_INTERVAL 52 +#define NEW_TK_SLIDING 53 +#define NEW_TK_FILL 54 +#define NEW_TK_VALUE 55 +#define NEW_TK_NONE 56 +#define NEW_TK_PREV 57 +#define NEW_TK_LINEAR 58 +#define NEW_TK_NEXT 59 +#define NEW_TK_GROUP 60 +#define NEW_TK_HAVING 61 +#define NEW_TK_ORDER 62 +#define NEW_TK_SLIMIT 63 +#define NEW_TK_SOFFSET 64 +#define NEW_TK_LIMIT 65 +#define NEW_TK_OFFSET 66 +#define NEW_TK_ASC 67 +#define NEW_TK_DESC 68 +#define NEW_TK_NULLS 69 +#define NEW_TK_FIRST 70 +#define NEW_TK_LAST 71 #define TK_SPACE 300 #define TK_COMMENT 301 @@ -217,6 +288,8 @@ #define TK_FILE 306 #define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query +#define TK_NIL 65535 + #endif diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 70cff7ed1a..c291ebd8fd 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -49,16 +49,19 @@ typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; uint32_t dbRentSec; - uint32_t stableRentSec; + uint32_t stbRentSec; } SCatalogCfg; typedef struct SSTableMetaVersion { + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; uint64_t suid; int16_t sversion; int16_t tversion; } SSTableMetaVersion; typedef struct SDbVgVersion { + char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; } SDbVgVersion; @@ -98,6 +101,10 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); +int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId); + +int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid); + /** * Get a table's meta data. * @param pCatalog (input, got with catalogGetHandle) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index d2e602a5d6..d9e1957e5d 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -24,8 +24,13 @@ extern "C" { typedef void* qTaskInfo_t; typedef void* DataSinkHandle; +struct SRpcMsg; struct SSubplan; +typedef struct SReadHandle { + void* reader; + void* meta; +} SReadHandle; /** * Create the exec task for streaming mode * @param pMsg @@ -34,7 +39,13 @@ struct SSubplan; */ qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); -int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); +/** + * + * @param tinfo + * @param input + * @return + */ +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); /** * Create the exec task object according to task json @@ -45,7 +56,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); * @param qId * @return */ -int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, @@ -208,6 +219,8 @@ void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle); */ void** qDeregisterQInfo(void* pMgmt, void* pQInfo); +void qProcessFetchRsp(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); + #ifdef __cplusplus } #endif diff --git a/include/libs/function/function.h b/include/libs/function/function.h index b15d5ad33a..aef5f7fec4 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -107,14 +107,14 @@ typedef struct SPoint1 { union{double val; char* ptr;}; } SPoint1; -struct SQLFunctionCtx; +struct SqlFunctionCtx; struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SExtTagsInfo { int16_t tagsLen; // keep the tags data for top/bottom query result int16_t numOfTagCols; - struct SQLFunctionCtx **pTagCtxList; + struct SqlFunctionCtx **pTagCtxList; } SExtTagsInfo; typedef struct SResultDataInfo { @@ -126,18 +126,18 @@ typedef struct SResultDataInfo { #define GET_RES_INFO(ctx) ((ctx)->resultInfo) typedef struct SFunctionFpSet { - bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(struct SQLFunctionCtx *pCtx); + bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment + void (*addInput)(struct SqlFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(struct SQLFunctionCtx *pCtx); - void (*combine)(struct SQLFunctionCtx *pCtx); + void (*finalize)(struct SqlFunctionCtx *pCtx); + void (*combine)(struct SqlFunctionCtx *pCtx); } SFunctionFpSet; extern SFunctionFpSet fpSet[1]; // sql function runtime context -typedef struct SQLFunctionCtx { +typedef struct SqlFunctionCtx { int32_t size; // number of rows void * pInput; // input data buffer uint32_t order; // asc|desc @@ -167,7 +167,7 @@ typedef struct SQLFunctionCtx { int32_t columnIndex; SFunctionFpSet* fpSet; -} SQLFunctionCtx; +} SqlFunctionCtx; enum { TEXPR_NODE_DUMMY = 0x0, @@ -216,14 +216,14 @@ typedef struct SAggFunctionInfo { int8_t sFunctionId; // Transfer function for super table query uint16_t status; - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SQLFunctionCtx *pCtx); + bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment + void (*addInput)(SqlFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(SQLFunctionCtx *pCtx); - void (*combine)(SQLFunctionCtx *pCtx); + void (*finalize)(SqlFunctionCtx *pCtx); + void (*combine)(SqlFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); + int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; struct SScalarFuncParam; @@ -279,9 +279,9 @@ void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc); tExprNode* exprdup(tExprNode* pTree); -void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num); +void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); -int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num); +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 3c365335be..aba647f5cd 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -22,7 +22,7 @@ extern "C" { #include "nodes.h" -struct SQLFunctionCtx; +struct SqlFunctionCtx; struct SResultRowEntryInfo; struct STimeWindow; @@ -32,9 +32,9 @@ typedef struct SFuncExecEnv { typedef void* FuncMgtHandle; typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv); -typedef bool (*FExecInit)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); -typedef void (*FExecProcess)(struct SQLFunctionCtx *pCtx); -typedef void (*FExecFinalize)(struct SQLFunctionCtx *pCtx); +typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); +typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx); +typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx); typedef struct SFuncExecFuncs { FExecGetEnv getEnv; diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 374ae02e4a..d245d04166 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -35,7 +35,7 @@ typedef struct SQueryNode { int16_t type; } SQueryNode; -#define nodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) +#define queryNodeType(nodeptr) (((const SQueryNode*)(nodeptr))->type) typedef struct SField { char name[TSDB_COL_NAME_LEN]; diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 42d3307ac8..31f5457c90 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -24,7 +24,6 @@ #endif OP_ENUM_MACRO(StreamScan) -OP_ENUM_MACRO(TableScan) OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) @@ -48,3 +47,5 @@ OP_ENUM_MACRO(AllTimeWindow) OP_ENUM_MACRO(AllMultiTableTimeInterval) OP_ENUM_MACRO(Order) OP_ENUM_MACRO(Exchange) + +//OP_ENUM_MACRO(TableScan) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 1925f0e3bd..2e4093590d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -81,20 +81,19 @@ typedef struct STableMeta { } STableMeta; typedef struct SDBVgroupInfo { - SRWLatch lock; - int64_t dbId; + uint64_t dbId; int32_t vgVersion; int8_t hashMethod; - SHashObj *vgInfo; //key:vgId, value:SVgroupInfo + SHashObj *vgHash; //key:vgId, value:SVgroupInfo } SDBVgroupInfo; typedef struct SUseDbOutput { - char db[TSDB_DB_FNAME_LEN]; - SDBVgroupInfo dbVgroup; + char db[TSDB_DB_FNAME_LEN]; + SDBVgroupInfo *dbVgroup; } SUseDbOutput; enum { - META_TYPE_NON_TABLE = 1, + META_TYPE_NULL_TABLE = 1, META_TYPE_CTABLE, META_TYPE_TABLE, META_TYPE_BOTH_TABLE @@ -103,8 +102,9 @@ enum { typedef struct STableMetaOutput { int32_t metaType; - char ctbFname[TSDB_TABLE_FNAME_LEN]; - char tbFname[TSDB_TABLE_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + char ctbName[TSDB_TABLE_NAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; SCTableMeta ctbMeta; STableMeta *tbMeta; } STableMetaOutput; @@ -128,20 +128,9 @@ typedef struct SMsgSendInfo { typedef struct SQueryNodeAddr { int32_t nodeId; // vgId or qnodeId - int8_t inUse; - int8_t numOfEps; - SEpAddr epAddr[TSDB_MAX_REPLICA]; + SEpSet epset; } SQueryNodeAddr; -static FORCE_INLINE void tConvertQueryAddrToEpSet(SEpSet* pEpSet, const SQueryNodeAddr* pAddr) { - pEpSet->inUse = pAddr->inUse; - pEpSet->numOfEps = pAddr->numOfEps; - for (int j = 0; j < TSDB_MAX_REPLICA; j++) { - pEpSet->port[j] = pAddr->epAddr[j].port; - memcpy(pEpSet->fqdn[j], pAddr->epAddr[j].fqdn, TSDB_FQDN_LEN); - } -} - int32_t initTaskQueue(); int32_t cleanupTaskQueue(); @@ -174,7 +163,7 @@ extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSi extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize); -#define SET_META_TYPE_NONE(t) (t) = META_TYPE_NON_TABLE +#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE #define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE #define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE #define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 5d815d15e0..5e3320ffdb 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -49,9 +49,10 @@ typedef struct { } SQWorkerStat; typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); +typedef int32_t (*sendReqToDnodeFp)(void *, struct SEpSet *, struct SRpcMsg *); - -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp); +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, + putReqToQueryQFp fp1, sendReqToDnodeFp fp2); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); @@ -65,6 +66,8 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); +int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 5a0da46490..e856adaf31 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -80,7 +80,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob); +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob); /** * Fetch query result from the remote query executor diff --git a/include/libs/transport/transport.h b/include/libs/transport/transport.h index a05a76931a..f5ffc125ea 100644 --- a/include/libs/transport/transport.h +++ b/include/libs/transport/transport.h @@ -20,21 +20,6 @@ extern "C" { #endif -//typedef struct SEpAddr { -// char fqdn[TSDB_FQDN_LEN]; -// uint16_t port; -//} SEpAddr; -// -//typedef struct SVgroup { -// int32_t vgId; -// int8_t numOfEps; -// SEpAddr epAddr[TSDB_MAX_REPLICA]; -//} SVgroup; -// -//typedef struct SVgroupsInfo { -// int32_t numOfVgroups; -// SVgroup vgroups[]; -//} SVgroupsInfo; #ifdef __cplusplus } diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 641b485f4c..45f1d88c30 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -19,6 +19,7 @@ #include "tarray.h" #include "tdef.h" #include "tlog.h" +#include "tmsg.h" #ifdef __cplusplus extern "C" { #endif @@ -159,7 +160,7 @@ int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); // write -int64_t walWrite(SWal *, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen); +int64_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen); void walFsync(SWal *, bool force); // apis for lifecycle management diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index e6b41adf14..ccb135aa0d 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -20,9 +20,21 @@ extern "C" { #endif -#include "tarray.h" #include "tdef.h" +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) + +#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) + +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) + +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) + typedef enum ENodeType { QUERY_NODE_COLUMN = 1, QUERY_NODE_VALUE, @@ -35,12 +47,16 @@ typedef enum ENodeType { QUERY_NODE_JOIN_TABLE, QUERY_NODE_GROUPING_SET, QUERY_NODE_ORDER_BY_EXPR, + QUERY_NODE_LIMIT, QUERY_NODE_STATE_WINDOW, QUERY_NODE_SESSION_WINDOW, QUERY_NODE_INTERVAL_WINDOW, + QUERY_NODE_NODE_LIST, + QUERY_NODE_FILL, QUERY_NODE_SET_OPERATOR, - QUERY_NODE_SELECT_STMT + QUERY_NODE_SELECT_STMT, + QUERY_NODE_SHOW_STMT } ENodeType; /** @@ -51,7 +67,21 @@ typedef struct SNode { ENodeType type; } SNode; -#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +typedef struct SListCell { + SNode* pNode; + struct SListCell* pNext; +} SListCell; + +typedef struct SNodeList { + int16_t length; + SListCell* pHead; + SListCell* pTail; +} SNodeList; + +typedef struct SNameStr { + int32_t len; + char* pName; +} SNameStr; typedef struct SDataType { uint8_t type; @@ -64,6 +94,7 @@ typedef struct SExprNode { ENodeType nodeType; SDataType resType; char aliasName[TSDB_COL_NAME_LEN]; + SNodeList* pAssociationList; } SExprNode; typedef enum EColumnType { @@ -77,7 +108,9 @@ typedef struct SColumnNode { EColumnType colType; // column or tag char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; + SNode* pProjectRef; } SColumnNode; typedef struct SValueNode { @@ -128,35 +161,43 @@ typedef enum ELogicConditionType { typedef struct SLogicConditionNode { ENodeType type; // QUERY_NODE_LOGIC_CONDITION ELogicConditionType condType; - SArray* pParameterList; + SNodeList* pParameterList; } SLogicConditionNode; typedef struct SIsNullCondNode { ENodeType type; // QUERY_NODE_IS_NULL_CONDITION SNode* pExpr; - bool isNot; + bool isNull; } SIsNullCondNode; +typedef struct SNodeListNode { + ENodeType type; // QUERY_NODE_NODE_LIST + SNodeList* pNodeList; +} SNodeListNode; + typedef struct SFunctionNode { SExprNode type; // QUERY_NODE_FUNCTION char functionName[TSDB_FUNC_NAME_LEN]; int32_t funcId; - SArray* pParameterList; // SNode + SNodeList* pParameterList; } SFunctionNode; typedef struct STableNode { ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; - char tableAliasName[TSDB_COL_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; } STableNode; +struct STableMeta; + typedef struct SRealTableNode { - STableNode type; // QUERY_NODE_REAL_TABLE - char dbName[TSDB_DB_NAME_LEN]; + STableNode table; // QUERY_NODE_REAL_TABLE + struct STableMeta* pMeta; } SRealTableNode; typedef struct STempTableNode { - STableNode type; // QUERY_NODE_TEMP_TABLE + STableNode table; // QUERY_NODE_TEMP_TABLE SNode* pSubquery; } STempTableNode; @@ -165,7 +206,7 @@ typedef enum EJoinType { } EJoinType; typedef struct SJoinTableNode { - STableNode type; // QUERY_NODE_JOIN_TABLE + STableNode table; // QUERY_NODE_JOIN_TABLE EJoinType joinType; SNode* pLeft; SNode* pRight; @@ -179,7 +220,7 @@ typedef enum EGroupingSetType { typedef struct SGroupingSetNode { ENodeType type; // QUERY_NODE_GROUPING_SET EGroupingSetType groupingSetType; - SArray* pParameterList; + SNodeList* pParameterList; } SGroupingSetNode; typedef enum EOrder { @@ -188,7 +229,8 @@ typedef enum EOrder { } EOrder; typedef enum ENullOrder { - NULL_ORDER_FIRST = 1, + NULL_ORDER_DEFAULT = 1, + NULL_ORDER_FIRST, NULL_ORDER_LAST } ENullOrder; @@ -199,10 +241,11 @@ typedef struct SOrderByExprNode { ENullOrder nullOrder; } SOrderByExprNode; -typedef struct SLimitInfo { +typedef struct SLimitNode { + ENodeType type; // QUERY_NODE_LIMIT uint64_t limit; uint64_t offset; -} SLimitInfo; +} SLimitNode; typedef struct SStateWindowNode { ENodeType type; // QUERY_NODE_STATE_WINDOW @@ -217,23 +260,40 @@ typedef struct SSessionWindowNode { typedef struct SIntervalWindowNode { ENodeType type; // QUERY_NODE_INTERVAL_WINDOW - int64_t interval; - int64_t sliding; - int64_t offset; + SNode* pInterval; // SValueNode + SNode* pOffset; // SValueNode + SNode* pSliding; // SValueNode + SNode* pFill; } SIntervalWindowNode; +typedef enum EFillMode { + FILL_MODE_NONE = 1, + FILL_MODE_VALUE, + FILL_MODE_PREV, + FILL_MODE_NULL, + FILL_MODE_LINEAR, + FILL_MODE_NEXT +} EFillMode; + +typedef struct SFillNode { + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode +} SFillNode; + typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; - SArray* pProjectionList; // SNode + SNodeList* pProjectionList; // SNode SNode* pFromTable; - SNode* pWhereCond; - SArray* pPartitionByList; // SNode - SNode* pWindowClause; - SArray* pGroupByList; // SGroupingSetNode - SArray* pOrderByList; // SOrderByExprNode - SLimitInfo limit; - SLimitInfo slimit; + SNode* pWhere; + SNodeList* pPartitionByList; // SNode + SNode* pWindow; + SNodeList* pGroupByList; // SGroupingSetNode + SNode* pHaving; + SNodeList* pOrderByList; // SOrderByExprNode + SNode* pLimit; + SNode* pSlimit; } SSelectStmt; typedef enum ESetOperatorType { @@ -245,24 +305,35 @@ typedef struct SSetOperator { ESetOperatorType opType; SNode* pLeft; SNode* pRight; + SNodeList* pOrderByList; // SOrderByExprNode + SNode* pLimit; } SSetOperator; +SNode* nodesMakeNode(ENodeType type); +void nodesDestroyNode(SNode* pNode); + +SNodeList* nodesMakeList(); +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); +void nodesDestroyList(SNodeList* pList); + typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); -bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext); -bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext); +void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); +void nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); +void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext); +void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext); -bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext); +bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); -bool nodeEqual(const SNode* a, const SNode* b); +bool nodesEqualNode(const SNode* a, const SNode* b); -void cloneNode(const SNode* pNode); +void nodesCloneNode(const SNode* pNode); -int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen); -int32_t stringToNode(const char* pStr, SNode** pNode); +int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); +int32_t nodesStringToNode(const char* pStr, SNode** pNode); -bool isTimeorderQuery(const SNode* pQuery); -bool isTimelineQuery(const SNode* pQuery); +bool nodesIsTimeorderQuery(const SNode* pQuery); +bool nodesIsTimelineQuery(const SNode* pQuery); #ifdef __cplusplus } diff --git a/source/nodes/src/nodesCode.c b/include/nodes/nodesShowStmts.h similarity index 65% rename from source/nodes/src/nodesCode.c rename to include/nodes/nodesShowStmts.h index 7fe919ffe8..312fbbc4f4 100644 --- a/source/nodes/src/nodesCode.c +++ b/include/nodes/nodesShowStmts.h @@ -13,12 +13,26 @@ * along with this program. If not, see . */ +#ifndef _TD_NODES_SHOW_STMTS_H_ +#define _TD_NODES_SHOW_STMTS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + #include "nodes.h" -int32_t nodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { +typedef enum EShowStmtType { + SHOW_TYPE_DATABASE = 1 +} EShowStmtType; +typedef struct SShowStmt { + ENodeType type; // QUERY_NODE_SHOW_STMT + EShowStmtType showType; +} SShowStmt; + +#ifdef __cplusplus } +#endif -int32_t stringToNode(const char* pStr, SNode** pNode) { - -} +#endif /*_TD_NODES_SHOW_STMTS_H_*/ diff --git a/include/os/osEndian.h b/include/os/osEndian.h index e573ba0a75..1f52ece535 100644 --- a/include/os/osEndian.h +++ b/include/os/osEndian.h @@ -30,4 +30,4 @@ static const int32_t endian_test_var = 1; } #endif -#endif /*_TD_OS_ENDIAN_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENDIAN_H_*/ diff --git a/include/os/osMath.h b/include/os/osMath.h index 948fbbf665..3fe46d557e 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -36,25 +36,25 @@ extern "C" { #else - #define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ + #define TSWAP(a, b, c) \ + do { \ + __typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ } while (0) - #define TMAX(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a > __b) ? __a : __b; \ + #define TMAX(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a > __b) ? __a : __b; \ }) - #define TMIN(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a < __b) ? __a : __b; \ + #define TMIN(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a < __b) ? __a : __b; \ }) #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 69bf085491..b5740b0118 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -253,6 +253,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E4) #define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) #define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) +#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) +#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E7) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) @@ -438,7 +440,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) //scheduler status error #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //scheduler internal error - +//parser +#define TSDB_CODE_PARSER_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) //invalid column name +#define TSDB_CODE_PARSER_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) //table not exist +#define TSDB_CODE_PARSER_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) //ambiguous column #ifdef __cplusplus } diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 63ba460d39..cfa5a65c2a 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -15,6 +15,7 @@ #ifndef _TD_UTIL_QUEUE_H #define _TD_UTIL_QUEUE_H +#include "os.h" #ifdef __cplusplus extern "C" { @@ -40,13 +41,13 @@ shall be used to set up the protection. typedef struct STaosQueue STaosQueue; typedef struct STaosQset STaosQset; typedef struct STaosQall STaosQall; -typedef void (*FProcessItem)(void *ahandle, void *pItem); -typedef void (*FProcessItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); +typedef void (*FItem)(void *ahandle, void *pItem); +typedef void (*FItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); -void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp); -void *taosAllocateQitem(int32_t size); +void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); +void * taosAllocateQitem(int32_t size); void taosFreeQitem(void *pItem); int32_t taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); @@ -66,8 +67,11 @@ int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle); void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue); int32_t taosGetQueueNumber(STaosQset *qset); -int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp); -int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp); +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp); +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp); + +int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId); +void taosResetQsetThread(STaosQset *qset, void *pItem); int32_t taosGetQueueItemsNumber(STaosQueue *queue); int32_t taosGetQsetItemsNumber(STaosQset *qset); diff --git a/include/util/tworker.h b/include/util/tworker.h index 27f03bd2b6..771c7c9433 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -15,57 +15,61 @@ #ifndef _TD_UTIL_WORKER_H #define _TD_UTIL_WORKER_H - #include "tqueue.h" #ifdef __cplusplus extern "C" { #endif -typedef struct SWorkerPool SWorkerPool; -typedef struct SMWorkerPool SMWorkerPool; +typedef struct SQWorkerPool SQWorkerPool; +typedef struct SWWorkerPool SWWorkerPool; -typedef struct SWorker { - int32_t id; // worker ID - pthread_t thread; // thread - SWorkerPool *pool; -} SWorker; +typedef struct SQWorker { + int32_t id; // worker ID + pthread_t thread; // thread + SQWorkerPool *pool; +} SQWorker, SFWorker; -typedef struct SWorkerPool { +typedef struct SQWorkerPool { int32_t max; // max number of workers int32_t min; // min number of workers int32_t num; // current number of workers - STaosQset *qset; - const char *name; - SWorker *workers; + STaosQset * qset; + const char * name; + SQWorker * workers; pthread_mutex_t mutex; -} SWorkerPool; +} SQWorkerPool, SFWorkerPool; -typedef struct SMWorker { +typedef struct SWWorker { int32_t id; // worker id pthread_t thread; // thread - STaosQall *qall; - STaosQset *qset; // queue set - SMWorkerPool *pool; -} SMWorker; + STaosQall * qall; + STaosQset * qset; // queue set + SWWorkerPool *pool; +} SWWorker; -typedef struct SMWorkerPool { +typedef struct SWWorkerPool { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic - const char *name; - SMWorker *workers; + const char * name; + SWWorker * workers; pthread_mutex_t mutex; -} SMWorkerPool; +} SWWorkerPool; -int32_t tWorkerInit(SWorkerPool *pool); -void tWorkerCleanup(SWorkerPool *pool); -STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp); -void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue); +int32_t tQWorkerInit(SQWorkerPool *pool); +void tQWorkerCleanup(SQWorkerPool *pool); +STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp); +void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue); -int32_t tMWorkerInit(SMWorkerPool *pool); -void tMWorkerCleanup(SMWorkerPool *pool); -STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp); -void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue); +int32_t tFWorkerInit(SFWorkerPool *pool); +void tFWorkerCleanup(SFWorkerPool *pool); +STaosQueue *tFWorkerAllocQueue(SFWorkerPool *pool, void *ahandle, FItem fp); +void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue); + +int32_t tWWorkerInit(SWWorkerPool *pool); +void tWWorkerCleanup(SWWorkerPool *pool); +STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); +void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue); #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c61f3da6bd..d7574466cd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -30,14 +30,28 @@ extern "C" { #include "tmsgtype.h" #include "trpc.h" #include "query.h" +#include "parser.h" + +#define CHECK_CODE_GOTO(expr, label) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + terrno = code; \ + goto label; \ + } \ + } while (0) #define HEARTBEAT_INTERVAL 1500 // ms typedef struct SAppInstInfo SAppInstInfo; -typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); +typedef struct SHbConnInfo { + void *param; + SClientHbReq *req; +} SHbConnInfo; typedef struct SAppHbMgr { + char *key; // statistics int32_t reportCnt; int32_t connKeyCnt; @@ -49,9 +63,15 @@ typedef struct SAppHbMgr { SAppInstInfo* pAppInstInfo; // info SHashObj* activeInfo; // hash - SHashObj* getInfoFuncs; // hash + SHashObj* connInfo; // hash } SAppHbMgr; + +typedef int32_t (*FHbRspHandle)(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp); + +typedef int32_t (*FHbReqHandle)(SClientHbKey *connKey, void* param, SClientHbReq *req); + + typedef struct SClientHbMgr { int8_t inited; // ctl @@ -59,12 +79,10 @@ typedef struct SClientHbMgr { pthread_t thread; pthread_mutex_t lock; // used when app init and cleanup SArray* appHbMgrs; // SArray one for each cluster - FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; + FHbReqHandle reqHandle[HEARTBEAT_TYPE_MAX]; + FHbRspHandle rspHandle[HEARTBEAT_TYPE_MAX]; } SClientHbMgr; -// TODO: embed param into function -// return type: SArray -typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param); typedef struct SQueryExecMetric { int64_t start; // start timestamp @@ -114,6 +132,7 @@ typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_FNAME_LEN]; + char ver[128]; int32_t acctId; uint32_t connId; int32_t connType; @@ -210,6 +229,11 @@ void *doFetchRow(SRequestObj* pRequest); void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); + +int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); + +int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); + // --- heartbeat // global, called by mgmt int hbMgrInit(); @@ -217,11 +241,11 @@ void hbMgrCleanUp(); int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo); -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); +void appHbMgrCleanup(void); // conn level -int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func); +int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 28faa76785..cb73701bfa 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -127,6 +127,8 @@ void* openTransporter(const char *user, const char *auth, int32_t numOfThread) { void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; + SClientHbKey connKey = {.connId = pTscObj->connId, .hbType = pTscObj->connType}; + hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); tscDebug("connObj 0x%"PRIx64" destroyed, totalConn:%"PRId64, pTscObj->id, pTscObj->pAppInfo->numOfConns); pthread_mutex_destroy(&pTscObj->mutex); @@ -517,4 +519,4 @@ setConfRet taos_set_config(const char *config){ pthread_mutex_unlock(&setConfMutex); return ret; } -#endif \ No newline at end of file +#endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 0f4ff6f725..d265ffaa94 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -15,33 +15,352 @@ #include "clientInt.h" #include "trpc.h" +#include "catalog.h" +#include "clientLog.h" static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbRspHandle(SClientHbRsp* pRsp) { +static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) { return 0; } -static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) { - if (code != 0) { - return -1; +static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t msgLen = 0; + int32_t code = 0; + + while (msgLen < valueLen) { + SUseDbRsp *rsp = (SUseDbRsp *)((char *)value + msgLen); + + rsp->vgVersion = ntohl(rsp->vgVersion); + rsp->vgNum = ntohl(rsp->vgNum); + rsp->uid = be64toh(rsp->uid); + + tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid); + + if (rsp->vgVersion < 0) { + code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); + } else { + SDBVgroupInfo vgInfo = {0}; + vgInfo.dbId = rsp->uid; + vgInfo.vgVersion = rsp->vgVersion; + vgInfo.hashMethod = rsp->hashMethod; + vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == vgInfo.vgHash) { + tscError("hash init[%d] failed", rsp->vgNum); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < rsp->vgNum; ++i) { + rsp->vgroupInfo[i].vgId = ntohl(rsp->vgroupInfo[i].vgId); + rsp->vgroupInfo[i].hashBegin = ntohl(rsp->vgroupInfo[i].hashBegin); + rsp->vgroupInfo[i].hashEnd = ntohl(rsp->vgroupInfo[i].hashEnd); + + for (int32_t n = 0; n < rsp->vgroupInfo[i].epset.numOfEps; ++n) { + rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); + } + + if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + tscError("hash push failed, errno:%d", errno); + taosHashCleanup(vgInfo.vgHash); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + code = catalogUpdateDBVgroup(pCatalog, rsp->db, &vgInfo); + if (code) { + taosHashCleanup(vgInfo.vgHash); + } + } + + if (code) { + return code; + } + + msgLen += sizeof(SUseDbRsp) + rsp->vgNum * sizeof(SVgroupInfo); } - SClientHbRsp* pRsp = (SClientHbRsp*) pMsg->pData; - return hbMqHbRspHandle(pRsp); + + return TSDB_CODE_SUCCESS; } -void hbMgrInitMqHbRspHandle() { - clientHbMgr.handle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle; +static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t msgLen = 0; + int32_t code = 0; + int32_t schemaNum = 0; + + while (msgLen < valueLen) { + STableMetaRsp *rsp = (STableMetaRsp *)((char *)value + msgLen); + + rsp->numOfColumns = ntohl(rsp->numOfColumns); + rsp->suid = be64toh(rsp->suid); + + if (rsp->numOfColumns < 0) { + schemaNum = 0; + + tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); + + code = catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid); + } else { + rsp->numOfTags = ntohl(rsp->numOfTags); + + schemaNum = rsp->numOfColumns + rsp->numOfTags; +/* + rsp->vgNum = ntohl(rsp->vgNum); + rsp->uid = be64toh(rsp->uid); + + SDBVgroupInfo vgInfo = {0}; + vgInfo.dbId = rsp->uid; + vgInfo.vgVersion = rsp->vgVersion; + vgInfo.hashMethod = rsp->hashMethod; + vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == vgInfo.vgHash) { + tscError("hash init[%d] failed", rsp->vgNum); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < rsp->vgNum; ++i) { + rsp->vgroupInfo[i].vgId = ntohl(rsp->vgroupInfo[i].vgId); + rsp->vgroupInfo[i].hashBegin = ntohl(rsp->vgroupInfo[i].hashBegin); + rsp->vgroupInfo[i].hashEnd = ntohl(rsp->vgroupInfo[i].hashEnd); + + for (int32_t n = 0; n < rsp->vgroupInfo[i].epset.numOfEps; ++n) { + rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); + } + + if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + tscError("hash push failed, errno:%d", errno); + taosHashCleanup(vgInfo.vgHash); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + code = catalogUpdateDBVgroup(pCatalog, rsp->db, &vgInfo); + if (code) { + taosHashCleanup(vgInfo.vgHash); + } +*/ + } + + if (code) { + return code; + } + + msgLen += sizeof(STableMetaRsp) + schemaNum * sizeof(SSchema); + } + + return TSDB_CODE_SUCCESS; +} + + +static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) { + SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); + if (NULL == info) { + tscWarn("fail to get connInfo, may be dropped, connId:%d, type:%d", pRsp->connKey.connId, pRsp->connKey.hbType); + return TSDB_CODE_SUCCESS; + } + + int32_t kvNum = pRsp->info ? taosArrayGetSize(pRsp->info) : 0; + + tscDebug("hb got %d rsp kv", kvNum); + + for (int32_t i = 0; i < kvNum; ++i) { + SKv *kv = taosArrayGet(pRsp->info, i); + switch (kv->key) { + case HEARTBEAT_KEY_DBINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } + case HEARTBEAT_KEY_STBINFO:{ + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } + default: + tscError("invalid hb key type:%d", kv->key); + break; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) { + static int32_t emptyRspNum = 0; + if (code != 0) { + tfree(param); + return -1; + } + char *key = (char *)param; + SClientHbBatchRsp pRsp = {0}; + tDeserializeSClientHbBatchRsp(pMsg->pData, &pRsp); + + int32_t rspNum = taosArrayGetSize(pRsp.rsps); + + SAppInstInfo** pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + if (pInst == NULL || NULL == *pInst) { + tscError("cluster not exist, key:%s", key); + tfree(param); + tFreeClientHbBatchRsp(&pRsp); + return -1; + } + + tfree(param); + + if (rspNum) { + tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); + } else { + atomic_add_fetch_32(&emptyRspNum, 1); + } + + for (int32_t i = 0; i < rspNum; ++i) { + SClientHbRsp* rsp = taosArrayGet(pRsp.rsps, i); + code = (*clientHbMgr.rspHandle[rsp->connKey.hbType])((*pInst)->pAppHbMgr, rsp); + if (code) { + break; + } + } + + tFreeClientHbBatchRsp(&pRsp); + + return code; +} + +int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SDbVgVersion *dbs = NULL; + uint32_t dbNum = 0; + int32_t code = 0; + + code = catalogGetExpiredDBs(pCatalog, &dbs, &dbNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (dbNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < dbNum; ++i) { + SDbVgVersion *db = &dbs[i]; + db->dbId = htobe64(db->dbId); + db->vgVersion = htonl(db->vgVersion); + } + + SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = sizeof(SDbVgVersion) * dbNum, .value = dbs}; + + tscDebug("hb got %d expired db, valueLen:%d", dbNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + +int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SSTableMetaVersion *stbs = NULL; + uint32_t stbNum = 0; + int32_t code = 0; + + code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (stbNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < stbNum; ++i) { + SSTableMetaVersion *stb = &stbs[i]; + stb->suid = htobe64(stb->suid); + stb->sversion = htons(stb->sversion); + stb->tversion = htons(stb->tversion); + } + + SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; + + tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) { + int64_t *clusterId = (int64_t *)param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + return code; + } + + code = hbGetExpiredDBInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + code = hbGetExpiredStbInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + + return TSDB_CODE_SUCCESS; +} + +int32_t hbMqHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) { + +} + +void hbMgrInitMqHbHandle() { + clientHbMgr.reqHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbReqHandle; + clientHbMgr.reqHandle[HEARTBEAT_TYPE_MQ] = hbMqHbReqHandle; + clientHbMgr.rspHandle[HEARTBEAT_TYPE_QUERY] = hbQueryHbRspHandle; + clientHbMgr.rspHandle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle; } static FORCE_INLINE void hbMgrInitHandle() { // init all handle - hbMgrInitMqHbRspHandle(); + hbMgrInitMqHbHandle(); } +void hbFreeReq(void *req) { + SClientHbReq *pReq = (SClientHbReq *)req; + tFreeReqKvHash(pReq->info); +} + + + SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq* pBatchReq = malloc(sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { @@ -51,38 +370,58 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); + int32_t code = 0; void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); while (pIter != NULL) { SClientHbReq* pOneReq = pIter; + + SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pOneReq->connKey, sizeof(SClientHbKey)); + if (info) { + code = (*clientHbMgr.reqHandle[pOneReq->connKey.hbType])(&pOneReq->connKey, info->param, pOneReq); + if (code) { + taosHashCancelIterate(pAppHbMgr->activeInfo, pIter); + break; + } + } + taosArrayPush(pBatchReq->reqs, pOneReq); - taosHashClear(pOneReq->info); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); } -#if 0 - pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, NULL); - while (pIter != NULL) { - FGetConnInfo getConnInfoFp = (FGetConnInfo)pIter; - SClientHbKey connKey; - taosHashCopyKey(pIter, &connKey); - SArray* pArray = getConnInfoFp(connKey, NULL); - - pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, pIter); + if (code) { + taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq); + tfree(pBatchReq); } -#endif return pBatchReq; } + +void hbClearReqInfo(SAppHbMgr *pAppHbMgr) { + void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); + while (pIter != NULL) { + SClientHbReq* pOneReq = pIter; + + tFreeReqKvHash(pOneReq->info); + taosHashClear(pOneReq->info); + + pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + } +} + + + static void* hbThreadFunc(void* param) { setThreadName("hb"); while (1) { - int8_t threadStop = atomic_load_8(&clientHbMgr.threadStop); - if(threadStop) { + int8_t threadStop = atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 1, 2); + if(1 == threadStop) { break; } + pthread_mutex_lock(&clientHbMgr.lock); + int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for(int i = 0; i < sz; i++) { SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); @@ -98,7 +437,9 @@ static void* hbThreadFunc(void* param) { int tlen = tSerializeSClientHbBatchReq(NULL, pReq); void *buf = malloc(tlen); if (buf == NULL) { - //TODO: error handling + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tFreeClientHbBatchReq(pReq, false); + hbClearReqInfo(pAppHbMgr); break; } void *abuf = buf; @@ -107,6 +448,7 @@ static void* hbThreadFunc(void* param) { if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq, false); + hbClearReqInfo(pAppHbMgr); free(buf); break; } @@ -114,7 +456,7 @@ static void* hbThreadFunc(void* param) { pInfo->msgInfo.pData = buf; pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; - pInfo->param = NULL; + pInfo->param = strdup(pAppHbMgr->key); pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; @@ -122,10 +464,14 @@ static void* hbThreadFunc(void* param) { int64_t transporterId = 0; SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); - tFreeClientHbBatchReq(pReq, false); + tFreeClientHbBatchReq(pReq, false); + hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); } + + pthread_mutex_unlock(&clientHbMgr.lock); + taosMsleep(HEARTBEAT_INTERVAL); } return NULL; @@ -145,10 +491,19 @@ static int32_t hbCreateThread() { } static void hbStopThread() { - atomic_store_8(&clientHbMgr.threadStop, 1); + if (atomic_val_compare_exchange_8(&clientHbMgr.threadStop, 0, 1)) { + tscDebug("hb thread already stopped"); + return; + } + + while (2 != atomic_load_8(&clientHbMgr.threadStop)) { + usleep(10); + } + + tscDebug("hb thread stopped"); } -SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) { +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { hbMgrInit(); SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { @@ -160,6 +515,7 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) { pAppHbMgr->connKeyCnt = 0; pAppHbMgr->reportCnt = 0; pAppHbMgr->reportBytes = 0; + pAppHbMgr->key = strdup(key); // init app info pAppHbMgr->pAppInstInfo = pAppInstInfo; @@ -174,28 +530,31 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) { } pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq; // init getInfoFunc - pAppHbMgr->getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - if (pAppHbMgr->getInfoFuncs == NULL) { + if (pAppHbMgr->connInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; free(pAppHbMgr); return NULL; } + pthread_mutex_lock(&clientHbMgr.lock); taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); + pthread_mutex_unlock(&clientHbMgr.lock); + return pAppHbMgr; } -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { +void appHbMgrCleanup(void) { pthread_mutex_lock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); - if (pAppHbMgr == pTarget) { - taosHashCleanup(pTarget->activeInfo); - taosHashCleanup(pTarget->getInfoFuncs); - } + taosHashCleanup(pTarget->activeInfo); + pTarget->activeInfo = NULL; + taosHashCleanup(pTarget->connInfo); + pTarget->connInfo = NULL; } pthread_mutex_unlock(&clientHbMgr.lock); @@ -219,31 +578,22 @@ int hbMgrInit() { } void hbMgrCleanUp() { + return; + hbStopThread(); + // destroy all appHbMgr int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); if (old == 0) return; - taosArrayDestroy(clientHbMgr.appHbMgrs); + pthread_mutex_lock(&clientHbMgr.lock); + appHbMgrCleanup(); + taosArrayDestroy(clientHbMgr.appHbMgrs); + pthread_mutex_unlock(&clientHbMgr.lock); + + clientHbMgr.appHbMgrs = NULL; } -int hbHandleRsp(SClientHbBatchRsp* hbRsp) { - int64_t reqId = hbRsp->reqId; - int64_t rspId = hbRsp->rspId; - - SArray* rsps = hbRsp->rsps; - int32_t sz = taosArrayGetSize(rsps); - for (int i = 0; i < sz; i++) { - SClientHbRsp* pRsp = taosArrayGet(rsps, i); - if (pRsp->connKey.hbType < HEARTBEAT_TYPE_MAX) { - clientHbMgr.handle[pRsp->connKey.hbType](pRsp); - } else { - // discard rsp - } - } - return 0; -} - -int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func) { +int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo *info) { // init hash in activeinfo void* data = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (data != NULL) { @@ -252,19 +602,49 @@ int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func SClientHbReq hbReq; hbReq.connKey = connKey; hbReq.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + taosHashPut(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey), &hbReq, sizeof(SClientHbReq)); + // init hash - if (func != NULL) { - taosHashPut(pAppHbMgr->getInfoFuncs, &connKey, sizeof(SClientHbKey), func, sizeof(FGetConnInfo)); + if (info != NULL) { + SClientHbReq * pReq = taosHashGet(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + info->req = pReq; + taosHashPut(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey), info, sizeof(SHbConnInfo)); } atomic_add_fetch_32(&pAppHbMgr->connKeyCnt, 1); return 0; } +int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { + SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; + SHbConnInfo info = {0}; + + switch (hbType) { + case HEARTBEAT_TYPE_QUERY: { + int64_t *pClusterId = malloc(sizeof(int64_t)); + *pClusterId = clusterId; + + info.param = pClusterId; + break; + } + case HEARTBEAT_TYPE_MQ: { + break; + } + default: + break; + } + + return hbRegisterConnImpl(pAppHbMgr, connKey, &info); +} + void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { - taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); - taosHashRemove(pAppHbMgr->getInfoFuncs, &connKey, sizeof(SClientHbKey)); + int32_t code = 0; + code = taosHashRemove(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); + code = taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); + if (code) { + return; + } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 28a4e0c87d..dfe7b12ce4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,5 +1,4 @@ -#include "../../libs/scheduler/inc/schedulerInt.h" #include "clientInt.h" #include "clientLog.h" #include "parser.h" @@ -13,15 +12,6 @@ #include "tpagedfile.h" #include "tref.h" -#define CHECK_CODE_GOTO(expr, label) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - terrno = code; \ - goto label; \ - } \ - } while (0) - static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); @@ -101,7 +91,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } if (port) { - epSet.epSet.port[0] = port; + epSet.epSet.eps[0].port = port; } } else { if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { @@ -115,11 +105,12 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, pthread_mutex_lock(&appInfo.mutex); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); + SAppInstInfo* p = NULL; if (pInst == NULL) { - SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); + p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - p->pAppHbMgr = appHbMgrInit(p); + p->pAppHbMgr = appHbMgrInit(p, key); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -218,12 +209,10 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, if (pQueryNode->type == TSDB_SQL_SELECT) { setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); - tfree(pSchema); pRequest->type = TDMT_VND_QUERY; - } else { - tfree(pSchema); } + tfree(pSchema); return code; } @@ -241,9 +230,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { + void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); + int32_t code = schedulerExecJob(pTransporter, NULL, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { @@ -257,466 +247,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) return pRequest->code; } - return schedulerAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); + return schedulerAsyncExecJob(pTransporter, pNodeList, pDag, pRequest->sqlstr, &pRequest->body.pQueryJob); } - -typedef struct SMqClientVg { - // statistics - int64_t pollCnt; - // offset - int64_t committedOffset; - int64_t currentOffset; - //connection info - int32_t vgId; - SEpSet epSet; -} SMqClientVg; - -typedef struct SMqClientTopic { - // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; - int32_t nextVgIdx; - SArray* vgs; //SArray -} SMqClientTopic; - -typedef struct tmq_resp_err_t { - int32_t code; -} tmq_resp_err_t; - -typedef struct tmq_topic_vgroup_t { - char* topic; - int32_t vgId; - int64_t commitOffset; -} tmq_topic_vgroup_t; - -typedef struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; - tmq_topic_vgroup_t* elems; -} tmq_topic_vgroup_list_t; - -typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); - -struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - char* ip; - uint16_t port; - tmq_commit_cb* commit_cb; -}; - -tmq_conf_t* tmq_conf_new() { - tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); - return conf; -} - -int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (strcmp(key, "group.id") == 0) { - strcpy(conf->groupId, value); - } - if (strcmp(key, "client.id") == 0) { - strcpy(conf->clientId, value); - } - return 0; -} - -struct tmq_t { - char groupId[256]; - char clientId[256]; - int64_t consumerId; - int64_t status; - STscObj* pTscObj; - tmq_commit_cb* commit_cb; - int32_t nextTopicIdx; - SArray* clientTopics; //SArray -}; - -tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - tmq_t* pTmq = calloc(sizeof(tmq_t), 1); - if (pTmq == NULL) { - return NULL; - } - pTmq->pTscObj = (STscObj*)conn; - pTmq->status = 0; - strcpy(pTmq->clientId, conf->clientId); - strcpy(pTmq->groupId, conf->groupId); - pTmq->commit_cb = conf->commit_cb; - pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); - pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); - return pTmq; -} - -struct tmq_list_t { - int32_t cnt; - int32_t tot; - char* elems[]; -}; -tmq_list_t* tmq_list_new() { - tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); - if (ptr == NULL) { - return ptr; - } - ptr->cnt = 0; - ptr->tot = 8; - return ptr; -} - -int32_t tmq_list_append(tmq_list_t* ptr, char* src) { - if (ptr->cnt >= ptr->tot-1) return -1; - ptr->elems[ptr->cnt] = strdup(src); - ptr->cnt++; - return 0; -} - - -TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { - SRequestObj *pRequest = NULL; - tmq->status = 1; - int32_t sz = topic_list->cnt; - tmq->clientTopics = taosArrayInit(sz, sizeof(void*)); - for (int i = 0; i < sz; i++) { - char* topicName = topic_list->elems[i]; - - SName name = {0}; - char* dbName = getDbOfConnection(tmq->pTscObj); - tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); - tNameFromString(&name, topicName, T_NAME_TABLE); - - char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); - if (topicFname == NULL) { - - } - tNameExtractFullName(&name, topicFname); - tscDebug("subscribe topic: %s", topicFname); - taosArrayPush(tmq->clientTopics, &topicFname); - /*SMqClientTopic topic = {*/ - /*.*/ - /*};*/ - } - SCMSubscribeReq req; - req.topicNum = taosArrayGetSize(tmq->clientTopics); - req.consumerId = tmq->consumerId; - req.consumerGroup = strdup(tmq->groupId); - req.topicNames = tmq->clientTopics; - - int tlen = tSerializeSCMSubscribeReq(NULL, &req); - void* buf = malloc(tlen); - if(buf == NULL) { - goto _return; - } - - void* abuf = buf; - tSerializeSCMSubscribeReq(&abuf, &req); - /*printf("formatted: %s\n", dagStr);*/ - - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); - if (pRequest == NULL) { - tscError("failed to malloc sqlObj"); - } - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - - SMsgSendInfo* body = buildMsgInfoImpl(pRequest); - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); - - tsem_wait(&pRequest->body.rspSem); - -_return: - if (body != NULL) { - destroySendMsgInfo(body); - } - - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; -} - -void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { - conf->commit_cb = cb; -} - -SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { - tmq_t* pTmq = (void*)param; - SArray* pArray = taosArrayInit(0, sizeof(SKv)); - if (pArray == NULL) { - return NULL; - } - SKv kv = {0}; - kv.key = malloc(256); - if (kv.key == NULL) { - taosArrayDestroy(pArray); - return NULL; - } - strcpy(kv.key, "mq-tmp"); - kv.keyLen = strlen("mq-tmp") + 1; - SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg)); - if (pMqHb == NULL) { - return pArray; - } - pMqHb->consumerId = connKey.connId; - SArray* clientTopics = pTmq->clientTopics; - int sz = taosArrayGetSize(clientTopics); - for (int i = 0; i < sz; i++) { - SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); - /*if (pCTopic->vgId == -1) {*/ - /*pMqHb->status = 1;*/ - /*break;*/ - /*}*/ - } - kv.value = pMqHb; - kv.valueLen = sizeof(SMqHbMsg); - taosArrayPush(pArray, &kv); - - return pArray; -} - -tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { - tmq_t* pTmq = malloc(sizeof(tmq_t)); - if (pTmq == NULL) { - return NULL; - } - strcpy(pTmq->groupId, conf->groupId); - strcpy(pTmq->clientId, conf->clientId); - pTmq->pTscObj = (STscObj*)conn; - pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; - - return pTmq; -} - -TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { - STscObj *pTscObj = (STscObj*)taos; - SRequestObj *pRequest = NULL; - SQueryNode *pQueryNode = NULL; - char *pStr = NULL; - - terrno = TSDB_CODE_SUCCESS; - if (taos == NULL || topicName == NULL || sql == NULL) { - tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { - tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - goto _return; - } - - tscDebug("start to create topic, %s", topicName); - - CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); - - SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo* ) pQueryNode; - pQueryStmtInfo->info.continueQuery = true; - - // todo check for invalid sql statement and return with error code - - SSchema *schema = NULL; - int32_t numOfCols = 0; - CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return); - - pStr = qDagToString(pRequest->body.pDag); - if(pStr == NULL) { - goto _return; - } - - printf("%s\n", pStr); - - // The topic should be related to a database that the queried table is belonged to. - SName name = {0}; - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName); - - tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB); - tNameFromString(&name, topicName, T_NAME_TABLE); - - char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; - tNameExtractFullName(&name, topicFname); - - SCMCreateTopicReq req = { - .name = (char*) topicFname, - .igExists = 1, - .physicalPlan = (char*) pStr, - .sql = (char*) sql, - .logicalPlan = "no logic plan", - }; - - int tlen = tSerializeSCMCreateTopicReq(NULL, &req); - void* buf = malloc(tlen); - if(buf == NULL) { - goto _return; - } - - void* abuf = buf; - tSerializeSCMCreateTopicReq(&abuf, &req); - /*printf("formatted: %s\n", dagStr);*/ - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - pRequest->type = TDMT_MND_CREATE_TOPIC; - - SMsgSendInfo* body = buildMsgInfoImpl(pRequest); - SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, body); - - tsem_wait(&pRequest->body.rspSem); - -_return: - qDestroyQuery(pQueryNode); - if (body != NULL) { - destroySendMsgInfo(body); - } - - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; -} - -/*typedef SMqConsumeRsp tmq_message_t;*/ - -struct tmq_message_t { - SMqConsumeRsp rsp; -}; - -int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { - return 0; -} - -int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { - tmq_t* tmq = (tmq_t*)param; - SMqCMGetSubEpRsp rsp; - tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); - int32_t sz = taosArrayGetSize(rsp.topics); - // TODO: lock - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); - topic.topicName = strdup(pTopicEp->topic); - int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgSz; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - SMqClientVg clientVg = { - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet - }; - taosArrayPush(topic.vgs, &clientVg); - } - taosArrayPush(tmq->clientTopics, &topic); - } - // unlock - return 0; -} - -tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { - if (tmq->clientTopics == NULL || taosArrayGetSize(tmq->clientTopics) == 0) { - return NULL; - } - SRequestObj *pRequest = NULL; - SMqConsumeReq req = {0}; - req.reqType = 1; - req.blockingTime = blocking_time; - req.consumerId = tmq->consumerId; - tmq_message_t* tmq_message = NULL; - strcpy(req.cgroup, tmq->groupId); - - if (taosArrayGetSize(tmq->clientTopics) == 0) { - int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* buf = malloc(tlen); - if (buf == NULL) { - tscError("failed to malloc get subscribe ep buf"); - } - buf->consumerId = htobe64(buf->consumerId); - - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); - if (pRequest == NULL) { - tscError("failed to malloc subscribe ep request"); - } - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = tmq; - sendInfo->fp = tmq_ask_ep_cb; - - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - } - - SMqClientTopic* pTopic = taosArrayGetP(tmq->clientTopics, tmq->nextTopicIdx); - tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - strcpy(req.topic, pTopic->topicName); - int32_t nextVgIdx = pTopic->nextVgIdx; - pTopic->nextVgIdx = (nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, nextVgIdx); - req.offset = pVg->currentOffset; - - pRequest->body.requestMsg = (SDataBuf){ .pData = &req, .len = sizeof(SMqConsumeReq) }; - pRequest->type = TDMT_VND_CONSUME; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = &tmq_message; - sendInfo->fp = tmq_poll_cb_inner; - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - - return tmq_message; - - /*tsem_wait(&pRequest->body.rspSem);*/ - - /*if (body != NULL) {*/ - /*destroySendMsgInfo(body);*/ - /*}*/ - - /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ - /*pRequest->code = terrno;*/ - /*}*/ - - /*return pRequest;*/ -} - -tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - SMqConsumeReq req = {0}; - return NULL; -} - -void tmq_message_destroy(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return; -} - - TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { STscObj *pTscObj = (STscObj *)taos; - if (sqlLen > (size_t) tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + if (sqlLen > (size_t) TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; } @@ -734,7 +271,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { if (qIsDdlQuery(pQueryNode)) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQueryNode), _return); } else { - CHECK_CODE_GOTO(getPlan(pRequest, pQueryNode, &pRequest->body.pDag, pNodeList), _return); CHECK_CODE_GOTO(scheduleQuery(pRequest, pRequest->body.pDag, pNodeList), _return); pRequest->code = terrno; @@ -764,7 +300,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe return -1; } - taosGetFqdnPortFromEp(firstEp, mgmtEpSet->fqdn[0], &(mgmtEpSet->port[0])); + taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); mgmtEpSet->numOfEps++; } @@ -774,7 +310,7 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe return -1; } - taosGetFqdnPortFromEp(secondEp, mgmtEpSet->fqdn[mgmtEpSet->numOfEps], &(mgmtEpSet->port[mgmtEpSet->numOfEps])); + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); mgmtEpSet->numOfEps++; } @@ -979,14 +515,7 @@ void* doFetchRow(SRequestObj* pRequest) { SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex); - epSet.numOfEps = pVgroupInfo->numOfEps; - epSet.inUse = pVgroupInfo->inUse; - - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); - epSet.port[i] = pVgroupInfo->epAddr[i].port; - } - + epSet = pVgroupInfo->epset; } else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { pRequest->type = TDMT_VND_SHOW_TABLES; SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo; @@ -1003,14 +532,7 @@ void* doFetchRow(SRequestObj* pRequest) { pRequest->body.requestMsg.pData = pShowReq; SMsgSendInfo* body = buildMsgInfoImpl(pRequest); - - epSet.numOfEps = pVgroupInfo->numOfEps; - epSet.inUse = pVgroupInfo->inUse; - - for (int32_t i = 0; i < epSet.numOfEps; ++i) { - strncpy(epSet.fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(epSet.fqdn[i])); - epSet.port[i] = pVgroupInfo->epAddr[i].port; - } + epSet = pVgroupInfo->epset; int64_t transporterId = 0; STscObj *pTscObj = pRequest->pTscObj; @@ -1018,8 +540,12 @@ void* doFetchRow(SRequestObj* pRequest) { tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; - } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE && pResultInfo->pData != NULL) { - return NULL; + } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { + epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + if (pResultInfo->completed) { + return NULL; + } } SMsgSendInfo* body = buildMsgInfoImpl(pRequest); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index ba2a21d7ea..b29813be03 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -46,6 +46,8 @@ void taos_cleanup(void) { clientConnRefPool = -1; taosCloseRef(id); + hbMgrCleanUp(); + rpcCleanup(); catalogDestroy(); taosCloseLog(); @@ -76,7 +78,7 @@ void taos_close(TAOS* taos) { STscObj *pTscObj = (STscObj *)taos; tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); - taosRemoveRef(clientConnRefPool, pTscObj->id); + /*taosRemoveRef(clientConnRefPool, pTscObj->id);*/ } int taos_errno(TAOS_RES *tres) { @@ -275,3 +277,70 @@ int taos_affected_rows(TAOS_RES *res) { } int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } + +int taos_select_db(TAOS *taos, const char *db) { + STscObj *pObj = (STscObj *)taos; + if (pObj == NULL) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + if (db == NULL || strlen(db) == 0) { + terrno = TSDB_CODE_TSC_INVALID_INPUT; + return terrno; + } + + char sql[256] = {0}; + snprintf(sql, tListLen(sql), "use %s", db); + + TAOS_RES* pRequest = taos_query(taos, sql); + int32_t code = taos_errno(pRequest); + + taos_free_result(pRequest); + return code; +} + +void taos_stop_query(TAOS_RES *res) { + if (res == NULL) { + return; + } + + SRequestObj* pRequest = (SRequestObj*) res; + int32_t numOfFields = taos_num_fields(pRequest); + + // It is not a query, no need to stop. + if (numOfFields == 0) { + return; + } + +// scheduleCancelJob(pRequest->body.pQueryJob); +} + +bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { + return false; +} + +int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { + return 0; +} + +int taos_validate_sql(TAOS *taos, const char *sql) { + return true; +} + +const char *taos_get_server_info(TAOS *taos) { + if (taos == NULL) { + return NULL; + } + + STscObj* pTscObj = (STscObj*) taos; + return pTscObj->ver; +} + +void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { + // TODO +} + +void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { + // TODO +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index ec088eb073..8ab1880069 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -18,6 +18,7 @@ #include "tname.h" #include "clientInt.h" #include "clientLog.h" +#include "catalog.h" int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); @@ -53,7 +54,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { assert(pConnect->epSet.numOfEps > 0); for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) { - pConnect->epSet.port[i] = htons(pConnect->epSet.port[i]); + pConnect->epSet.eps[i].port = htons(pConnect->epSet.eps[i].port); } if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { @@ -61,18 +62,21 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { } for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.fqdn[i], pConnect->epSet.port[i], pTscObj->id); + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.eps[i].fqdn, + pConnect->epSet.eps[i].port, pTscObj->id); } pTscObj->connId = pConnect->connId; pTscObj->acctId = pConnect->acctId; + tstrncpy(pTscObj->ver, pConnect->sVersion, tListLen(pTscObj->ver)); // update the appInstInfo pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); - SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); + pTscObj->connType = HEARTBEAT_TYPE_QUERY; + + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY); // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, @@ -197,6 +201,7 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; + pResInfo->completed = pRetrieve->completed; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); @@ -288,7 +293,6 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) { } int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { - // todo: Remove cache in catalog cache. SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); @@ -296,6 +300,15 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } + SDropDbRsp *rsp = (SDropDbRsp *)pMsg->pData; + + struct SCatalog *pCatalog = NULL; + rsp->uid = be64toh(rsp->uid); + + catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + + catalogRemoveDB(pCatalog, rsp->db, rsp->uid); + tsem_post(&pRequest->body.rspSem); return code; } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c new file mode 100644 index 0000000000..e57901ed2e --- /dev/null +++ b/source/client/src/tmq.c @@ -0,0 +1,800 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE + +#include "clientInt.h" +#include "clientLog.h" +#include "parser.h" +#include "planner.h" +#include "scheduler.h" +#include "tdef.h" +#include "tep.h" +#include "tglobal.h" +#include "tmsgtype.h" +#include "tnote.h" +#include "tpagedfile.h" +#include "tref.h" + +struct tmq_list_t { + int32_t cnt; + int32_t tot; + char* elems[]; +}; +struct tmq_topic_vgroup_t { + char* topic; + int32_t vgId; + int64_t offset; +}; + +struct tmq_topic_vgroup_list_t { + int32_t cnt; + int32_t size; + tmq_topic_vgroup_t* elems; +}; + +struct tmq_conf_t { + char clientId[256]; + char groupId[256]; + /*char* ip;*/ + /*uint16_t port;*/ + tmq_commit_cb* commit_cb; +}; + +struct tmq_t { + char groupId[256]; + char clientId[256]; + SRWLatch lock; + int64_t consumerId; + int64_t epoch; + int64_t status; + tsem_t rspSem; + STscObj* pTscObj; + tmq_commit_cb* commit_cb; + int32_t nextTopicIdx; + SArray* clientTopics; // SArray + // stat + int64_t pollCnt; +}; + +struct tmq_message_t { + SMqConsumeRsp rsp; +}; + +typedef struct SMqClientVg { + // statistics + int64_t pollCnt; + // offset + int64_t committedOffset; + int64_t currentOffset; + // connection info + int32_t vgId; + SEpSet epSet; +} SMqClientVg; + +typedef struct SMqClientTopic { + // subscribe info + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + int32_t nextVgIdx; + SArray* vgs; // SArray +} SMqClientTopic; + +typedef struct SMqSubscribeCbParam { + tmq_t* tmq; + tsem_t rspSem; + tmq_resp_err_t rspErr; +} SMqSubscribeCbParam; + +typedef struct SMqAskEpCbParam { + tmq_t* tmq; + int32_t wait; +} SMqAskEpCbParam; + +typedef struct SMqConsumeCbParam { + tmq_t* tmq; + SMqClientVg* pVg; + tmq_message_t** retMsg; + tsem_t rspSem; +} SMqConsumeCbParam; + +typedef struct SMqCommitCbParam { + tmq_t* tmq; + SMqClientVg* pVg; + int32_t async; + tsem_t rspSem; +} SMqCommitCbParam; + +tmq_conf_t* tmq_conf_new() { + tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + return conf; +} + +void tmq_conf_destroy(tmq_conf_t* conf) { + if (conf) free(conf); +} + +tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + if (strcmp(key, "group.id") == 0) { + strcpy(conf->groupId, value); + } + if (strcmp(key, "client.id") == 0) { + strcpy(conf->clientId, value); + } + return TMQ_CONF_OK; +} + +tmq_list_t* tmq_list_new() { + tmq_list_t* ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); + if (ptr == NULL) { + return ptr; + } + ptr->cnt = 0; + ptr->tot = 8; + return ptr; +} + +int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { + if (ptr->cnt >= ptr->tot - 1) return -1; + ptr->elems[ptr->cnt] = strdup(src); + ptr->cnt++; + return 0; +} + +int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; + pParam->rspErr = code; + tsem_post(&pParam->rspSem); + return 0; +} + +int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqCommitCbParam* pParam = (SMqCommitCbParam*) param; + tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; + if (pParam->tmq->commit_cb) { + pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); + } + if (!pParam->async) tsem_post(&pParam->rspSem); + return 0; +} + +tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + tmq_t* pTmq = calloc(sizeof(tmq_t), 1); + if (pTmq == NULL) { + return NULL; + } + pTmq->pTscObj = (STscObj*)conn; + pTmq->status = 0; + pTmq->pollCnt = 0; + pTmq->epoch = 0; + taosInitRWLatch(&pTmq->lock); + strcpy(pTmq->clientId, conf->clientId); + strcpy(pTmq->groupId, conf->groupId); + pTmq->commit_cb = conf->commit_cb; + tsem_init(&pTmq->rspSem, 0, 0); + pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); + pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + return pTmq; +} + +tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { + SRequestObj* pRequest = NULL; + int32_t sz = topic_list->cnt; + // destroy ex + taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + + SCMSubscribeReq req; + req.topicNum = sz; + req.consumerId = tmq->consumerId; + req.consumerGroup = strdup(tmq->groupId); + req.topicNames = taosArrayInit(sz, sizeof(void*)); + + for (int i = 0; i < sz; i++) { + char* topicName = topic_list->elems[i]; + + SName name = {0}; + char* dbName = getDbOfConnection(tmq->pTscObj); + tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + if (topicFname == NULL) { + } + tNameExtractFullName(&name, topicFname); + tscDebug("subscribe topic: %s", topicFname); + SMqClientTopic topic = { + .nextVgIdx = 0, + .sql = NULL, + .sqlLen = 0, + .topicId = 0, + .topicName = topicFname, + .vgs = NULL + }; + topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); + taosArrayPush(tmq->clientTopics, &topic); + /*SMqClientTopic topic = {*/ + /*.*/ + /*};*/ + taosArrayPush(req.topicNames, &topicFname); + free(dbName); + } + + int tlen = tSerializeSCMSubscribeReq(NULL, &req); + void* buf = malloc(tlen); + if (buf == NULL) { + goto _return; + } + + void* abuf = buf; + tSerializeSCMSubscribeReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); + if (pRequest == NULL) { + tscError("failed to malloc sqlObj"); + } + + SMqSubscribeCbParam param = {.rspErr = TMQ_RESP_ERR__SUCCESS, .tmq = tmq}; + tsem_init(¶m.rspSem, 0, 0); + + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->param = ¶m; + sendInfo->fp = tmqSubscribeCb; + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(¶m.rspSem); + tsem_destroy(¶m.rspSem); + +_return: + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + return param.rspErr; +} + +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } + +SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { + tmq_t* pTmq = (void*)param; + SArray* pArray = taosArrayInit(0, sizeof(SKv)); + if (pArray == NULL) { + return NULL; + } + SKv kv = {0}; + kv.key = HEARTBEAT_KEY_MQ_TMP; + + SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg)); + if (pMqHb == NULL) { + return pArray; + } + pMqHb->consumerId = connKey.connId; + SArray* clientTopics = pTmq->clientTopics; + int sz = taosArrayGetSize(clientTopics); + for (int i = 0; i < sz; i++) { + SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); + /*if (pCTopic->vgId == -1) {*/ + /*pMqHb->status = 1;*/ + /*break;*/ + /*}*/ + } + kv.value = pMqHb; + kv.valueLen = sizeof(SMqHbMsg); + taosArrayPush(pArray, &kv); + + return pArray; +} + +TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { + STscObj* pTscObj = (STscObj*)taos; + SRequestObj* pRequest = NULL; + SQueryNode* pQueryNode = NULL; + char* pStr = NULL; + + terrno = TSDB_CODE_SUCCESS; + if (taos == NULL || topicName == NULL || sql == NULL) { + tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { + tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + goto _return; + } + + tscDebug("start to create topic, %s", topicName); + + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); + + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*)pQueryNode; + pQueryStmtInfo->info.continueQuery = true; + + // todo check for invalid sql statement and return with error code + + SSchema* schema = NULL; + int32_t numOfCols = 0; + CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), + _return); + + pStr = qDagToString(pRequest->body.pDag); + if (pStr == NULL) { + goto _return; + } + + /*printf("%s\n", pStr);*/ + + // The topic should be related to a database that the queried table is belonged to. + SName name = {0}; + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName); + + tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; + tNameExtractFullName(&name, topicFname); + + SCMCreateTopicReq req = { + .name = (char*)topicFname, + .igExists = 1, + .physicalPlan = (char*)pStr, + .sql = (char*)sql, + .logicalPlan = (char*)"no logic plan", + }; + + int tlen = tSerializeSCMCreateTopicReq(NULL, &req); + void* buf = malloc(tlen); + if (buf == NULL) { + goto _return; + } + + void* abuf = buf; + tSerializeSCMCreateTopicReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + pRequest->type = TDMT_MND_CREATE_TOPIC; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(&pRequest->body.rspSem); + +_return: + qDestroyQuery(pQueryNode); + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + + return pRequest; +} + +static char* formatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} + +void tmqShowMsg(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return; + + static bool noPrintSchema; + char pBuf[128]; + SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + int32_t colNum = pRsp->schemas->nCols; + if (!noPrintSchema) { + printf("|"); + for (int32_t i = 0; i < colNum; i++) { + if (i == 0) + printf(" %25s |", pRsp->schemas->pSchema[i].name); + else + printf(" %15s |", pRsp->schemas->pSchema[i].name); + } + printf("\n"); + printf("===============================================\n"); + noPrintSchema = true; + } + int32_t sz = taosArrayGetSize(pRsp->pBlockData); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pRsp->pBlockData, i); + int32_t rows = pDataBlock->info.rows; + for (int32_t j = 0; j < rows; j++) { + printf("|"); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + printf(" %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t*)var); + break; + } + } + printf("\n"); + } + } +} + +int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; + SMqClientVg* pVg = pParam->pVg; + if (code != 0) { + /*printf("msg discard\n");*/ + tsem_post(&pParam->rspSem); + return 0; + } + + SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp)); + if (pRsp == NULL) { + tsem_post(&pParam->rspSem); + return -1; + } + tDecodeSMqConsumeRsp(pMsg->pData, pRsp); + /*printf("rsp %ld %ld %d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ + if (pRsp->numOfTopics == 0) { + /*printf("no data\n");*/ + free(pRsp); + tsem_post(&pParam->rspSem); + return 0; + } + *pParam->retMsg = (tmq_message_t*)pRsp; + pVg->currentOffset = pRsp->rspOffset; + /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ + /*printf("-----msg begin----\n");*/ + tsem_post(&pParam->rspSem); + /*printf("\n-----msg end------\n");*/ + return 0; +} + +int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + tmq_t* tmq = pParam->tmq; + if (code != 0) { + printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait); + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } + return 0; + } + tscDebug("tmq ask ep cb called"); + bool set = false; + SMqCMGetSubEpRsp rsp; + tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); + int32_t sz = taosArrayGetSize(rsp.topics); + // TODO: lock + /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ + /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ + if (rsp.epoch != tmq->epoch) { + // TODO + if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); + topic.topicName = strdup(pTopicEp->topic); + int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); + topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); + for (int32_t j = 0; j < vgSz; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + SMqClientVg clientVg = { + .pollCnt = 0, .committedOffset = -1, .currentOffset = -1, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet}; + taosArrayPush(topic.vgs, &clientVg); + set = true; + } + taosArrayPush(tmq->clientTopics, &topic); + } + tmq->epoch = rsp.epoch; + } + if (set) { + atomic_store_64(&tmq->status, 1); + } + // unlock + /*tsem_post(&tmq->rspSem);*/ + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } + tDeleteSMqCMGetSubEpRsp(&rsp); + return 0; +} + +int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { + int32_t tlen = sizeof(SMqCMGetSubEpReq); + SMqCMGetSubEpReq* buf = malloc(tlen); + if (buf == NULL) { + tscError("failed to malloc get subscribe ep buf"); + goto END; + } + buf->consumerId = htobe64(tmq->consumerId); + strcpy(buf->cgroup, tmq->groupId); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); + if (pRequest == NULL) { + tscError("failed to malloc subscribe ep request"); + goto END; + } + + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + + SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + tscError("failed to malloc subscribe param"); + goto END; + } + pParam->tmq = tmq; + pParam->wait = wait; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqAskEpCb; + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + +END: + if (wait) tsem_wait(&tmq->rspSem); + return 0; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, + SMqClientVg* pVg) { + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); + if (pReq == NULL) { + return NULL; + } + pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + strcpy(pReq->cgroup, tmq->groupId); + + if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { + pReq->offset = pVg->currentOffset; + } else { + pReq->offset = pVg->currentOffset + 1; + } + + pReq->head.vgId = htonl(pVg->vgId); + pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + return pReq; +} + +tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { + tmq_message_t* tmq_message = NULL; + + int64_t status = atomic_load_64(&tmq->status); + tmqAsyncAskEp(tmq, status == 0); + + if (blocking_time < 0) blocking_time = 1; + if (blocking_time > 1000) blocking_time = 1000; + /*blocking_time = 1;*/ + + if (taosArrayGetSize(tmq->clientTopics) == 0) { + tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); + usleep(blocking_time * 1000); + return NULL; + } + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); + if (taosArrayGetSize(pTopic->vgs) == 0) { + usleep(blocking_time * 1000); + return NULL; + } + + tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); + if (pReq == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + + SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); + if (param == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + param->tmq = tmq; + param->retMsg = &tmq_message; + param->pVg = pVg; + tsem_init(¶m->rspSem, 0, 0); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = param; + sendInfo->fp = tmqPollCb; + + /*printf("req offset: %ld\n", pReq->offset);*/ + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + tmq->pollCnt++; + + tsem_wait(¶m->rspSem); + tsem_destroy(¶m->rspSem); + free(param); + + if (tmq_message == NULL) { + usleep(blocking_time * 1000); + } + + return tmq_message; + + /*tsem_wait(&pRequest->body.rspSem);*/ + + /*if (body != NULL) {*/ + /*destroySendMsgInfo(body);*/ + /*}*/ + + /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ + /*pRequest->code = terrno;*/ + /*}*/ + + /*return pRequest;*/ +} + +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { + + if (tmq_topic_vgroup_list != NULL) { + //TODO + } + + //TODO: change semaphore to gate + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; + SMqCommitCbParam *pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + continue; + } + pParam->tmq = tmq; + pParam->pVg = pVg; + pParam->async = async; + if (!async) tsem_init(&pParam->rspSem, 0, 0); + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + + if (!async) tsem_wait(&pParam->rspSem); + } + } + + return 0; +} + +void tmq_message_destroy(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return; + SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + tDeleteSMqConsumeRsp(pRsp); + free(tmq_message); +} + +tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { + return TMQ_RESP_ERR__SUCCESS; +} + +const char* tmq_err2str(tmq_resp_err_t err) { + if (err == TMQ_RESP_ERR__SUCCESS) { + return "success"; + } + return "fail"; +} +#if 0 +tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { + tmq_t* pTmq = malloc(sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + strcpy(pTmq->groupId, conf->groupId); + strcpy(pTmq->clientId, conf->clientId); + pTmq->pTscObj = (STscObj*)conn; + pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; + return pTmq; +} + + +static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->msgInfo.pData); + tfree(pMsgBody); +} +#endif diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 6886206363..ee5109860e 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -5,10 +5,16 @@ MESSAGE(STATUS "build parser unit test") SET(CMAKE_CXX_STANDARD 11) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) -ADD_EXECUTABLE(clientTest ${SOURCE_LIST}) +ADD_EXECUTABLE(clientTest clientTests.cpp) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom +) + +ADD_EXECUTABLE(tmqTest tmqTest.cpp) +TARGET_LINK_LIBRARIES( + tmqTest + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom ) TARGET_INCLUDE_DIRECTORIES( @@ -16,3 +22,9 @@ TARGET_INCLUDE_DIRECTORIES( PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/" PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc" ) + +TARGET_INCLUDE_DIRECTORIES( + tmqTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/client/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/client/inc" +) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index a1adf58f6a..1e68faa4f4 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,6 +53,7 @@ TEST(testCase, driverInit_Test) { // taos_init(); } +#if 0 TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -61,474 +62,165 @@ TEST(testCase, connect_Test) { taos_close(pConn); } -//TEST(testCase, create_user_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} +TEST(testCase, create_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, create_account_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, drop_account_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, show_user_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "show users"); - //TAOS_ROW pRow = NULL; - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, drop_user_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, show_db_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "show databases"); - //TAOS_ROW pRow = NULL; - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_account_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show users"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_user_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show databases"); + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_close(pConn); +} + +TEST(testCase, create_db_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database abc1 vgroups 4"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } + taos_close(pConn); +} + +TEST(testCase, create_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); + if (taos_errno(pRes) != 0) { + printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_close(pConn); -//} + pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); + if (taos_errno(pRes) != 0) { + printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + taos_close(pConn); +} -//TEST(testCase, create_db_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create database abc1 vgroups 4"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_close(pConn); -//} - -//TEST(testCase, create_dnode_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - //if (taos_errno(pRes) != 0) { - //printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //taos_close(pConn); -//} - -//TEST(testCase, drop_dnode_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); - //if (taos_errno(pRes) != 0) { - //printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - - //pRes = taos_query(pConn, "drop dnode 4"); - //if (taos_errno(pRes) != 0) { - //printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - //} +TEST(testCase, drop_dnode_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //taos_free_result(pRes); - //taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } -//TEST(testCase, use_db_test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in use db, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - - //taos_close(pConn); -//} - - //TEST(testCase, drop_db_test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //showDB(pConn); - - //TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //showDB(pConn); - - //pRes = taos_query(pConn, "create database abc1"); - //if (taos_errno(pRes) != 0) { - //printf("create to drop db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, create_stable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); - //if (taos_errno(pRes) != 0) { - //printf("error in create db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - //} - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //ASSERT_TRUE(pFields == NULL); - - //int32_t numOfFields = taos_num_fields(pRes); - //ASSERT_EQ(numOfFields, 0); - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); - //} - - //pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); - //pRes = taos_query(pConn, "drop stable `123_$^)`"); - //if (taos_errno(pRes) != 0) { - //printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); - //} - - //taos_close(pConn); -//} - -//TEST(testCase, create_table_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); - //ASSERT_EQ(taos_errno(pRes), 0); - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); - //ASSERT_NE(taos_errno(pRes), 0); - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, create_ctable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, show_stable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, show_vgroup_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "show vgroups"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //TAOS_ROW pRow = NULL; - - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, create_multiple_tables) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //ASSERT_NE(pConn, nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //taos_close(pConn); - //return; - //} - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //taos_free_result(pRes); - //pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} - - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} - - //taos_free_result(pRes); - - //for (int32_t i = 0; i < 20; ++i) { - //char sql[512] = {0}; - //snprintf(sql, tListLen(sql), - //"create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - //(i + 1) * 30, (i + 2) * 40); - //TAOS_RES* pres = taos_query(pConn, sql); - //if (taos_errno(pres) != 0) { - //printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - //} - //taos_free_result(pres); - //} - - //taos_close(pConn); -//} - -//TEST(testCase, show_table_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "show tables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //} - - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "show abc1.tables"); - //if (taos_errno(pRes) != 0) { - //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //} - - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - - //int32_t count = 0; - //char str[512] = {0}; - - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%d: %s\n", ++count, str); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, drop_stable_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != nullptr); - - //TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in using db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //pRes = taos_query(pConn, "drop stable st1"); - //if (taos_errno(pRes) != 0) { - //printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - //} - - //taos_free_result(pRes); - //taos_close(pConn); -//} - -//TEST(testCase, generated_request_id_test) { - //SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - //for (int32_t i = 0; i < 50000; ++i) { - //uint64_t v = generateRequestId(); - //void* result = taosHashGet(phash, &v, sizeof(v)); - //if (result != nullptr) { - //printf("0x%lx, index:%d\n", v, i); - //} - //assert(result == nullptr); - //taosHashPut(phash, &v, sizeof(v), NULL, 0); - //} - - //taosHashCleanup(phash); -//} - -TEST(testCase, create_topic_Test) { + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + pRes = taos_query(pConn, "drop dnode 4"); + if (taos_errno(pRes) != 0) { + printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, use_db_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -536,200 +228,456 @@ TEST(testCase, create_topic_Test) { if (taos_errno(pRes) != 0) { printf("error in use db, reason:%s\n", taos_errstr(pRes)); } - taos_free_result(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == nullptr); + ASSERT_TRUE(pFields == NULL); int32_t numOfFields = taos_num_fields(pRes); ASSERT_EQ(numOfFields, 0); + taos_close(pConn); +} + +// TEST(testCase, drop_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// showDB(pConn); +// +// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// showDB(pConn); +// +// pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// taos_close(pConn); +//} + +TEST(testCase, create_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + } taos_free_result(pRes); - char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("error in create stable, reason:%s\n", taos_errstr(pRes)); + } + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == NULL); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + pRes = taos_query(pConn, "drop stable `123_$^)`"); + if (taos_errno(pRes) != 0) { + printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); + } + + taos_close(pConn); +} + +TEST(testCase, create_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); + ASSERT_EQ(taos_errno(pRes), 0); + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); + ASSERT_NE(taos_errno(pRes), 0); + taos_free_result(pRes); taos_close(pConn); } -//TEST(testCase, insert_test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //ASSERT_NE(pConn, nullptr); +TEST(testCase, create_ctable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); - //pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); - //taos_free_result(pRes); - //taos_close(pConn); -//} + pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + } -//TEST(testCase, tmq_subscribe_Test) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //assert(pConn != NULL); - - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //if (taos_errno(pRes) != 0) { - //printf("error in use db, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); - - //tmq_conf_t* conf = tmq_conf_new(); - //tmq_conf_set(conf, "group.id", "tg1"); - //tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); - - //tmq_list_t* topic_list = tmq_list_new(); - //tmq_list_append(topic_list, "test_topic_1"); - //tmq_subscribe(tmq, topic_list); - - //while (1) { - //tmq_message_t* msg = tmq_consume_poll(tmq, 0); - //printf("get msg\n"); - //if (msg == NULL) break; - //} -//} - -TEST(testCase, tmq_consume_Test) { + taos_free_result(pRes); + taos_close(pConn); } -TEST(testCase, tmq_commit_TEST) { +TEST(testCase, show_stable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != nullptr); + + TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); + if (taos_errno(pRes) != 0) { + printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); } -//TEST(testCase, insert_test) { +TEST(testCase, show_vgroup_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "show vgroups"); + if (taos_errno(pRes) != 0) { + printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_multiple_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to create db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + taos_close(pConn); + return; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + taos_close(pConn); + return; + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create stable tables, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + taos_free_result(pRes); + pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + + for (int32_t i = 0; i < 20; ++i) { + char sql[512] = {0}; + snprintf(sql, tListLen(sql), + "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, + (i + 1) * 30, (i + 2) * 40); + TAOS_RES* pres = taos_query(pConn, sql); + if (taos_errno(pres) != 0) { + printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); + } + taos_free_result(pres); + } + + taos_close(pConn); +} + +TEST(testCase, show_table_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "show tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + taos_free_result(pRes); + + pRes = taos_query(pConn, "show abc1.tables"); + if (taos_errno(pRes) != 0) { + printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + int32_t count = 0; + char str[512] = {0}; + + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%d: %s\n", ++count, str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +//TEST(testCase, drop_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); +// assert(pConn != nullptr); // -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } // taos_free_result(pRes); // -// pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); +// pRes = taos_query(pConn, "use abc1"); // if (taos_errno(pRes) != 0) { -// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); // } // // taos_free_result(pRes); // taos_close(pConn); //} -//TEST(testCase, projection_query_tables) { - //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - //ASSERT_NE(pConn, nullptr); +TEST(testCase, generated_request_id_test) { + SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - //TAOS_RES* pRes = taos_query(pConn, "use abc1"); - //taos_free_result(pRes); + for (int32_t i = 0; i < 50000; ++i) { + uint64_t v = generateRequestId(); + void* result = taosHashGet(phash, &v, sizeof(v)); + if (result != nullptr) { + printf("0x%lx, index:%d\n", v, i); + } + assert(result == nullptr); + taosHashPut(phash, &v, sizeof(v), NULL, 0); + } - //pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); + taosHashCleanup(phash); +} - //pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - //if (taos_errno(pRes) != 0) { - //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - //} - //taos_free_result(pRes); +TEST(testCase, insert_test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); - //for(int32_t i = 0; i < 100000; ++i) { - //char sql[512] = {0}; - //sprintf(sql, "insert into tu values(now+%da, %d)", i, i); - //TAOS_RES* p = taos_query(pConn, sql); - //if (taos_errno(p) != 0) { - //printf("failed to insert data, reason:%s\n", taos_errstr(p)); - //} + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); - //taos_free_result(p); - //} + pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create into table t_2, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } - //pRes = taos_query(pConn, "select * from tu"); - //if (taos_errno(pRes) != 0) { - //printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - //taos_free_result(pRes); - //ASSERT_TRUE(false); - //} + taos_free_result(pRes); + taos_close(pConn); +} +#endif - //TAOS_ROW pRow = NULL; - //TAOS_FIELD* pFields = taos_fetch_fields(pRes); - //int32_t numOfFields = taos_num_fields(pRes); - //char str[512] = {0}; - //while ((pRow = taos_fetch_row(pRes)) != NULL) { - //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - //printf("%s\n", str); - //} +#if 0 +TEST(testCase, projection_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); - //taos_free_result(pRes); - //taos_close(pConn); -//} + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); -//TEST(testCase, projection_query_stables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "select ts from m1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); -//TEST(testCase, agg_query_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use dbv"); -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tx using st tags(111111111111111)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "select count(*) from tu"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 100000; ++i) { + char sql[512] = {0}; + sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); + } + + pRes = taos_query(pConn, "select * from tu"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, projection_query_stables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "select ts from st1"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, agg_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tx using st1 tags(111111111111111)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "select count(*) from t_x_19"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + } + + taos_free_result(pRes); + taos_close(pConn); +} +#endif #pragma GCC diagnostic pop diff --git a/source/client/test/tmqTest.cpp b/source/client/test/tmqTest.cpp new file mode 100644 index 0000000000..f767e7faef --- /dev/null +++ b/source/client/test/tmqTest.cpp @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "../inc/clientInt.h" +#include "taos.h" + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, driverInit_Test) { + taosInitGlobalCfg(); +// taos_init(); +} + +TEST(testCase, create_topic_ctb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + //taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from tu"; + pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, create_topic_stb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + //taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from st1"; + pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + +#if 0 +TEST(testCase, tmq_subscribe_ctb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg1"); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_ctb_topic_1"); + tmq_subscribe(tmq, topic_list); + + while (1) { + tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); + tmq_message_destroy(msg); + //printf("get msg\n"); + //if (msg == NULL) break; + } +} + +TEST(testCase, tmq_subscribe_stb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + tmq_subscribe(tmq, topic_list); + + int cnt = 1; + while (1) { + tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); + if (msg == NULL) continue; + tmqShowMsg(msg); + if (cnt++ % 10 == 0){ + tmq_commit(tmq, NULL, 0); + } + //tmq_commit(tmq, NULL, 0); + tmq_message_destroy(msg); + //printf("get msg\n"); + } +} + +TEST(testCase, tmq_consume_Test) { +} + +TEST(testCase, tmq_commit_Test) { +} + +#endif diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 9cc99e7f51..45587a8856 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -1,33 +1,45 @@ #include "tep.h" +#include "common.h" #include "tglobal.h" #include "tlockfree.h" -int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { - *port = 0; - strcpy(fqdn, ep); +int taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { + pEp->port = 0; + strcpy(pEp->fqdn, ep); - char *temp = strchr(fqdn, ':'); + char *temp = strchr(pEp->fqdn, ':'); if (temp) { *temp = 0; - *port = atoi(temp+1); + pEp->port = atoi(temp+1); } - if (*port == 0) { - *port = tsServerPort; + if (pEp->port == 0) { + pEp->port = tsServerPort; return -1; } return 0; } +void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) { + if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { + return; + } + + int32_t index = pEpSet->numOfEps; + tstrncpy(pEpSet->eps[index].fqdn, fqdn, tListLen(pEpSet->eps[index].fqdn)); + pEpSet->eps[index].port = port; + pEpSet->numOfEps += 1; +} + bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; } for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->port[i] != s2->port[i] - || strncmp(s1->fqdn[i], s2->fqdn[i], TSDB_FQDN_LEN) != 0) + if (s1->eps[i].port != s2->eps[i].port + || strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0) return false; } return true; @@ -48,3 +60,99 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { return ep; } +bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { + if (pColAgg != NULL) { + if (pColAgg->numOfNull == totalRows) { + ASSERT(pColumnInfoData->nullbitmap == NULL); + return true; + } else if (pColAgg->numOfNull == 0) { + ASSERT(pColumnInfoData->nullbitmap == NULL); + return false; + } + } + + if (pColumnInfoData->nullbitmap == NULL) { + return false; + } + + uint8_t v = (pColumnInfoData->nullbitmap[row>>3] & (1<<(8 - (row&0x07)))); + return (v == 1); +} + +bool colDataIsNull_f(const char* bitmap, uint32_t row) { + return (bitmap[row>>3] & (1<<(8 - (row&0x07)))); +} + +void colDataSetNull_f(char* bitmap, uint32_t row) { // TODO + return; +} + +void* colDataGet(const SColumnInfoData* pColumnInfoData, uint32_t row) { + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + uint32_t offset = ((uint32_t*)pColumnInfoData->pData)[row]; + return (char*)(pColumnInfoData->pData) + offset; // the first part is the pointer to the true binary data + } else { + return (char*)(pColumnInfoData->pData) + (row * pColumnInfoData->info.bytes); + } +} + +int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { + ASSERT(pColumnInfoData != NULL); + + if (isNull) { + // TODO set null value in the nullbitmap + return 0; + } + + int32_t type = pColumnInfoData->info.type; + if (IS_VAR_DATA_TYPE(type)) { + // TODO continue append var_type + } else { + char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; + switch(type) { + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} + default: + assert(0); + } + + } + + return 0; +} + +size_t colDataGetCols(const SSDataBlock* pBlock) { + ASSERT(pBlock); + + size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0; + ASSERT( pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock) + constantCols); + return pBlock->info.numOfCols; +} + +size_t colDataGetRows(const SSDataBlock* pBlock) { + return pBlock->info.rows; +} + +int32_t colDataUpdateTsWindow(SSDataBlock* pDataBlock) { + if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { + return 0; + } + + if (pDataBlock->info.numOfCols <= 0) { + return -1; + } + + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); + if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { + return 0; + } + + ASSERT(pColInfoData->nullbitmap == NULL); + pDataBlock->info.window.skey = *(TSKEY*) colDataGet(pColInfoData, 0); + pDataBlock->info.window.ekey = *(TSKEY*) colDataGet(pColInfoData, (pDataBlock->info.rows - 1)); + return 0; +} + + + + diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7a0966df57..afba1190ba 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -78,7 +78,6 @@ int32_t tsCompressColData = -1; int32_t tsCompatibleModel = 1; // client -int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; @@ -594,16 +593,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "maxSQLLength"; - cfg.ptr = &tsMaxSQLStringLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MAX_SQL_LEN; - cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - cfg.option = "maxWildCardsLength"; cfg.ptr = &tsMaxWildCardsLen; cfg.valType = TAOS_CFG_VTYPE_INT32; @@ -1080,9 +1069,7 @@ static void doInitGlobalConfig(void) { void taosInitGlobalCfg() { pthread_once(&tsInitGlobalCfgOnce, doInitGlobalConfig); } int32_t taosCheckAndPrintCfg() { - char fqdn[TSDB_FQDN_LEN]; - uint16_t port; - + SEp ep = {0}; if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { taosSetAllDebugFlag(); } @@ -1097,15 +1084,15 @@ int32_t taosCheckAndPrintCfg() { if (tsFirst[0] == 0) { strcpy(tsFirst, tsLocalEp); } else { - taosGetFqdnPortFromEp(tsFirst, fqdn, &port); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", fqdn, port); + taosGetFqdnPortFromEp(tsFirst, &ep); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", ep.fqdn, ep.port); } if (tsSecond[0] == 0) { strcpy(tsSecond, tsLocalEp); } else { - taosGetFqdnPortFromEp(tsSecond, fqdn, &port); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port); + taosGetFqdnPortFromEp(tsSecond, &ep); + snprintf(tsSecond, sizeof(tsSecond), "%s:%u", ep.fqdn, ep.port); } taosCheckDataDirCfg(); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 48e9dce3c1..3fabd2ce0d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -91,13 +91,11 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { int32_t kvNum = taosHashGetSize(pReq->info); tlen += taosEncodeFixedI32(buf, kvNum); - SKv kv; + SKv *kv; void* pIter = taosHashIterate(pReq->info, NULL); while (pIter != NULL) { - taosHashGetKey(pIter, &kv.key, (size_t *)&kv.keyLen); - kv.valueLen = taosHashGetDataLen(pIter); - kv.value = pIter; - tlen += taosEncodeSKv(buf, &kv); + kv = pIter; + tlen += taosEncodeSKv(buf, kv); pIter = taosHashIterate(pReq->info, pIter); } @@ -116,7 +114,7 @@ void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { for(int i = 0; i < kvNum; i++) { SKv kv; buf = taosDecodeSKv(buf, &kv); - taosHashPut(pReq->info, kv.key, kv.keyLen, kv.value, kv.valueLen); + taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); } return buf; @@ -124,17 +122,28 @@ void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp) { int tlen = 0; + int32_t kvNum = taosArrayGetSize(pRsp->info); tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey); tlen += taosEncodeFixedI32(buf, pRsp->status); - tlen += taosEncodeFixedI32(buf, pRsp->bodyLen); - tlen += taosEncodeBinary(buf, pRsp->body, pRsp->bodyLen); + tlen += taosEncodeFixedI32(buf, kvNum); + for (int i = 0; i < kvNum; i++) { + SKv *kv = (SKv *)taosArrayGet(pRsp->info, i); + tlen += taosEncodeSKv(buf, kv); + } return tlen; } void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp) { + int32_t kvNum = 0; buf = taosDecodeSClientHbKey(buf, &pRsp->connKey); buf = taosDecodeFixedI32(buf, &pRsp->status); - buf = taosDecodeFixedI32(buf, &pRsp->bodyLen); - buf = taosDecodeBinary(buf, &pRsp->body, pRsp->bodyLen); + buf = taosDecodeFixedI32(buf, &kvNum); + pRsp->info = taosArrayInit(kvNum, sizeof(SKv)); + for (int i = 0; i < kvNum; i++) { + SKv kv = {0}; + buf = taosDecodeSKv(buf, &kv); + taosArrayPush(pRsp->info, &kv); + } + return buf; } @@ -155,6 +164,7 @@ void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) { if (pBatchReq->reqs == NULL) { pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq)); } + int32_t reqNum; buf = taosDecodeFixedI32(buf, &reqNum); for (int i = 0; i < reqNum; i++) { @@ -310,3 +320,18 @@ void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { return buf; } + +int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedU8(buf, pReq->type); + return tlen; +} + +void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { + buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeString(buf, &pReq->name); + buf = taosDecodeFixedU8(buf, &pReq->type); + return buf; +} diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 73f755bf1a..1cab413c3f 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -630,14 +630,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -#define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ - } while (0) - - void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { case TSDB_DATA_TYPE_INT: diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 7ab3f46fdb..4214dca11d 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -31,8 +31,8 @@ typedef struct { SDnode *pDnode; STaosQueue *queue; union { - SWorkerPool pool; - SMWorkerPool mpool; + SQWorkerPool pool; + SWWorkerPool mpool; }; } SDnodeWorker; @@ -109,10 +109,10 @@ typedef struct { int32_t openVnodes; int32_t totalVnodes; SRWLatch latch; - SWorkerPool queryPool; - SWorkerPool fetchPool; - SMWorkerPool syncPool; - SMWorkerPool writePool; + SQWorkerPool queryPool; + SFWorkerPool fetchPool; + SWWorkerPool syncPool; + SWWorkerPool writePool; } SVnodesMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 74fb5f1437..02dced53c2 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -289,6 +289,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { .charset = pCfg->charset, .nthreads = pCfg->numOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, + .sendReqToDnodeFp = dndSendReqToDnode }; if (vnodeInit(&vnodeOpt) != 0) { diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index b127fb1d64..0674d719b9 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -57,13 +57,13 @@ void dndGetDnodeEp(SDnode *pDnode, int32_t dnodeId, char *pEp, char *pFqdn, uint SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { if (pPort != NULL) { - *pPort = pDnodeEp->port; + *pPort = pDnodeEp->ep.port; } if (pFqdn != NULL) { - tstrncpy(pFqdn, pDnodeEp->fqdn, TSDB_FQDN_LEN); + tstrncpy(pFqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN); } if (pEp != NULL) { - snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port); + snprintf(pEp, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); } } @@ -85,12 +85,12 @@ void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { - dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]); - if (strcmp(epSet.fqdn[i], pDnode->cfg.localFqdn) == 0 && epSet.port[i] == pDnode->cfg.serverPort) { + dDebug("mnode index:%d %s:%u", i, epSet.eps[i].fqdn, epSet.eps[i].port); + if (strcmp(epSet.eps[i].fqdn, pDnode->cfg.localFqdn) == 0 && epSet.eps[i].port == pDnode->cfg.serverPort) { epSet.inUse = (i + 1) % epSet.numOfEps; } - epSet.port[i] = htons(epSet.port[i]); + epSet.eps[i].port = htons(epSet.eps[i].port); } rpcSendRedirectRsp(pReq->handle, &epSet); @@ -104,7 +104,7 @@ static void dndUpdateMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { pMgmt->mnodeEpSet = *pEpSet; for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]); + dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } taosWUnLockLatch(&pMgmt->latch); @@ -116,7 +116,7 @@ static void dndPrintDnodes(SDnode *pDnode) { dDebug("print dnode ep list, num:%d", pMgmt->dnodeEps->num); for (int32_t i = 0; i < pMgmt->dnodeEps->num; i++) { SDnodeEp *pEp = &pMgmt->dnodeEps->eps[i]; - dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->fqdn, pEp->port, pEp->isMnode); + dDebug("dnode:%d, fqdn:%s port:%u isMnode:%d", pEp->id, pEp->ep.fqdn, pEp->ep.port, pEp->isMnode); } } @@ -145,8 +145,8 @@ static void dndResetDnodes(SDnode *pDnode, SDnodeEps *pDnodeEps) { if (!pDnodeEp->isMnode) continue; if (mIndex >= TSDB_MAX_REPLICA) continue; pMgmt->mnodeEpSet.numOfEps++; - strcpy(pMgmt->mnodeEpSet.fqdn[mIndex], pDnodeEp->fqdn); - pMgmt->mnodeEpSet.port[mIndex] = pDnodeEp->port; + + pMgmt->mnodeEpSet.eps[mIndex] = pDnodeEp->ep; mIndex++; } @@ -167,7 +167,7 @@ static bool dndIsEpChanged(SDnode *pDnode, int32_t dnodeId, char *pEp) { SDnodeEp *pDnodeEp = taosHashGet(pMgmt->dnodeHash, &dnodeId, sizeof(int32_t)); if (pDnodeEp != NULL) { char epstr[TSDB_EP_LEN + 1]; - snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->fqdn, pDnodeEp->port); + snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); changed = strcmp(pEp, epstr) != 0; } @@ -251,11 +251,12 @@ static int32_t dndReadDnodes(SDnode *pDnode) { SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i]; - cJSON *dnodeId = cJSON_GetObjectItem(node, "id"); - if (!dnodeId || dnodeId->type != cJSON_Number) { + cJSON *did = cJSON_GetObjectItem(node, "id"); + if (!did || did->type != cJSON_Number) { dError("failed to read %s since dnodeId not found", pMgmt->file); goto PRASE_DNODE_OVER; } + pDnodeEp->id = dnodeId->valueint; cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); @@ -263,14 +264,15 @@ static int32_t dndReadDnodes(SDnode *pDnode) { dError("failed to read %s since dnodeFqdn not found", pMgmt->file); goto PRASE_DNODE_OVER; } - tstrncpy(pDnodeEp->fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); + tstrncpy(pDnodeEp->ep.fqdn, dnodeFqdn->valuestring, TSDB_FQDN_LEN); cJSON *dnodePort = cJSON_GetObjectItem(node, "port"); if (!dnodePort || dnodePort->type != cJSON_Number) { dError("failed to read %s since dnodePort not found", pMgmt->file); goto PRASE_DNODE_OVER; } - pDnodeEp->port = dnodePort->valueint; + + pDnodeEp->ep.port = dnodePort->valueint; cJSON *isMnode = cJSON_GetObjectItem(node, "isMnode"); if (!isMnode || isMnode->type != cJSON_Number) { @@ -298,7 +300,8 @@ PRASE_DNODE_OVER: pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp)); pMgmt->dnodeEps->num = 1; pMgmt->dnodeEps->eps[0].isMnode = 1; - taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); + + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, &(pMgmt->dnodeEps->eps[0].ep)); } dndResetDnodes(pDnode, pMgmt->dnodeEps); @@ -329,8 +332,8 @@ static int32_t dndWriteDnodes(SDnode *pDnode) { for (int32_t i = 0; i < pMgmt->dnodeEps->num; ++i) { SDnodeEp *pDnodeEp = &pMgmt->dnodeEps->eps[i]; len += snprintf(content + len, maxLen - len, " \"id\": %d,\n", pDnodeEp->id); - len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->fqdn); - len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->port); + len += snprintf(content + len, maxLen - len, " \"fqdn\": \"%s\",\n", pDnodeEp->ep.fqdn); + len += snprintf(content + len, maxLen - len, " \"port\": %u,\n", pDnodeEp->ep.port); len += snprintf(content + len, maxLen - len, " \"isMnode\": %d\n", pDnodeEp->isMnode); if (i < pMgmt->dnodeEps->num - 1) { len += snprintf(content + len, maxLen - len, " },{\n"); @@ -395,7 +398,7 @@ void dndSendStatusReq(SDnode *pDnode) { static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); + dInfo("set dnodeId:%d clusterId:0x%" PRId64, pCfg->dnodeId, pCfg->clusterId); taosWLockLatch(&pMgmt->latch); pMgmt->dnodeId = pCfg->dnodeId; pMgmt->clusterId = pCfg->clusterId; @@ -450,7 +453,7 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { pDnodeEps->num = htonl(pDnodeEps->num); for (int32_t i = 0; i < pDnodeEps->num; ++i) { pDnodeEps->eps[i].id = htonl(pDnodeEps->eps[i].id); - pDnodeEps->eps[i].port = htons(pDnodeEps->eps[i].port); + pDnodeEps->eps[i].ep.port = htons(pDnodeEps->eps[i].ep.port); } dndUpdateDnodeEps(pDnode, pDnodeEps); @@ -529,7 +532,7 @@ int32_t dndInitMgmt(SDnode *pDnode) { } if (pMgmt->dropped) { - dError("dnode will not start for its already dropped"); + dError("dnode not start since its already dropped"); return -1; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 275b09fc0f..257c72bff5 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -122,6 +122,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH_RSP)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_TABLE_META)] = dndProcessVnodeFetchMsg; @@ -148,6 +149,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_CONSUME)] = dndProcessVnodeFetchMsg; } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index f487859fd6..c4d14ef697 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "dndVnodes.h" #include "dndTransport.h" +#include "dndMgmt.h" typedef struct { int32_t vgId; @@ -253,7 +254,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ } for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + cJSON * vnode = cJSON_GetArrayItem(vnodes, i); SWrapperCfg *pCfg = &pCfgs[i]; cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); @@ -382,7 +383,7 @@ static void *dnodeOpenVnodeFunc(void *param) { dndReportStartup(pDnode, "open-vnodes", stepDesc); SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId}; - SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); + SVnode * pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; @@ -527,7 +528,6 @@ static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; pCfg->wsize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize; - pCfg->wsize = pCreate->cacheBlockSize; pCfg->lsize = pCreate->cacheBlockSize; pCfg->isHeapAllocator = true; pCfg->ttl = 4; @@ -578,6 +578,12 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { SWrapperCfg wrapperCfg = {0}; dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg); + if (pCreate->dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; + dDebug("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); + return -1; + } + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId); if (pVnode != NULL) { dDebug("vgId:%d, already exist", pCreate->vgId); @@ -910,27 +916,27 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); - SWorkerPool *pPool = &pMgmt->queryPool; - pPool->name = "vnode-query"; - pPool->min = minQueryThreads; - pPool->max = maxQueryThreads; - if (tWorkerInit(pPool) != 0) return -1; + SQWorkerPool *pQPool = &pMgmt->queryPool; + pQPool->name = "vnode-query"; + pQPool->min = minQueryThreads; + pQPool->max = maxQueryThreads; + if (tQWorkerInit(pQPool) != 0) return -1; - pPool = &pMgmt->fetchPool; - pPool->name = "vnode-fetch"; - pPool->min = minFetchThreads; - pPool->max = maxFetchThreads; - if (tWorkerInit(pPool) != 0) return -1; + SFWorkerPool *pFPool = &pMgmt->fetchPool; + pFPool->name = "vnode-fetch"; + pFPool->min = minFetchThreads; + pFPool->max = maxFetchThreads; + if (tFWorkerInit(pFPool) != 0) return -1; - SMWorkerPool *pMPool = &pMgmt->writePool; - pMPool->name = "vnode-write"; - pMPool->max = maxWriteThreads; - if (tMWorkerInit(pMPool) != 0) return -1; + SWWorkerPool *pWPool = &pMgmt->writePool; + pWPool->name = "vnode-write"; + pWPool->max = maxWriteThreads; + if (tWWorkerInit(pWPool) != 0) return -1; - pMPool = &pMgmt->syncPool; - pMPool->name = "vnode-sync"; - pMPool->max = maxSyncThreads; - if (tMWorkerInit(pMPool) != 0) return -1; + pWPool = &pMgmt->syncPool; + pWPool->name = "vnode-sync"; + pWPool->max = maxSyncThreads; + if (tWWorkerInit(pWPool) != 0) return -1; dDebug("vnode workers is initialized"); return 0; @@ -938,21 +944,21 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { static void dndCleanupVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerCleanup(&pMgmt->fetchPool); - tWorkerCleanup(&pMgmt->queryPool); - tMWorkerCleanup(&pMgmt->writePool); - tMWorkerCleanup(&pMgmt->syncPool); + tFWorkerCleanup(&pMgmt->fetchPool); + tQWorkerCleanup(&pMgmt->queryPool); + tWWorkerCleanup(&pMgmt->writePool); + tWWorkerCleanup(&pMgmt->syncPool); dDebug("vnode workers is closed"); } static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue); - pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue); - pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); - pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue); - pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue); + pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeWriteQueue); + pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeApplyQueue); + pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)dndProcessVnodeSyncQueue); + pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)dndProcessVnodeFetchQueue); + pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)dndProcessVnodeQueryQueue); if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || pVnode->pQueryQ == NULL) { @@ -965,11 +971,11 @@ static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); - tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); + tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); pVnode->pWriteQ = NULL; pVnode->pApplyQ = NULL; pVnode->pSyncQ = NULL; diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c index e0db262f89..5ccf6640c0 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/impl/src/dndWorker.c @@ -31,28 +31,28 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c pWorker->pDnode = pDnode; if (pWorker->type == DND_WORKER_SINGLE) { - SWorkerPool *pPool = &pWorker->pool; + SQWorkerPool *pPool = &pWorker->pool; pPool->name = name; pPool->min = minNum; pPool->max = maxNum; - if (tWorkerInit(pPool) != 0) { + if (tQWorkerInit(pPool) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)queueFp); + pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } else if (pWorker->type == DND_WORKER_MULTI) { - SMWorkerPool *pPool = &pWorker->mpool; + SWWorkerPool *pPool = &pWorker->mpool; pPool->name = name; pPool->max = maxNum; - if (tMWorkerInit(pPool) != 0) { + if (tWWorkerInit(pPool) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tMWorkerAllocQueue(pPool, pDnode, (FProcessItems)queueFp); + pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -70,11 +70,11 @@ void dndCleanupWorker(SDnodeWorker *pWorker) { } if (pWorker->type == DND_WORKER_SINGLE) { - tWorkerCleanup(&pWorker->pool); - tWorkerFreeQueue(&pWorker->pool, pWorker->queue); + tQWorkerCleanup(&pWorker->pool); + tQWorkerFreeQueue(&pWorker->pool, pWorker->queue); } else if (pWorker->type == DND_WORKER_MULTI) { - tMWorkerCleanup(&pWorker->mpool); - tMWorkerFreeQueue(&pWorker->mpool, pWorker->queue); + tWWorkerCleanup(&pWorker->mpool); + tWWorkerFreeQueue(&pWorker->mpool, pWorker->queue); } else { } } diff --git a/source/dnode/mgmt/impl/test/sut/src/client.cpp b/source/dnode/mgmt/impl/test/sut/src/client.cpp index 3d61db8268..8403dbf034 100644 --- a/source/dnode/mgmt/impl/test/sut/src/client.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/client.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tep.h" #include "sut.h" static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { @@ -61,11 +62,7 @@ void TestClient::Cleanup() { SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) { SEpSet epSet = {0}; - epSet.inUse = 0; - epSet.numOfEps = 1; - epSet.port[0] = port; - memcpy(epSet.fqdn[0], fqdn, TSDB_FQDN_LEN); - + addEpIntoEpSet(&epSet, fqdn, port); rpcSendRequest(clientRpc, &epSet, pReq, NULL); tsem_wait(&sem); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index c6e6c0006d..dc8390fba3 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -125,7 +125,7 @@ const char* Testbase::GetMetaName(int32_t index) { int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } -const char* Testbase::GetMetaTbName() { return pMeta->tbFname; } +const char* Testbase::GetMetaTbName() { return pMeta->tbName; } void Testbase::SendShowRetrieveReq() { int32_t contLen = sizeof(SRetrieveTableReq); @@ -144,7 +144,7 @@ void Testbase::SendShowRetrieveReq() { pos = 0; } -const char* Testbase::GetShowName() { return pMeta->tbFname; } +const char* Testbase::GetShowName() { return pMeta->tbName; } int8_t Testbase::GetShowInt8() { int8_t data = *((int8_t*)(pData + pos)); diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 1799bf66b0..11b32fbf0f 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -68,6 +68,44 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED); } } + + { + int32_t contLen = sizeof(SCreateVnodeReq); + + SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(3); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(1); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9527); + } + + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_INVALID_OPTION); + } } TEST_F(DndTestVnode, 02_ALTER_Vnode) { @@ -161,19 +199,16 @@ TEST_F(DndTestVnode, 03_Create_Stb) { req.stbCfg.nTagCols = 3; req.stbCfg.pTagSchema = &schemas[2]; - int32_t bsize = tSerializeSVCreateTbReq(NULL, &req); - void* buf = rpcMallocCont(sizeof(SMsgHead) + bsize); - SMsgHead* pMsgHead = (SMsgHead*)buf; + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(2); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); - void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - int32_t contLen = sizeof(SMsgHead) + bsize; - - SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, buf, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); if (i == 0) { ASSERT_EQ(pRsp->code, 0); @@ -197,20 +232,28 @@ TEST_F(DndTestVnode, 04_ALTER_Stb) { } TEST_F(DndTestVnode, 05_DROP_Stb) { -#if 0 { for (int i = 0; i < 3; ++i) { - SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, pReq, contLen); + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char*)"stb1"; + req.suid = 9599; + req.type = TD_SUPER_TABLE; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); + + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); - if (i == 0) { - ASSERT_EQ(pRsp->code, 0); - test.Restart(); - } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_INVALID_TABLE_ID); - } + ASSERT_EQ(pRsp->code, 0); } } -#endif } TEST_F(DndTestVnode, 06_DROP_Vnode) { diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 91f502be7d..5a3e6ed26e 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -26,6 +26,7 @@ int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); SDbObj *mndAcquireDb(SMnode *pMnode, char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); +int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 02ce3a1591..e8a9a68466 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -128,6 +128,8 @@ typedef struct { int32_t failedTimes; void* rpcHandle; void* rpcAHandle; + void* rpcRsp; + int32_t rpcRspLen; SArray* redoLogs; SArray* undoLogs; SArray* commitLogs; @@ -363,9 +365,7 @@ typedef struct SMqConsumerEp { int64_t consumerId; // -1 for unassigned int64_t lastConsumerHbTs; int64_t lastVgHbTs; - uint32_t qmsgLen; char* qmsg; - //SSubQueryMsg qExec; } SMqConsumerEp; static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { @@ -374,9 +374,10 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon tlen += taosEncodeFixedI32(buf, pConsumerEp->status); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); + tlen += taosEncodeFixedI64(buf, pConsumerEp->lastConsumerHbTs); + tlen += taosEncodeFixedI64(buf, pConsumerEp->lastVgHbTs); //tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); - tlen += taosEncodeFixedU32(buf, pConsumerEp->qmsgLen); - tlen += taosEncodeBinary(buf, pConsumerEp->qmsg, pConsumerEp->qmsgLen); + tlen += taosEncodeString(buf, pConsumerEp->qmsg); return tlen; } @@ -385,12 +386,19 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu buf = taosDecodeFixedI32(buf, &pConsumerEp->status); buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); + buf = taosDecodeFixedI64(buf, &pConsumerEp->lastConsumerHbTs); + buf = taosDecodeFixedI64(buf, &pConsumerEp->lastVgHbTs); //buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); - buf = taosDecodeFixedU32(buf, &pConsumerEp->qmsgLen); - buf = taosDecodeBinary(buf, (void**)&pConsumerEp->qmsg, pConsumerEp->qmsgLen); + buf = taosDecodeString(buf, &pConsumerEp->qmsg); return buf; } +static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { + if (pConsumerEp) { + tfree(pConsumerEp->qmsg); + } +} + // unit for rebalance typedef struct SMqSubscribeObj { char key[TSDB_SUBSCRIBE_KEY_LEN]; @@ -423,18 +431,27 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { free(pSub); return NULL; } - pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); - if (pSub->assigned == NULL) { + pSub->lostConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); + if (pSub->lostConsumer == NULL) { taosArrayDestroy(pSub->availConsumer); - taosArrayDestroy(pSub->idleConsumer); + taosArrayDestroy(pSub->assigned); + free(pSub); + return NULL; + } + pSub->idleConsumer = taosArrayInit(0, sizeof(SMqConsumerEp)); + if (pSub->idleConsumer == NULL) { + taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); free(pSub); return NULL; } pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); - if (pSub->assigned == NULL) { + if (pSub->unassignedVg == NULL) { taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); taosArrayDestroy(pSub->idleConsumer); - taosArrayDestroy(pSub->unassignedVg); free(pSub); return NULL; } @@ -461,6 +478,13 @@ static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeOb tlen += tEncodeSMqConsumerEp(buf, pCEp); } + sz = taosArrayGetSize(pSub->lostConsumer); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp* pCEp = taosArrayGet(pSub->lostConsumer, i); + tlen += tEncodeSMqConsumerEp(buf, pCEp); + } + sz = taosArrayGetSize(pSub->idleConsumer); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { @@ -485,37 +509,67 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) int32_t sz; buf = taosDecodeFixedI32(buf, &sz); - pSub->assigned = taosArrayInit(sz, sizeof(int64_t)); - if (pSub->assigned == NULL) { + pSub->availConsumer = taosArrayInit(sz, sizeof(int64_t)); + if (pSub->availConsumer == NULL) { return NULL; } for (int32_t i = 0; i < sz; i++) { int64_t consumerId; buf = taosDecodeFixedI64(buf, &consumerId); - taosArrayPush(pSub->assigned, &consumerId); + taosArrayPush(pSub->availConsumer, &consumerId); + } + + buf = taosDecodeFixedI32(buf, &sz); + pSub->assigned = taosArrayInit(sz, sizeof(SMqConsumerEp)); + if (pSub->assigned == NULL) { + taosArrayDestroy(pSub->availConsumer); + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp cEp = {0}; + buf = tDecodeSMqConsumerEp(buf, &cEp); + taosArrayPush(pSub->assigned, &cEp); + } + + buf = taosDecodeFixedI32(buf, &sz); + pSub->lostConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); + if (pSub->lostConsumer == NULL) { + taosArrayDestroy(pSub->availConsumer); + taosArrayDestroy(pSub->assigned); + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqConsumerEp cEp = {0}; + buf = tDecodeSMqConsumerEp(buf, &cEp); + taosArrayPush(pSub->lostConsumer, &cEp); } buf = taosDecodeFixedI32(buf, &sz); pSub->idleConsumer = taosArrayInit(sz, sizeof(SMqConsumerEp)); if (pSub->idleConsumer == NULL) { + taosArrayDestroy(pSub->availConsumer); taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->idleConsumer, &cEp); } + buf = taosDecodeFixedI32(buf, &sz); pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); if (pSub->unassignedVg == NULL) { + taosArrayDestroy(pSub->availConsumer); taosArrayDestroy(pSub->assigned); + taosArrayDestroy(pSub->lostConsumer); taosArrayDestroy(pSub->idleConsumer); return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->unassignedVg, &cEp); } @@ -523,6 +577,33 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return buf; } +static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { + if (pSub->availConsumer) { + taosArrayDestroy(pSub->availConsumer); + pSub->availConsumer = NULL; + } + if (pSub->assigned) { + //taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->assigned); + pSub->assigned = NULL; + } + if (pSub->unassignedVg) { + //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->unassignedVg); + pSub->unassignedVg = NULL; + } + if (pSub->idleConsumer) { + //taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->idleConsumer); + pSub->idleConsumer = NULL; + } + if (pSub->lostConsumer) { + //taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->lostConsumer); + pSub->lostConsumer = NULL; + } +} + typedef struct SMqCGroup { char name[TSDB_CONSUMER_GROUP_LEN]; int32_t status; // 0 - uninitialized, 1 - wait rebalance, 2- normal @@ -536,7 +617,7 @@ typedef struct SMqTopicObj { int64_t createTime; int64_t updateTime; uint64_t uid; - uint64_t dbUid; + int64_t dbUid; int32_t version; SRWLatch lock; int32_t sqlLen; @@ -556,7 +637,7 @@ typedef struct SMqConsumerTopic { } SMqConsumerTopic; static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, - SMqSubscribeObj* pSub) { + SMqSubscribeObj* pSub, int64_t* oldConsumerId) { SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic)); if (pCTopic == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -569,6 +650,7 @@ static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqT int32_t unassignedVgSz = taosArrayGetSize(pSub->unassignedVg); if (unassignedVgSz > 0) { SMqConsumerEp* pCEp = taosArrayPop(pSub->unassignedVg); + *oldConsumerId = pCEp->consumerId; pCEp->consumerId = consumerId; taosArrayPush(pCTopic->pVgInfo, &pCEp->vgId); taosArrayPush(pSub->assigned, pCEp); @@ -580,7 +662,10 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerTopic(void** buf, SMqConsumerTopic int32_t tlen = 0; tlen += taosEncodeString(buf, pConsumerTopic->name); tlen += taosEncodeFixedI32(buf, pConsumerTopic->epoch); - int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); + int32_t sz = 0; + if (pConsumerTopic->pVgInfo != NULL) { + sz = taosArrayGetSize(pConsumerTopic->pVgInfo); + } tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { int32_t* pVgInfo = taosArrayGet(pConsumerTopic->pVgInfo, i); @@ -609,12 +694,17 @@ typedef struct SMqConsumerObj { SRWLatch lock; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray - // SHashObj *topicHash; //SHashObj + int64_t epoch; + // stat + int64_t pollCnt; } SMqConsumerObj; static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeFixedI64(buf, pConsumer->connId); + tlen += taosEncodeFixedI64(buf, pConsumer->epoch); + tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt); tlen += taosEncodeString(buf, pConsumer->cgroup); int32_t sz = taosArrayGetSize(pConsumer->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -627,6 +717,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerO static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) { buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeFixedI64(buf, &pConsumer->connId); + buf = taosDecodeFixedI64(buf, &pConsumer->epoch); + buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt); buf = taosDecodeStringTo(buf, pConsumer->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 48847dc6a3..0855b5bd4d 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -28,6 +28,9 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen); + + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index fda3fed13d..f1a213790c 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -36,16 +36,17 @@ typedef struct { int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, SRpcMsg *pMsg); +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, const SRpcMsg *pReq); void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); +void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); -void mndTransProcessRsp(SMnodeMsg *pMsg); +void mndTransProcessRsp(SMnodeMsg *pRsp); void mndTransPullup(SMnode *pMnode); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 189cbfea6d..c754494c24 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -426,7 +426,7 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_BNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 5f80672369..a53298efb3 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -186,7 +186,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp cols++; pMeta->numOfColumns = htonl(cols); - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -196,7 +196,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp pShow->numOfRows = 1; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index e642b578fa..fd960eb63e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -54,13 +55,14 @@ void mndCleanupConsumer(SMnode *pMnode) {} SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; @@ -75,6 +77,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_SUCCESS; CM_ENCODE_OVER: + tfree(buf); if (terrno != 0) { mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -87,6 +90,7 @@ CM_ENCODE_OVER: SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; @@ -105,7 +109,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); - void *buf = malloc(len); + buf = malloc(len); if (buf == NULL) goto CM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); @@ -117,6 +121,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; CM_DECODE_OVER: + tfree(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); tfree(pRow); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 39be41a4e5..1bfe8c9d9a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -226,10 +226,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1; if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1; if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1; - if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1; + if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1; if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1; if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1; if (pCfg->minRows < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRows > TSDB_MAX_MIN_ROW_FBLOCK) return -1; @@ -498,7 +498,7 @@ static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbReq *pAlter) { +static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { @@ -649,7 +649,7 @@ static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - int32_t code = mndSetDbCfgFromAlterDbMsg(&dbObj, pAlter); + int32_t code = mndSetDbCfgFromAlterDbReq(&dbObj, pAlter); if (code != 0) { mndReleaseDb(pMnode, pDb); mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); @@ -767,6 +767,14 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; + + int32_t rspLen = sizeof(SDropDbRsp); + SDropDbRsp *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) goto DROP_DB_OVER; + memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); + pRsp->uid = htobe64(pDb->uid); + mndTransSetRpcRsp(pTrans, pRsp, rspLen); + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; code = 0; @@ -805,6 +813,44 @@ static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgList, int32_t *vgNum) { + int32_t vindex = 0; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + while (vindex < pDb->cfg.numOfVgroups) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + if (pVgroup->dbUid == pDb->uid) { + SVgroupInfo *pInfo = &vgList[vindex]; + pInfo->vgId = htonl(pVgroup->vgId); + pInfo->hashBegin = htonl(pVgroup->hashBegin); + pInfo->hashEnd = htonl(pVgroup->hashEnd); + pInfo->epset.numOfEps = pVgroup->replica; + for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { + SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; + SEp * pEp = &pInfo->epset.eps[gid]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode != NULL) { + memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + pEp->port = htons(pDnode->port); + } + mndReleaseDnode(pMnode, pDnode); + if (pVgid->role == TAOS_SYNC_STATE_LEADER) { + pInfo->epset.inUse = gid; + } + } + vindex++; + } + + sdbRelease(pSdb, pVgroup); + } + + *vgNum = vindex; +} + static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; @@ -826,45 +872,16 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { return -1; } - int32_t vindex = 0; + int32_t vgNum = 0; if (pUse->vgVersion < pDb->vgVersion) { - void *pIter = NULL; - while (vindex < pDb->cfg.numOfVgroups) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - if (pVgroup->dbUid == pDb->uid) { - SVgroupInfo *pInfo = &pRsp->vgroupInfo[vindex]; - pInfo->vgId = htonl(pVgroup->vgId); - pInfo->hashBegin = htonl(pVgroup->hashBegin); - pInfo->hashEnd = htonl(pVgroup->hashEnd); - pInfo->numOfEps = pVgroup->replica; - for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEpAddr *pEpArrr = &pInfo->epAddr[gid]; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode != NULL) { - memcpy(pEpArrr->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - pEpArrr->port = htons(pDnode->port); - } - mndReleaseDnode(pMnode, pDnode); - if (pVgid->role == TAOS_SYNC_STATE_LEADER) { - pInfo->inUse = gid; - } - } - vindex++; - } - - sdbRelease(pSdb, pVgroup); - } + mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); } memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); pRsp->uid = htobe64(pDb->uid); pRsp->vgVersion = htonl(pDb->vgVersion); - pRsp->vgNum = htonl(vindex); + pRsp->vgNum = htonl(vgNum); pRsp->hashMethod = pDb->hashMethod; pReq->pCont = pRsp; @@ -874,6 +891,77 @@ static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { return 0; } +int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen) { + SSdb *pSdb = pMnode->pSdb; + int32_t bufSize = num * (sizeof(SUseDbRsp) + TSDB_DEFAULT_VN_PER_DB * sizeof(SVgroupInfo)); + void *buf = malloc(bufSize); + int32_t len = 0; + int32_t contLen = 0; + int32_t bufOffset = 0; + SUseDbRsp *pRsp = NULL; + + for (int32_t i = 0; i < num; ++i) { + SDbVgVersion *db = &dbs[i]; + db->dbId = be64toh(db->dbId); + db->vgVersion = ntohl(db->vgVersion); + + len = 0; + + SDbObj *pDb = mndAcquireDb(pMnode, db->dbFName); + if (pDb == NULL) { + mInfo("db %s not exist", db->dbFName); + + len = sizeof(SUseDbRsp); + } else if (pDb->uid != db->dbId || db->vgVersion < pDb->vgVersion) { + len = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); + } + + if (0 == len) { + mndReleaseDb(pMnode, pDb); + + continue; + } + + contLen += len; + + if (contLen > bufSize) { + buf = realloc(buf, contLen); + } + + pRsp = (SUseDbRsp *)((char *)buf + bufOffset); + memcpy(pRsp->db, db->dbFName, TSDB_DB_FNAME_LEN); + if (pDb) { + int32_t vgNum = 0; + mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); + + pRsp->uid = htobe64(pDb->uid); + pRsp->vgVersion = htonl(pDb->vgVersion); + pRsp->vgNum = htonl(vgNum); + pRsp->hashMethod = pDb->hashMethod; + } else { + pRsp->uid = htobe64(db->dbId); + pRsp->vgNum = htonl(0); + pRsp->hashMethod = 0; + pRsp->vgVersion = htonl(-1); + } + + bufOffset += len; + + mndReleaseDb(pMnode, pDb); + } + + if (contLen > 0) { + *rsp = buf; + *rspLen = contLen; + } else { + *rsp = NULL; + tfree(buf); + *rspLen = 0; + } + + return 0; +} + static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; SSyncDbReq *pSync = pReq->rpcMsg.pCont; @@ -1025,7 +1113,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMe pShow->numOfRows = sdbGetSize(pSdb, SDB_DB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -1143,7 +1231,7 @@ static int32_t mndRetrieveDbs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 prec = TSDB_TIME_PRECISION_NANO_STR; break; default: - assert(false); + prec = "none"; break; } STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index efdacf957b..3b43cd8081 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -203,8 +203,8 @@ void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode) { } SEpSet mndGetDnodeEpset(SDnodeObj *pDnode) { - SEpSet epSet = {.inUse = 0, .numOfEps = 1, .port[0] = pDnode->port}; - memcpy(epSet.fqdn[0], pDnode->fqdn, TSDB_FQDN_LEN); + SEpSet epSet = {0}; + addEpIntoEpSet(&epSet, pDnode->fqdn, pDnode->port); return epSet; } @@ -261,8 +261,8 @@ static void mndGetDnodeData(SMnode *pMnode, SDnodeEps *pEps, int32_t maxEps) { SDnodeEp *pEp = &pEps->eps[numOfEps]; pEp->id = htonl(pDnode->id); - pEp->port = htons(pDnode->port); - memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + pEp->ep.port = htons(pDnode->port); + memcpy(pEp->ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN); pEp->isMnode = 0; if (mndIsMnode(pMnode, pDnode->id)) { pEp->isMnode = 1; @@ -608,7 +608,7 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp pShow->numOfRows = TSDB_CONFIG_NUMBER; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -715,7 +715,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index f9dde7bc75..b3fc0e1da0 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -20,6 +20,7 @@ #include "mndTrans.h" #define SDB_FUNC_VER 1 +#define SDB_FUNC_RESERVE_SIZE 64 static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); @@ -60,7 +61,7 @@ void mndCleanupFunc(SMnode *pMnode) {} static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj); + int32_t size = pFunc->commentSize + pFunc->codeSize + sizeof(SFuncObj) + SDB_FUNC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_FUNC, SDB_FUNC_VER, size); if (pRaw == NULL) goto FUNC_ENCODE_OVER; @@ -78,6 +79,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, FUNC_ENCODE_OVER); terrno = 0; @@ -131,6 +133,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, FUNC_DECODE_OVER) terrno = 0; @@ -479,7 +482,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 75ed5b0a1e..dde48853ad 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -237,13 +237,11 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { if (pIter == NULL) break; if (pObj->pDnode == NULL) break; - pEpSet->port[pEpSet->numOfEps] = htons(pObj->pDnode->port); - memcpy(pEpSet->fqdn[pEpSet->numOfEps], pObj->pDnode->fqdn, TSDB_FQDN_LEN); if (pObj->role == TAOS_SYNC_STATE_LEADER) { pEpSet->inUse = pEpSet->numOfEps; } - pEpSet->numOfEps++; + addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, htons(pObj->pDnode->port)); sdbRelease(pSdb, pObj); } } @@ -626,7 +624,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); mndUpdateMnodeRole(pMnode); return 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 76fabc96ce..d63ade4320 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,14 +14,16 @@ */ #define _DEFAULT_SOURCE +#include "tglobal.h" #include "mndProfile.h" -#include "mndConsumer.h" +//#include "mndConsumer.h" #include "mndDb.h" +#include "mndStb.h" #include "mndMnode.h" #include "mndShow.h" -#include "mndTopic.h" +//#include "mndTopic.h" #include "mndUser.h" -#include "mndVgroup.h" +//#include "mndVgroup.h" #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 @@ -230,10 +232,12 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { goto CONN_OVER; } - pRsp->acctId = htonl(pUser->acctId); + pRsp->acctId = htonl(pUser->acctId); pRsp->superUser = pUser->superUser; pRsp->clusterId = htobe64(pMnode->clusterId); - pRsp->connId = htonl(pConn->id); + pRsp->connId = htonl(pConn->id); + + snprintf(pRsp->sVersion, tListLen(pRsp->sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); mndGetMnodeEpSet(pMnode, &pRsp->epSet); pReq->contLen = sizeof(SConnectRsp); @@ -351,7 +355,48 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { for (int i = 0; i < sz; i++) { SClientHbReq* pHbReq = taosArrayGet(pArray, i); if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { + int32_t kvNum = taosHashGetSize(pHbReq->info); + if (NULL == pHbReq->info || kvNum <= 0) { + continue; + } + SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = taosArrayInit(kvNum, sizeof(SKv))}; + + void *pIter = taosHashIterate(pHbReq->info, NULL); + while (pIter != NULL) { + SKv* kv = pIter; + + switch (kv->key) { + case HEARTBEAT_KEY_DBINFO: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateDBInfo(pMnode, (SDbVgVersion *)kv->value, kv->valueLen/sizeof(SDbVgVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv); + } + break; + } + case HEARTBEAT_KEY_STBINFO: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateStbInfo(pMnode, (SSTableMetaVersion *)kv->value, kv->valueLen/sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv); + } + break; + } + default: + mError("invalid kv key:%d", kv->key); + hbRsp.status = TSDB_CODE_MND_APP_ERROR; + break; + } + + pIter = taosHashIterate(pHbReq->info, pIter); + } + + taosArrayPush(batchRsp.rsps, &hbRsp); } else if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_MQ) { SClientHbRsp *pRsp = mndMqHbBuildRsp(pMnode, pHbReq); if (pRsp != NULL) { @@ -366,6 +411,18 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { void* buf = rpcMallocCont(tlen); void* abuf = buf; tSerializeSClientHbBatchRsp(&abuf, &batchRsp); + + int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); + for (int32_t i = 0; i < rspNum; ++i) { + SClientHbRsp *rsp = taosArrayGet(batchRsp.rsps, i); + int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info): 0; + for (int32_t n = 0; n < kvNum; ++n) { + SKv *kv = taosArrayGet(rsp->info, n); + tfree(kv->value); + } + taosArrayDestroy(rsp->info); + } + taosArrayDestroy(batchRsp.rsps); pReq->contLen = tlen; pReq->pCont = buf; @@ -574,7 +631,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -743,7 +800,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = 1000000; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 6077a95a7b..e29ab5a5bd 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -426,7 +426,7 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_QNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 7688ea76ab..cbba659719 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -428,7 +428,7 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_SNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 45b8428663..4ccd4b63c4 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -181,23 +181,26 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); - atomic_exchange_32(&pOld->updateTime, pNew->updateTime); - atomic_exchange_32(&pOld->version, pNew->version); taosWLockLatch(&pOld->lock); - pOld->numOfColumns = pNew->numOfColumns; - pOld->numOfTags = pNew->numOfTags; int32_t totalCols = pNew->numOfTags + pNew->numOfColumns; int32_t totalSize = totalCols * sizeof(SSchema); - if (pOld->numOfTags + pOld->numOfColumns < totalCols) { void *pSchema = malloc(totalSize); if (pSchema != NULL) { free(pOld->pSchema); pOld->pSchema = pSchema; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); } } + pOld->updateTime = pNew->updateTime; + pOld->version = pNew->version; + pOld->numOfColumns = pNew->numOfColumns; + pOld->numOfTags = pNew->numOfTags; memcpy(pOld->pSchema, pNew->pSchema, totalSize); taosWUnLockLatch(&pOld->lock); return 0; @@ -228,15 +231,11 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { } static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SVCreateTbReq req; - void *buf; - int32_t bsize; - SMsgHead *pMsgHead; - - req.ver = 0; SName name = {0}; tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + SVCreateTbReq req = {0}; + req.ver = 0; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; @@ -247,40 +246,48 @@ static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb req.stbCfg.nTagCols = pStb->numOfTags; req.stbCfg.pTagSchema = pStb->pSchema + pStb->numOfColumns; - bsize = tSerializeSVCreateTbReq(NULL, &req); - buf = malloc(sizeof(SMsgHead) + bsize); - if (buf == NULL) { + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pMsgHead = (SMsgHead *)buf; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(pVgroup->vgId); - - void *pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - *pContLen = sizeof(SMsgHead) + bsize; - return buf; + *pContLen = contLen; + return pHead; } -static SVDropTbReq *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { - int32_t contLen = sizeof(SVDropTbReq); +static void *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVDropTbReq *pDrop = calloc(1, contLen); - if (pDrop == NULL) { + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char *)tNameGetTableName(&name); + req.type = TD_SUPER_TABLE; + req.suid = pStb->uid; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pDrop->head.contLen = htonl(contLen); - pDrop->head.vgId = htonl(pVgroup->vgId); - memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN); - pDrop->suid = htobe64(pStb->uid); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - return pDrop; + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; } static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { @@ -358,7 +365,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - int32_t contLen; + int32_t contLen; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); @@ -400,7 +407,8 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SVDropTbReq *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb); + int32_t contLen = 0; + void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb, &contLen); if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -411,7 +419,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pReq; - action.contLen = sizeof(SVDropTbReq); + action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -498,9 +506,9 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { } // topic should have different name with stb - SStbObj *pTopic = mndAcquireStb(pMnode, pCreate->name); - if (pTopic != NULL) { - sdbRelease(pMnode->pSdb, pTopic); + SStbObj *pTopicStb = mndAcquireStb(pMnode, pCreate->name); + if (pTopicStb != NULL) { + mndReleaseStb(pMnode, pTopicStb); terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -517,7 +525,6 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { mndReleaseDb(pMnode, pDb); if (code != 0) { - terrno = code; mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -603,15 +610,6 @@ static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pS return 0; } -static int32_t mndSetDropStbUndoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); if (pCommitRaw == NULL) return -1; @@ -621,22 +619,54 @@ static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj * return 0; } -static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } +static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; -static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; -static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) { + int32_t contLen = 0; + void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DROP_STB; + action.acceptableCode = TSDB_CODE_VND_TB_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); - if (pTrans == NULL)goto DROP_STB_OVER; + if (pTrans == NULL) goto DROP_STB_OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; + if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto DROP_STB_OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_STB_OVER; code = 0; @@ -664,7 +694,16 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { } } - int32_t code = mndDropStb(pMnode, pReq, pStb); + SDbObj *pDb = mndAcquireDbByStb(pMnode, pDrop->name); + if (pDb == NULL) { + mndReleaseStb(pMnode, pStb); + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + int32_t code = mndDropStb(pMnode, pReq, pDb, pStb); + mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); if (code != 0) { @@ -684,20 +723,23 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; STableInfoReq *pInfo = pReq->rpcMsg.pCont; - mDebug("stb:%s, start to retrieve meta", pInfo->tableFname); + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", pInfo->dbFName, pInfo->tbName); - SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->tableFname); + mDebug("stb:%s, start to retrieve meta", tbFName); + + SDbObj *pDb = mndAcquireDb(pMnode, pInfo->dbFName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to retrieve meta since %s", tbFName, terrstr()); return -1; } - SStbObj *pStb = mndAcquireStb(pMnode, pInfo->tableFname); + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); terrno = TSDB_CODE_MND_INVALID_STB; - mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); return -1; } @@ -711,11 +753,13 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); return -1; } - memcpy(pMeta->tbFname, pStb->name, TSDB_TABLE_FNAME_LEN); + strcpy(pMeta->dbFName, pStb->db); + strcpy(pMeta->tbName, pInfo->tbName); + strcpy(pMeta->stbName, pInfo->tbName); pMeta->numOfTags = htonl(pStb->numOfTags); pMeta->numOfColumns = htonl(pStb->numOfColumns); pMeta->precision = pDb->cfg.precision; @@ -740,10 +784,113 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { pReq->pCont = pMeta; pReq->contLen = contLen; - mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pStb->numOfColumns, pStb->numOfTags); + mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", tbFName, pStb->numOfColumns, pStb->numOfTags); return 0; } +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen) { + SSdb *pSdb = pMnode->pSdb; + int32_t bufSize = num * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + void *buf = malloc(bufSize); + int32_t len = 0; + int32_t contLen = 0; + STableMetaRsp *pRsp = NULL; + + for (int32_t i = 0; i < num; ++i) { + SSTableMetaVersion *stb = &stbs[i]; + stb->suid = be64toh(stb->suid); + stb->sversion = ntohs(stb->sversion); + stb->tversion = ntohs(stb->tversion); + + if ((contLen + sizeof(STableMetaRsp)) > bufSize) { + bufSize = contLen + (num -i) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + buf = realloc(buf, bufSize); + } + + pRsp = (STableMetaRsp *)((char *)buf + contLen); + + strcpy(pRsp->dbFName, stb->dbFName); + strcpy(pRsp->tbName, stb->stbName); + strcpy(pRsp->stbName, stb->stbName); + + mDebug("start to retrieve meta, db:%s, stb:%s", stb->dbFName, stb->stbName); + + SDbObj *pDb = mndAcquireDb(pMnode, stb->dbFName); + if (pDb == NULL) { + pRsp->numOfColumns = -1; + pRsp->suid = htobe64(stb->suid); + contLen += sizeof(STableMetaRsp); + mWarn("db:%s, failed to require db since %s", stb->dbFName, terrstr()); + continue; + } + + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", stb->dbFName, stb->stbName); + + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + pRsp->numOfColumns = -1; + pRsp->suid = htobe64(stb->suid); + contLen += sizeof(STableMetaRsp); + mWarn("stb:%s, failed to get meta since %s", tbFName, terrstr()); + continue; + } + + taosRLockLatch(&pStb->lock); + + if (stb->suid == pStb->uid && stb->sversion == pStb->version) { + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + continue; + } + + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + int32_t len = totalCols * sizeof(SSchema); + + contLen += sizeof(STableMetaRsp) + len; + + if (contLen > bufSize) { + bufSize = contLen + (num -i - 1) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + buf = realloc(buf, bufSize); + } + + pRsp->numOfTags = htonl(pStb->numOfTags); + pRsp->numOfColumns = htonl(pStb->numOfColumns); + pRsp->precision = pDb->cfg.precision; + pRsp->tableType = TSDB_SUPER_TABLE; + pRsp->update = pDb->cfg.update; + pRsp->sversion = htonl(pStb->version); + pRsp->suid = htobe64(pStb->uid); + pRsp->tuid = htobe64(pStb->uid); + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pRsp->pSchema[i]; + SSchema *pSrcSchema = &pStb->pSchema[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + } + + if (contLen > 0) { + *rsp = buf; + *rspLen = contLen; + } else { + *rsp = NULL; + tfree(buf); + *rspLen = 0; + } + + return 0; +} + + static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { SSdb *pSdb = pMnode->pSdb; @@ -816,7 +963,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pM pShow->numOfRows = sdbGetSize(pSdb, SDB_STB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -856,7 +1003,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 if (pStb->dbUid != pDb->uid) { if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) { - mError("Inconsistent table data, name:%s, db:%s, dbUid:%"PRIu64, pStb->name, pDb->name, pDb->uid); + mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); } sdbRelease(pSdb, pStb); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index df6a4a82f3..9352c0cd84 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndSubscribe.h" #include "mndConsumer.h" @@ -30,6 +31,8 @@ #define MND_SUBSCRIBE_VER_NUMBER 1 #define MND_SUBSCRIBE_RESERVE_SIZE 64 +#define MND_SUBSCRIBE_REBALANCE_MS 5000 + static char *mndMakeSubscribeKey(char *cgroup, char *topicName); static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); @@ -46,7 +49,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub); + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub, + int64_t oldConsumerId); int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -66,19 +70,21 @@ int32_t mndInitSubscribe(SMnode *pMnode) { static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; - SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->pCont; - SMqCMGetSubEpRsp rsp; + SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; + SMqCMGetSubEpRsp rsp = {0}; int64_t consumerId = be64toh(pReq->consumerId); + int64_t currentTs = taosGetTimestampMs(); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); if (pConsumer == NULL) { - /*terrno = */ + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; return -1; } ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); strcpy(rsp.cgroup, pReq->cgroup); rsp.consumerId = consumerId; + rsp.epoch = pConsumer->epoch; SArray *pTopics = pConsumer->topics; int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -88,27 +94,49 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, pConsumerTopic->name); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, pConsumerTopic->name); - int32_t assignedSz = taosArrayGetSize(pSub->assigned); + ASSERT(pSub); + bool found = 0; + bool changed = 0; + for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { + if (*(int64_t *)taosArrayGet(pSub->availConsumer, j) == consumerId) { + found = 1; + break; + } + } + if (found == 0) { + taosArrayPush(pSub->availConsumer, &consumerId); + } + + int32_t assignedSz = taosArrayGetSize(pSub->assigned); topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp)); for (int32_t j = 0; j < assignedSz; j++) { - SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i); + SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, j); if (pCEp->consumerId == consumerId) { - taosArrayPush(pSub->assigned, pCEp); + pCEp->lastConsumerHbTs = currentTs; + SMqSubVgEp vgEp = {.epSet = pCEp->epSet, .vgId = pCEp->vgId}; + taosArrayPush(topicEp.vgs, &vgEp); + changed = 1; } } if (taosArrayGetSize(topicEp.vgs) != 0) { taosArrayPush(rsp.topics, &topicEp); } + if (changed || found) { + SSdbRaw *pRaw = mndSubActionEncode(pSub); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + sdbWrite(pMnode->pSdb, pRaw); + } + mndReleaseSubscribe(pMnode, pSub); } int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp); - void *buf = malloc(tlen); + void *buf = rpcMallocCont(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } void *abuf = buf; tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); - //TODO: free rsp + tDeleteSMqCMGetSubEpRsp(&rsp); pMsg->pCont = buf; pMsg->contLen = tlen; return 0; @@ -120,9 +148,9 @@ static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) { i++; } key[i] = 0; - *topic = strdup(key); + *cgroup = strdup(key); key[i] = ':'; - *cgroup = strdup(&key[i + 1]); + *topic = strdup(&key[i + 1]); return 0; } @@ -131,9 +159,40 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SSdb *pSdb = pMnode->pSdb; SMqSubscribeObj *pSub = NULL; void *pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); - int sz; + int64_t currentTs = taosGetTimestampMs(); + int32_t sz; while (pIter != NULL) { - if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) { + for (int i = 0; i < taosArrayGetSize(pSub->assigned); i++) { + SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i); + int64_t consumerId = pCEp->consumerId; + if (pCEp->lastConsumerHbTs != -1 && currentTs - pCEp->lastConsumerHbTs > MND_SUBSCRIBE_REBALANCE_MS) { + // put consumer into lostConsumer + taosArrayPush(pSub->lostConsumer, pCEp); + // put vg into unassigned + taosArrayPush(pSub->unassignedVg, pCEp); + // remove from assigned + // TODO: swap with last one, reduce size and reset i + taosArrayRemove(pSub->assigned, i); + // remove from available consumer + for (int j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { + if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) { + taosArrayRemove(pSub->availConsumer, j); + break; + } + // TODO: acquire consumer, set status to unavail + } +#if 0 + SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId); + pConsumer->epoch++; + printf("current epoch %ld size %ld", pConsumer->epoch, pConsumer->topics->size); + SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + sdbWriteNotFree(pMnode->pSdb, pRaw); + mndReleaseConsumer(pMnode, pConsumer); +#endif + } + } + if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0 && taosArrayGetSize(pSub->availConsumer) > 0) { char *topic = NULL; char *cgroup = NULL; mndSplitSubscribeKey(pSub->key, &topic, &cgroup); @@ -142,50 +201,60 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { // create trans STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx); SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); + int64_t oldConsumerId = pCEp->consumerId; pCEp->consumerId = consumerId; taosArrayPush(pSub->assigned, pCEp); pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); + pConsumer->epoch++; + /*SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ + /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ + /*sdbWriteNotFree(pMnode->pSdb, pConsumerRaw);*/ + mndReleaseConsumer(pMnode, pConsumer); + // build msg - SMqSetCVgReq *pReq = malloc(sizeof(SMqSetCVgReq)); - if (pReq == NULL) { + SMqSetCVgReq req = {0}; + strcpy(req.cgroup, cgroup); + strcpy(req.topicName, topic); + req.sql = pTopic->sql; + req.logicalPlan = pTopic->logicalPlan; + req.physicalPlan = pTopic->physicalPlan; + req.qmsg = pCEp->qmsg; + req.oldConsumerId = oldConsumerId; + req.newConsumerId = consumerId; + req.vgId = pCEp->vgId; + int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + void *buf = malloc(sizeof(SMsgHead) + tlen); + if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - strcpy(pReq->cgroup, cgroup); - strcpy(pReq->topicName, topic); - pReq->sql = strdup(pTopic->sql); - pReq->logicalPlan = strdup(pTopic->logicalPlan); - pReq->physicalPlan = strdup(pTopic->physicalPlan); - pReq->qmsgLen = pCEp->qmsgLen; - /*memcpy(pReq->qmsg, pCEp->qmsg, pCEp->qmsgLen);*/ - pReq->qmsg = strdup(pCEp->qmsg); - int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq); - void *reqStr = malloc(tlen); - if (reqStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - void *abuf = reqStr; - tEncodeSMqSetCVgReq(&abuf, pReq); + SMsgHead *pMsgHead = (SMsgHead *)buf; + + pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); + pMsgHead->vgId = htonl(pCEp->vgId); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSMqSetCVgReq(&abuf, &req); // persist msg + // TODO: no need for txn STransAction action = {0}; action.epSet = pCEp->epSet; - action.pCont = reqStr; - action.contLen = tlen; + action.pCont = buf; + action.contLen = sizeof(SMsgHead) + tlen; action.msgType = TDMT_VND_MQ_SET_CONN; mndTransAppendRedoAction(pTrans, &action); // persist raw SSdbRaw *pRaw = mndSubActionEncode(pSub); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); - free(pReq); tfree(topic); tfree(cgroup); } @@ -195,7 +264,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); } - pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); } return 0; } @@ -203,38 +272,49 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { // convert phyplan to dag SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); - SArray *pArray; + SArray *pArray = NULL; SArray *inner = taosArrayGet(pDag->pSubplans, 0); SSubplan *plan = taosArrayGetP(inner, 0); - plan->execNode.inUse = 0; - strcpy(plan->execNode.epAddr[0].fqdn, "localhost"); - plan->execNode.epAddr[0].port = 6030; - plan->execNode.nodeId = 2; - plan->execNode.numOfEps = 1; + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; - if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - return -1; - } - int32_t sz = taosArrayGetSize(pArray); - // convert dag to msg - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp CEp; + void *pIter = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pTopic->dbUid) continue; + + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup); + + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql); + return -1; + } + if (pArray && taosArrayGetSize(pArray) != 1) { + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + mError("unsupport topic: %s, sql: %s, plan level: %ld", pTopic->name, pTopic->sql, taosArrayGetSize(pArray)); + return -1; + } + + SMqConsumerEp CEp = {0}; CEp.status = 0; + CEp.consumerId = -1; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; - STaskInfo *pTaskInfo = taosArrayGet(pArray, i); - tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); - /*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], - * CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/ + STaskInfo *pTaskInfo = taosArrayGet(pArray, 0); + CEp.epSet = pTaskInfo->addr.epset; CEp.vgId = pTaskInfo->addr.nodeId; + + ASSERT(CEp.vgId == pVgroup->vgId); CEp.qmsg = strdup(pTaskInfo->msg->msg); - CEp.qmsgLen = strlen(CEp.qmsg) + 1; - printf("abc:\n%s\n", CEp.qmsg); - /*CEp.qmsg = malloc(CEp.qmsgLen);*/ - /*if (CEp.qmsg == NULL) {*/ - /*return -1;*/ - /*}*/ - /*memcpy(CEp.qmsg, pTaskInfo->msg->msg, pTaskInfo->msg->contentLen);*/ taosArrayPush(unassignedVg, &CEp); + // TODO: free taskInfo + taosArrayDestroy(pArray); + + /*SEpSet *pEpSet = &plan->execNode.epset;*/ + /*pEpSet->inUse = 0;*/ + /*addEpIntoEpSet(pEpSet, "localhost", 6030);*/ } /*qDestroyQueryDag(pDag);*/ @@ -242,14 +322,15 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp) { + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp, + int64_t oldConsumerId) { int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); for (int32_t i = 0; i < sz; i++) { int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i); SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); SMqSetCVgReq req = { .vgId = vgId, - .oldConsumerId = -1, + .oldConsumerId = oldConsumerId, .newConsumerId = pConsumer->consumerId, }; strcpy(req.cgroup, pConsumer->cgroup); @@ -257,8 +338,7 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume req.sql = pTopic->sql; req.logicalPlan = pTopic->logicalPlan; req.physicalPlan = pTopic->physicalPlan; - req.qmsg = strdup(pCEp->qmsg); - req.qmsgLen = strlen(req.qmsg); + req.qmsg = pCEp->qmsg; int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); void *buf = malloc(sizeof(SMsgHead) + tlen); if (buf == NULL) { @@ -293,13 +373,14 @@ void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; @@ -314,6 +395,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_SUCCESS; SUB_ENCODE_OVER: + tfree(buf); if (terrno != 0) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -326,6 +408,7 @@ SUB_ENCODE_OVER: static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; @@ -345,7 +428,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - void *buf = malloc(tlen + 1); + buf = malloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -357,6 +440,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; SUB_DECODE_OVER: + tfree(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); // TODO free subscribeobj @@ -374,6 +458,7 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) { static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) { mTrace("subscribe:%s, perform delete action", pSub->key); + tDeleteSMqSubscribeObj(pSub); return 0; } @@ -382,10 +467,6 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc return 0; } -static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) { - return 0; -} - static char *mndMakeSubscribeKey(char *cgroup, char *topicName) { char *key = malloc(TSDB_SHOW_SUBQUERY_LEN); if (key == NULL) { @@ -439,10 +520,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pConsumer->epoch = 1; pConsumer->consumerId = consumerId; strcpy(pConsumer->cgroup, consumerGroup); taosInitRWLatch(&pConsumer->lock); } else { + pConsumer->epoch++; oldSub = pConsumer->topics; } pConsumer->topics = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); @@ -546,6 +629,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName); + bool create = false; if (pSub == NULL) { mDebug("create new subscription, group: %s, topic %s", consumerGroup, newTopicName); pSub = tNewSubscribeObj(); @@ -554,14 +638,24 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return -1; } char *key = mndMakeSubscribeKey(consumerGroup, newTopicName); + if (key == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } strcpy(pSub->key, key); + free(key); // set unassigned vg - mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); + if (mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg) < 0) { + // TODO: free memory + return -1; + } // TODO: disable alter + create = true; } taosArrayPush(pSub->availConsumer, &consumerId); - SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); + int64_t oldConsumerId; + SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub, &oldConsumerId); taosArrayPush(pConsumer->topics, pConsumerTopic); if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { @@ -569,7 +663,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); SMqConsumerEp *pCEp = taosArrayGetLast(pSub->assigned); if (pCEp->vgId == vgId) { - if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp) < 0) { + if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp, oldConsumerId) < 0) { // TODO return -1; } @@ -580,6 +674,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SSdbRaw *pRaw = mndSubActionEncode(pSub); sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); + if (!create) mndReleaseSubscribe(pMnode, pSub); #if 0 SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); if (pGroup == NULL) { @@ -631,14 +726,14 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); if (newSub) taosArrayDestroy(newSub); mndTransDrop(pTrans); - mndReleaseConsumer(pMnode, pConsumer); + /*mndReleaseConsumer(pMnode, pConsumer);*/ return -1; } if (newSub) taosArrayDestroy(newSub); mndTransDrop(pTrans); - mndReleaseConsumer(pMnode, pConsumer); - return 0; + /*mndReleaseConsumer(pMnode, pConsumer);*/ + return TSDB_CODE_MND_ACTION_IN_PROGRESS; } static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pRsp) { @@ -650,7 +745,7 @@ static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; STableInfoReq *pInfo = pMsg->rpcMsg.pCont; - mDebug("subscribe:%s, start to retrieve meta", pInfo->tableFname); + mDebug("subscribe:%s, start to retrieve meta", pInfo->tbName); #if 0 SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname); @@ -781,7 +876,7 @@ static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRs pShow->numOfRows = sdbGetSize(pSdb, SDB_CONSUMER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index aacaa0d728..777492697a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -103,6 +103,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { if (walEndSnapshot(pWal) < 0) { goto WAL_RESTORE_OVER; } + } code = 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index fa043cf7a0..ea3cc7c2c6 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -237,18 +237,17 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); topicObj.createTime = taosGetTimestampMs(); topicObj.updateTime = topicObj.createTime; - topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); + topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); topicObj.dbUid = pDb->uid; topicObj.version = 1; - topicObj.sql = strdup(pCreate->sql); - topicObj.physicalPlan = strdup(pCreate->physicalPlan); - topicObj.logicalPlan = strdup(pCreate->logicalPlan); + topicObj.sql = pCreate->sql; + topicObj.physicalPlan = pCreate->physicalPlan; + topicObj.logicalPlan = pCreate->logicalPlan; topicObj.sqlLen = strlen(pCreate->sql); SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); if (pTopicRaw == NULL) return -1; if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; - // TODO: replace with trans to support recovery return sdbWrite(pMnode->pSdb, pTopicRaw); } @@ -339,7 +338,7 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; STableInfoReq *pInfo = pMsg->rpcMsg.pCont; - mDebug("topic:%s, start to retrieve meta", pInfo->tableFname); + mDebug("topic:%s, start to retrieve meta", pInfo->tbName); #if 0 SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname); @@ -470,7 +469,7 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_TOPIC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 03226c7400..6686c0887f 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -308,6 +308,11 @@ static void mndTransDropData(STrans *pTrans) { mndTransDropLogs(pTrans->commitLogs); mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->undoActions); + if (pTrans->rpcRsp != NULL) { + rpcFreeCont(pTrans->rpcRsp); + pTrans->rpcRsp = NULL; + pTrans->rpcRspLen = 0; + } } static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { @@ -339,7 +344,7 @@ void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) { sdbRelease(pSdb, pTrans); } -STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, SRpcMsg *pMsg) { +STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, const SRpcMsg *pReq) { STrans *pTrans = calloc(1, sizeof(STrans)); if (pTrans == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -350,8 +355,8 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, SRpcMsg *pMsg) { pTrans->id = sdbGetMaxId(pMnode->pSdb, SDB_TRANS); pTrans->stage = TRN_STAGE_PREPARE; pTrans->policy = policy; - pTrans->rpcHandle = pMsg->handle; - pTrans->rpcAHandle = pMsg->ahandle; + pTrans->rpcHandle = pReq->handle; + pTrans->rpcAHandle = pReq->ahandle; pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); @@ -436,6 +441,11 @@ int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { return mndTransAppendAction(pTrans->undoActions, pAction); } +void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) { + pTrans->rpcRsp = pCont; + pTrans->rpcRspLen = contLen; +} + static int32_t mndTransSync(SMnode *pMnode, STrans *pTrans) { SSdbRaw *pRaw = mndTransActionEncode(pTrans); if (pRaw == NULL) { @@ -479,6 +489,11 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { pNew->rpcHandle = pTrans->rpcHandle; pNew->rpcAHandle = pTrans->rpcAHandle; + pNew->rpcRsp = pTrans->rpcRsp; + pNew->rpcRspLen = pTrans->rpcRspLen; + pTrans->rpcRsp = NULL; + pTrans->rpcRspLen = 0; + mndTransExecute(pMnode, pNew); mndReleaseTrans(pMnode, pNew); return 0; @@ -529,15 +544,21 @@ static void mndTransSendRpcRsp(STrans *pTrans) { if (sendRsp && pTrans->rpcHandle != NULL) { mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); - SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle}; + SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, + .code = pTrans->code, + .ahandle = pTrans->rpcAHandle, + .pCont = pTrans->rpcRsp, + .contLen = pTrans->rpcRspLen}; rpcSendResponse(&rspMsg); pTrans->rpcHandle = NULL; + pTrans->rpcRsp = NULL; + pTrans->rpcRspLen = 0; } } -void mndTransProcessRsp(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - int64_t signature = (int64_t)(pMsg->rpcMsg.ahandle); +void mndTransProcessRsp(SMnodeMsg *pRsp) { + SMnode *pMnode = pRsp->pMnode; + int64_t signature = (int64_t)(pRsp->rpcMsg.ahandle); int32_t transId = (int32_t)(signature >> 32); int32_t action = (int32_t)((signature << 32) >> 32); @@ -571,10 +592,10 @@ void mndTransProcessRsp(SMnodeMsg *pMsg) { STransAction *pAction = taosArrayGet(pArray, action); if (pAction != NULL) { pAction->msgReceived = 1; - pAction->errCode = pMsg->rpcMsg.code; + pAction->errCode = pRsp->rpcMsg.code; } - mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%x", transId, action, pMsg->rpcMsg.code, + mDebug("trans:%d, action:%d response is received, code:0x%x, accept:0x%x", transId, action, pRsp->rpcMsg.code, pAction->acceptableCode); mndTransExecute(pMnode, pTrans); @@ -930,4 +951,6 @@ void mndTransPullup(SMnode *pMnode) { mndTransExecute(pMnode, pTrans); sdbRelease(pMnode->pSdb, pTrans); } + + sdbWriteFile(pMnode->pSdb); } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c36e128893..3040cd8a28 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -474,7 +474,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 04fedbb3ce..33aa4ce07c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -178,7 +178,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId); - if (pVgroup == NULL) { + if (pVgroup == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; } return pVgroup; @@ -424,9 +424,7 @@ SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) { epset.inUse = epset.numOfEps; } - epset.port[epset.numOfEps] = pDnode->port; - memcpy(&epset.fqdn[epset.numOfEps], pDnode->fqdn, TSDB_FQDN_LEN); - epset.numOfEps++; + addEpIntoEpSet(&epset, pDnode->fqdn, pDnode->port); mndReleaseDnode(pMnode, pDnode); } @@ -527,7 +525,7 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -640,7 +638,7 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->replica = dnodeId; pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 5d5947b644..2d1574467e 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -202,6 +202,10 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); + + SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; + pDrop->uid = htobe64(pDrop->uid); + EXPECT_STREQ(pDrop->db, "1.d1"); } test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); @@ -249,6 +253,8 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_EQ(test.GetShowRows(), 1); CheckBinary("d2", TSDB_DB_NAME_LEN - 1); + uint64_t d2_uid = 0; + { int32_t contLen = sizeof(SUseDbReq); @@ -262,6 +268,8 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; EXPECT_STREQ(pRsp->db, "1.d2"); + pRsp->uid = htobe64(pRsp->uid); + d2_uid = pRsp->uid; pRsp->vgVersion = htonl(pRsp->vgVersion); pRsp->vgNum = htonl(pRsp->vgNum); pRsp->hashMethod = pRsp->hashMethod; @@ -277,9 +285,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, 0); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); - EXPECT_EQ(pInfo->inUse, 0); - EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddr* pAddr = &pInfo->epAddr[0]; + EXPECT_EQ(pInfo->epset.inUse, 0); + EXPECT_EQ(pInfo->epset.numOfEps, 1); + SEp* pAddr = &pInfo->epset.eps[0]; pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); @@ -293,9 +301,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); - EXPECT_EQ(pInfo->inUse, 0); - EXPECT_EQ(pInfo->numOfEps, 1); - SEpAddr* pAddr = &pInfo->epAddr[0]; + EXPECT_EQ(pInfo->epset.inUse, 0); + EXPECT_EQ(pInfo->epset.numOfEps, 1); + SEp* pAddr = &pInfo->epset.eps[0]; pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); @@ -311,5 +319,10 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); + + SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; + pDrop->uid = htobe64(pDrop->uid); + EXPECT_STREQ(pDrop->db, "1.d2"); + EXPECT_EQ(pDrop->uid, d2_uid); } } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index 4ad979cdd3..ccf13d5d66 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -44,7 +44,7 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { pRsp->acctId = htonl(pRsp->acctId); pRsp->clusterId = htobe64(pRsp->clusterId); pRsp->connId = htonl(pRsp->connId); - pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); + pRsp->epSet.eps[0].port = htons(pRsp->epSet.eps[0].port); EXPECT_EQ(pRsp->acctId, 1); EXPECT_GT(pRsp->clusterId, 0); @@ -53,8 +53,8 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9031); - EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); + EXPECT_EQ(pRsp->epSet.eps[0].port, 9031); + EXPECT_STREQ(pRsp->epSet.eps[0].fqdn, "localhost"); connId = pRsp->connId; } @@ -102,11 +102,10 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) { req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ}; req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); SKv kv; - kv.key = (void*)"abc"; - kv.keyLen = 4; + kv.key = 123; kv.value = (void*)"bcd"; kv.valueLen = 4; - taosHashPut(req.info, kv.key, kv.keyLen, kv.value, kv.valueLen); + taosHashPut(req.info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); taosArrayPush(batchReq.reqs, &req); int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index bfdc0f42b6..bc31630ffe 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -64,7 +64,7 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { test.SendShowMetaReq(TSDB_MGMT_TABLE_CONNS, ""); STableMetaRsp* pMeta = test.GetShowMeta(); - EXPECT_STREQ(pMeta->tbFname, "show connections"); + EXPECT_STREQ(pMeta->tbName, "show connections"); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->numOfColumns, 7); EXPECT_EQ(pMeta->precision, 0); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index beb52d68c1..a0e2460334 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -126,7 +126,8 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { int32_t contLen = sizeof(STableInfoReq); STableInfoReq* pReq = (STableInfoReq*)rpcMallocCont(contLen); - strcpy(pReq->tableFname, "1.d1.stb"); + strcpy(pReq->dbFName, "1.d1"); + strcpy(pReq->tbName, "stb"); SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -146,8 +147,9 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pSchema->bytes = htonl(pSchema->bytes); } - EXPECT_STREQ(pRsp->tbFname, "1.d1.stb"); - EXPECT_STREQ(pRsp->stbFname, ""); + EXPECT_STREQ(pRsp->dbFName, "1.d1"); + EXPECT_STREQ(pRsp->tbName, "stb"); + EXPECT_STREQ(pRsp->stbName, "stb"); EXPECT_EQ(pRsp->numOfColumns, 2); EXPECT_EQ(pRsp->numOfTags, 3); EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index 39b5bb4d5b..26809ea059 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -64,11 +64,7 @@ SSdb *sdbInit(SSdbOpt *pOption) { void sdbCleanup(SSdb *pSdb) { mDebug("start to cleanup sdb"); - if (pSdb->curVer > pSdb->lastCommitVer) { - mDebug("write sdb file for current ver:%" PRId64 " larger than last commit ver:%" PRId64, pSdb->curVer, - pSdb->lastCommitVer); - sdbWriteFile(pSdb); - } + sdbWriteFile(pSdb); if (pSdb->currDir != NULL) { tfree(pSdb->currDir); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 1f6d6cbda8..6d17423324 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -221,7 +221,7 @@ PARSE_SDB_DATA_ERROR: return code; } -int32_t sdbWriteFile(SSdb *pSdb) { +static int32_t sdbWriteFileImp(SSdb *pSdb) { int32_t code = 0; char tmpfile[PATH_MAX] = {0}; @@ -229,7 +229,8 @@ int32_t sdbWriteFile(SSdb *pSdb) { char curfile[PATH_MAX] = {0}; snprintf(curfile, sizeof(curfile), "%s%ssdb.data", pSdb->currDir, TD_DIRSEP); - mDebug("start to write file:%s", curfile); + mDebug("start to write file:%s, current ver:%" PRId64 ", commit ver:%" PRId64, curfile, pSdb->curVer, + pSdb->lastCommitVer); FileFd fd = taosOpenFileCreateWriteTrunc(tmpfile); if (fd <= 0) { @@ -323,12 +324,20 @@ int32_t sdbWriteFile(SSdb *pSdb) { return code; } +int32_t sdbWriteFile(SSdb *pSdb) { + if (pSdb->curVer == pSdb->lastCommitVer) { + return 0; + } + + return sdbWriteFileImp(pSdb); +} + int32_t sdbDeploy(SSdb *pSdb) { if (sdbRunDeployFp(pSdb) != 0) { return -1; } - if (sdbWriteFile(pSdb) != 0) { + if (sdbWriteFileImp(pSdb) != 0) { return -1; } diff --git a/source/dnode/vnode/inc/meta.h b/source/dnode/vnode/inc/meta.h index 383073871e..44a352ec54 100644 --- a/source/dnode/vnode/inc/meta.h +++ b/source/dnode/vnode/inc/meta.h @@ -37,11 +37,6 @@ typedef struct SMetaCfg { uint64_t lruSize; } SMetaCfg; -typedef struct { - uint32_t nCols; - SSchema *pSchema; -} SSchemaWrapper; - typedef struct SMTbCursor SMTbCursor; typedef struct SMCtbCursor SMCtbCursor; diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 3a1e5b9c95..2aceaeb016 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -35,71 +35,8 @@ extern "C" { #endif -typedef struct STqMsgHead { - int32_t protoVer; - int32_t msgType; - int64_t cgId; - int64_t clientId; -} STqMsgHead; - -typedef struct STqOneAck { - int64_t topicId; - int64_t consumeOffset; -} STqOneAck; - -typedef struct STqAcks { - int32_t ackNum; - // should be sorted - STqOneAck acks[]; -} STqAcks; - -typedef struct STqSetCurReq { - STqMsgHead head; - int64_t topicId; - int64_t offset; -} STqSetCurReq; - -typedef struct STqConsumeReq { - STqMsgHead head; - int64_t blockingTime; // milisec - STqAcks acks; -} STqConsumeReq; - -typedef struct STqMsgContent { - int64_t topicId; - int64_t msgLen; - char msg[]; -} STqMsgContent; - -typedef struct STqConsumeRsp { - STqMsgHead head; - int64_t bodySize; - STqMsgContent msgs[]; -} STqConsumeRsp; - -typedef struct STqSubscribeReq { - STqMsgHead head; - int32_t topicNum; - int64_t topic[]; -} STqSubscribeReq; - -typedef struct STqHeartbeatReq { -} STqHeartbeatReq; - -typedef struct STqHeartbeatRsp { -} STqHeartbeatRsp; - #define TQ_BUFFER_SIZE 8 -typedef struct STqExec { - void* runtimeEnv; - SSDataBlock* (*exec)(void* runtimeEnv); - void* (*assign)(void* runtimeEnv, void* inputData); - void (*clear)(void* runtimeEnv); - char* (*serialize)(struct STqExec*); - struct STqExec* (*deserialize)(char*); -} STqExec; - typedef struct STqRspHandle { void* handle; void* ahandle; @@ -107,53 +44,13 @@ typedef struct STqRspHandle { typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; -typedef struct STqTopic STqTopic; - -typedef struct STqBufferItem { - int64_t offset; - // executors are identical but not concurrent - // so there must be a copy in each item - STqExec* executor; - int32_t status; - int64_t size; - void* content; - STqTopic* pTopic; -} STqMsgItem; - -struct STqTopic { - // char* topic; //c style, end with '\0' - // int64_t cgId; - // void* ahandle; - // int32_t head; - // int32_t tail; - int64_t nextConsumeOffset; - int64_t floatingCursor; - int64_t topicId; - void* logReader; - STqMsgItem buffer[TQ_BUFFER_SIZE]; -}; - -typedef struct STqListHandle { - STqTopic topic; - struct STqListHandle* next; -} STqList; - -typedef struct STqGroup { - int64_t clientId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - STqList* head; - SList* topicList; // SList - STqRspHandle rspHandle; -} STqGroup; - typedef struct STqTaskItem { - int8_t status; - int64_t offset; - void* dst; - qTaskInfo_t task; - SSubQueryMsg* pQueryMsg; + int8_t status; + int64_t offset; + void* dst; + qTaskInfo_t task; + STqReadHandle* pReadHandle; + SSubQueryMsg* pQueryMsg; } STqTaskItem; // new version @@ -181,11 +78,6 @@ typedef struct STqConsumerHandle { SArray* topics; // SArray } STqConsumerHandle; -typedef struct STqQueryMsg { - STqMsgItem* item; - struct STqQueryMsg* next; -} STqQueryMsg; - typedef struct STqMemRef { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; @@ -304,20 +196,6 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqSetCursor(STQ*, STqSetCurReq* pMsg); - -#if 0 -int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); -int tqSetCursor(STQ*, STqSetCurReq* pMsg); -int tqBufferSetOffset(STqTopic*, int64_t offset); -STqTopic* tqFindTopic(STqGroup*, int64_t topicId); -STqGroup* tqGetGroup(STQ*, int64_t clientId); -STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqRegisterContext(STqGroup*, void* ahandle); -int tqSendLaunchQuery(STqMsgItem*, int64_t offset); -#endif - int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 6a9eb43ca8..a2f935683e 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -75,7 +75,6 @@ typedef struct STsdbQueryCond { } STsdbQueryCond; typedef struct { - void *pTable; TSKEY lastKey; uint64_t uid; } STableKeyInfo; @@ -92,7 +91,7 @@ int tsdbCommit(STsdb *pTsdb); int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); -typedef void* tsdbReadHandleT; +typedef void* tsdbReaderT; /** * Get the data block iterator, starting from position according to the query condition @@ -104,7 +103,7 @@ typedef void* tsdbReadHandleT; * @param qinfo query info handle from query processor * @return */ -tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); +tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -116,13 +115,13 @@ tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup * @param tableInfo table list. * @return */ -//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, // SMemRef *pRef); -tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); -bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); +bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle); /** * @@ -139,9 +138,9 @@ bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); * @param reqId * @return */ -int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId); + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId); /** * get num of rows in mem table * @@ -149,7 +148,7 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch * @return row size */ -int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); +int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle); /** * move to next block if exists @@ -157,7 +156,7 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); * @param pTsdbReadHandle * @return */ -bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); +bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); /** * Get current data block information @@ -166,7 +165,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); +void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); /** * @@ -178,7 +177,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis); /** * @@ -190,7 +189,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataS * @param pColumnIdList required data columns id list * @return */ -SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList); +SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); /** * destroy the created table group list, which is generated by tag query @@ -206,7 +205,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); /** * @@ -221,7 +220,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGro * clean up the query handle * @param queryHandle */ -void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle); +void tsdbCleanupReadHandle(tsdbReaderT queryHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2accfd6279..7549772613 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -33,13 +33,13 @@ extern "C" { typedef struct SVnode SVnode; typedef struct SDnode SDnode; typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); typedef struct STqCfg { // TODO int32_t reserved; } STqCfg; - typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; @@ -64,17 +64,22 @@ typedef struct { const char *charset; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) PutReqToVQueryQFp putReqToVQueryQFp; + SendReqToDnodeFp sendReqToDnodeFp; } SVnodeOpt; typedef struct STqReadHandle { - int64_t ver; - uint64_t tbUid; - SSubmitMsg* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - SMeta* pMeta; - SArray* pColIdList; + int64_t ver; + uint64_t tbUid; + SHashObj *tbIdHash; + const SSubmitMsg *pMsg; + SSubmitBlk *pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; + SMeta *pVnodeMeta; + SArray *pColIdList; // SArray + int32_t sver; + SSchemaWrapper *pSchemaWrapper; + STSchema *pSchema; } STqReadHandle; /* ------------------------ SVnode ------------------------ */ @@ -196,22 +201,34 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); /* ------------------------- TQ QUERY -------------------------- */ -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta); +STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); -static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { +static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList) { pReadHandle->pColIdList = pColIdList; } -static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, uint64_t tbUid) { - pHandle->tbUid = tbUid; +// static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, const SArray* pTableIdList) { +// pHandle->tbUid = pTableIdList; +//} + +static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + if (pHandle->tbIdHash == NULL) { + return -1; + } + for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i); + taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); + // pHandle->tbUid = tbUid; + } + return 0; } -void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); -bool tqNextDataBlock(STqReadHandle* pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); +void tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitMsg *pMsg, int64_t ver); +bool tqNextDataBlock(STqReadHandle *pHandle); +int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); // return SArray -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); - +SArray *tqRetrieveDataBlock(STqReadHandle *pHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index a9ba825a29..cefe9eff72 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -16,9 +16,9 @@ #ifndef _TD_TQ_INT_H_ #define _TD_TQ_INT_H_ -#include "tq.h" #include "meta.h" #include "tlog.h" +#include "tq.h" #include "trpc.h" #ifdef __cplusplus extern "C" { @@ -26,29 +26,48 @@ extern "C" { extern int32_t tqDebugFlag; -#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }} -#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }} -#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }} -#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }} -#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} -#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} +#define tqFatal(...) \ + { \ + if (tqDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \ + } \ + } +#define tqError(...) \ + { \ + if (tqDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ + } \ + } +#define tqWarn(...) \ + { \ + if (tqDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ + } \ + } +#define tqInfo(...) \ + { \ + if (tqDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TQ ", 255, __VA_ARGS__); \ + } \ + } +#define tqDebug(...) \ + { \ + if (tqDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } +#define tqTrace(...) \ + { \ + if (tqDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } -// create persistent storage for meta info such as consuming offset -// return value > 0: cgId -// return value <= 0: error code -// int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); -// create ring buffer in memory and load consuming offset -// int tqOpenTCGroup(STQ*, const char* topic, int cgId); -// destroy ring buffer and persist consuming offset -// int tqCloseTCGroup(STQ*, const char* topic, int cgId); -// delete persistent storage for meta info -// int tqDropTCGroup(STQ*, const char* topic, int cgId); +int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); +const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); -//int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -//const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); -int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h index ef71d8bf14..3bf9bb7138 100644 --- a/source/dnode/vnode/src/inc/tqMetaStore.h +++ b/source/dnode/vnode/src/inc/tqMetaStore.h @@ -40,6 +40,7 @@ int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); // delete committed kv pair // notice that a delete action still needs to be committed int32_t tqHandleDel(STqMetaStore*, int64_t key); +int32_t tqHandlePurge(STqMetaStore*, int64_t key); int32_t tqHandleCommit(STqMetaStore*, int64_t key); int32_t tqHandleAbort(STqMetaStore*, int64_t key); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 1fa65b2a73..f442697fb0 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -55,6 +55,7 @@ typedef struct SVnodeMgr { // For vnode Mgmt SDnode* pDnode; PutReqToVQueryQFp putReqToVQueryQFp; + SendReqToDnodeFp sendReqToDnodeFp; } SVnodeMgr; extern SVnodeMgr vnodeMgr; @@ -85,6 +86,7 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq); +void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq); // For Log extern int32_t vDebugFlag; diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index dfdc144750..e5ccd02e48 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -20,6 +20,10 @@ #include "tcoding.h" #include "thash.h" +#define IMPL_WITH_LOCK 1 +// #if IMPL_WITH_LOCK +// #endif + typedef struct { tb_uid_t uid; int32_t sver; @@ -27,6 +31,9 @@ typedef struct { } SSchemaKey; struct SMetaDB { +#if IMPL_WITH_LOCK + pthread_rwlock_t rwlock; +#endif // DB DB *pTbDB; DB *pSchemaDB; @@ -58,6 +65,9 @@ static void * metaDecodeTbInfo(void *buf, STbCfg *pTbCfg); static void metaClearTbCfg(STbCfg *pTbCfg); static int metaEncodeSchema(void **buf, SSchemaWrapper *pSW); static void * metaDecodeSchema(void *buf, SSchemaWrapper *pSW); +static void metaDBWLock(SMetaDB *pDB); +static void metaDBRLock(SMetaDB *pDB); +static void metaDBULock(SMetaDB *pDB); #define BDB_PERR(info, code) fprintf(stderr, info " reason: %s", db_strerror(code)) @@ -130,8 +140,10 @@ void metaCloseDB(SMeta *pMeta) { int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { tb_uid_t uid; char buf[512]; + char buf1[512]; void * pBuf; - DBT key, value; + DBT key1, value1; + DBT key2, value2; SSchema *pSchema = NULL; if (pTbCfg->type == META_SUPER_TABLE) { @@ -143,19 +155,17 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { { // save table info pBuf = buf; - memset(&key, 0, sizeof(key)); - memset(&value, 0, sizeof(key)); + memset(&key1, 0, sizeof(key1)); + memset(&value1, 0, sizeof(key1)); - key.data = &uid; - key.size = sizeof(uid); + key1.data = &uid; + key1.size = sizeof(uid); metaEncodeTbInfo(&pBuf, pTbCfg); - value.data = buf; - value.size = POINTER_DISTANCE(pBuf, buf); - value.app_data = pTbCfg; - - pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key, &value, 0); + value1.data = buf; + value1.size = POINTER_DISTANCE(pBuf, buf); + value1.app_data = pTbCfg; } // save schema @@ -169,23 +179,28 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { } if (pSchema) { - pBuf = buf; - memset(&key, 0, sizeof(key)); - memset(&value, 0, sizeof(key)); + pBuf = buf1; + memset(&key2, 0, sizeof(key2)); + memset(&value2, 0, sizeof(key2)); SSchemaKey schemaKey = {uid, 0 /*TODO*/, 0}; - key.data = &schemaKey; - key.size = sizeof(schemaKey); + key2.data = &schemaKey; + key2.size = sizeof(schemaKey); SSchemaWrapper sw = {.nCols = ncols, .pSchema = pSchema}; metaEncodeSchema(&pBuf, &sw); - value.data = buf; - value.size = POINTER_DISTANCE(pBuf, buf); - - pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key, &value, 0); + value2.data = buf1; + value2.size = POINTER_DISTANCE(pBuf, buf1); } + metaDBWLock(pMeta->pDB); + pMeta->pDB->pTbDB->put(pMeta->pDB->pTbDB, NULL, &key1, &value1, 0); + if (pSchema) { + pMeta->pDB->pSchemaDB->put(pMeta->pDB->pSchemaDB, NULL, &key2, &value2, 0); + } + metaDBULock(pMeta->pDB); + return 0; } @@ -234,11 +249,18 @@ static SMetaDB *metaNewDB() { return NULL; } +#if IMPL_WITH_LOCK + pthread_rwlock_init(&pDB->rwlock, NULL); +#endif + return pDB; } static void metaFreeDB(SMetaDB *pDB) { if (pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_destroy(&pDB->rwlock); +#endif free(pDB); } } @@ -467,7 +489,9 @@ STbCfg *metaGetTbInfoByUid(SMeta *pMeta, tb_uid_t uid) { key.size = sizeof(uid); // Query + metaDBRLock(pDB); ret = pDB->pTbDB->get(pDB->pTbDB, NULL, &key, &value, 0); + metaDBULock(pDB); if (ret != 0) { return NULL; } @@ -496,7 +520,9 @@ STbCfg *metaGetTbInfoByName(SMeta *pMeta, char *tbname, tb_uid_t *uid) { key.size = strlen(tbname); // Query + metaDBRLock(pDB); ret = pDB->pNameIdx->pget(pDB->pNameIdx, NULL, &key, &pkey, &pvalue, 0); + metaDBULock(pDB); if (ret != 0) { return NULL; } @@ -529,7 +555,9 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo key.size = sizeof(schemaKey); // Query + metaDBRLock(pDB); ret = pDB->pSchemaDB->get(pDB->pSchemaDB, NULL, &key, &value, 0); + metaDBULock(pDB); if (ret != 0) { printf("failed to query schema DB since %s================\n", db_strerror(ret)); return NULL; @@ -687,4 +715,22 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { } else { return 0; } -} \ No newline at end of file +} + +static void metaDBWLock(SMetaDB *pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_wrlock(&(pDB->rwlock)); +#endif +} + +static void metaDBRLock(SMetaDB *pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_rdlock(&(pDB->rwlock)); +#endif +} + +static void metaDBULock(SMetaDB *pDB) { +#if IMPL_WITH_LOCK + pthread_rwlock_unlock(&(pDB->rwlock)); +#endif +} diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c164487aa2..a625980505 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -12,29 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE +#include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" -#include "tcompare.h" - -// static -// read next version data -// -// send to fetch queue -// -// handle management message -// - -int tqGroupSSize(const STqGroup* pGroup); -int tqTopicSSize(); -int tqItemSSize(); - -void* tqSerializeListHandle(STqList* listHandle, void* ptr); -void* tqSerializeTopic(STqTopic* pTopic, void* ptr); -void* tqSerializeItem(STqMsgItem* pItem, void* ptr); - -const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic); -const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); int tqInit() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); @@ -88,177 +70,6 @@ void tqClose(STQ* pTq) { // TODO } -static int tqProtoCheck(STqMsgHead* pMsg) { - // TODO - return pMsg->protoVer == 0; -} - -static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) { - // clean old item and move forward - int32_t consumeOffset = pAck->consumeOffset; - int idx = consumeOffset % TQ_BUFFER_SIZE; - ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor); - tfree(pTopic->buffer[idx].content); - if (1 /* TODO: need to launch new query */) { - STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); - if (pNewQuery == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - // TODO: lock executor - // TODO: read from wal and assign to src - /*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/ - /*pNewQuery->exec->src = 0;*/ - /*pNewQuery->exec->dest = &pTopic->buffer[idx];*/ - /*pNewQuery->next = *ppQuery;*/ - /**ppQuery = pNewQuery;*/ - } - return 0; -} - -static int tqAck(STqGroup* pGroup, STqAcks* pAcks) { - int32_t ackNum = pAcks->ackNum; - STqOneAck* acks = pAcks->acks; - // double ptr for acks and list - int i = 0; - STqList* node = pGroup->head; - int ackCnt = 0; - STqQueryMsg* pQuery = NULL; - while (i < ackNum && node->next) { - if (acks[i].topicId == node->next->topic.topicId) { - ackCnt++; - tqAckOneTopic(&node->next->topic, &acks[i], &pQuery); - } else if (acks[i].topicId < node->next->topic.topicId) { - i++; - } else { - node = node->next; - } - } - if (pQuery) { - // post message - } - return ackCnt; -} - -static int tqCommitGroup(STqGroup* pGroup) { - // persist modification into disk - return 0; -} - -int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) { - // create in disk - STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup)); - if (pGroup == NULL) { - // TODO - return -1; - } - *ppGroup = pGroup; - memset(pGroup, 0, sizeof(STqGroup)); - - pGroup->topicList = tdListNew(sizeof(STqTopic)); - if (pGroup->topicList == NULL) { - free(pGroup); - return -1; - } - *ppGroup = pGroup; - - return 0; -} - -STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId); - if (pGroup == NULL) { - int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup); - if (code < 0) { - // TODO - return NULL; - } - tqHandleMovePut(pTq->tqMeta, cId, pGroup); - } - ASSERT(pGroup); - - return pGroup; -} - -int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - // TODO - return 0; -} - -int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - // delete from disk - return 0; -} - -static int tqFetch(STqGroup* pGroup, STqConsumeRsp** pRsp) { - STqList* pHead = pGroup->head; - STqList* pNode = pHead; - int totSize = 0; - int numOfMsgs = 0; - // TODO: make it a macro - int sizeLimit = 4 * 1024; - - void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + sizeLimit); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - *pRsp = ptr; - STqMsgContent* buffer = (*pRsp)->msgs; - - // iterate the list to get msgs of all topics - // until all topic iterated or msgs over sizeLimit - while (pNode->next) { - pNode = pNode->next; - STqTopic* pTopic = &pNode->topic; - int idx = pTopic->nextConsumeOffset % TQ_BUFFER_SIZE; - if (pTopic->buffer[idx].content != NULL && pTopic->buffer[idx].offset == pTopic->nextConsumeOffset) { - totSize += pTopic->buffer[idx].size; - if (totSize > sizeLimit) { - void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + totSize); - if (ptr == NULL) { - totSize -= pTopic->buffer[idx].size; - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // return msgs already copied - break; - } - *pRsp = ptr; - break; - } - *((int64_t*)buffer) = pTopic->topicId; - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = pTopic->buffer[idx].size; - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); - numOfMsgs++; - if (totSize > sizeLimit) { - break; - } - } - } - (*pRsp)->bodySize = totSize; - return numOfMsgs; -} - -STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); } - -int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) { - if (tqQueryExecuting(bufItem->status)) { - return 0; - } - bufItem->status = 1; - // load data from wal or buffer pool - // put into exec - // send exec into non blocking queue - // when query finished, put into buffer pool - return 0; -} - -/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ -/*return 0;*/ -/*}*/ - int tqPushMsg(STQ* pTq, void* p, int64_t version) { // add reference // judge and launch new query @@ -270,221 +81,10 @@ int tqCommit(STQ* pTq) { return 0; } -int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) { - int code; - memset(pTopic->buffer, 0, sizeof(pTopic->buffer)); - // launch query - for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) { - int pos = i % TQ_BUFFER_SIZE; - code = tqSendLaunchQuery(&pTopic->buffer[pos], offset); - if (code < 0) { - // TODO: error handling - } - } - // set offset - pTopic->nextConsumeOffset = offset; - pTopic->floatingCursor = offset; - return 0; -} - -STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) { - // TODO - return NULL; -} - -int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) { - int code; - int64_t clientId = pMsg->head.clientId; - int64_t topicId = pMsg->topicId; - int64_t offset = pMsg->offset; - STqGroup* gHandle = tqGetGroup(pTq, clientId); - if (gHandle == NULL) { - // client not connect - return -1; - } - STqTopic* topicHandle = tqFindTopic(gHandle, topicId); - if (topicHandle == NULL) { - return -1; - } - if (pMsg->offset == topicHandle->nextConsumeOffset) { - return 0; - } - // TODO: check log last version - - code = tqBufferSetOffset(topicHandle, offset); - if (code < 0) { - // set error code - return -1; - } - - return 0; -} - -// temporary -int tqProcessCMsg(STQ* pTq, STqConsumeReq* pMsg, STqRspHandle* pRsp) { - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - terrno = TSDB_CODE_TQ_GROUP_NOT_SET; - return -1; - } - pGroup->rspHandle.handle = pRsp->handle; - pGroup->rspHandle.ahandle = pRsp->ahandle; - - return 0; -} - -int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { - STqConsumeReq* pMsg = pReq->pCont; - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - terrno = TSDB_CODE_TQ_GROUP_NOT_SET; - return -1; - } - - SList* topicList = pGroup->topicList; - - int totSize = 0; - int numOfMsgs = 0; - int sizeLimit = 4096; - - STqConsumeRsp* pCsmRsp = (*pRsp)->pCont; - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - (*pRsp)->pCont = ptr; - - SListIter iter; - tdListInitIter(topicList, &iter, TD_LIST_FORWARD); - - STqMsgContent* buffer = NULL; - SArray* pArray = taosArrayInit(0, sizeof(void*)); - - SListNode* pn; - while ((pn = tdListNext(&iter)) != NULL) { - STqTopic* pTopic = *(STqTopic**)pn->data; - int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; - STqMsgItem* pItem = &pTopic->buffer[idx]; - if (pItem->content != NULL && pItem->offset == pTopic->floatingCursor) { - if (pItem->status == TQ_ITEM_READY) { - // if has data - totSize += pTopic->buffer[idx].size; - if (totSize > sizeLimit) { - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + totSize); - if (ptr == NULL) { - totSize -= pTopic->buffer[idx].size; - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // return msgs already copied - break; - } - (*pRsp)->pCont = ptr; - break; - } - *((int64_t*)buffer) = htobe64(pTopic->topicId); - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = htobe64(pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); - numOfMsgs++; - if (totSize > sizeLimit) { - break; - } - } else if (pItem->status == TQ_ITEM_PROCESS) { - // if not have data but in process - - } else if (pItem->status == TQ_ITEM_EMPTY) { - // if not have data and not in process - int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_EMPTY, TQ_ITEM_PROCESS); - if (old != TQ_ITEM_EMPTY) { - continue; - } - pItem->offset = pTopic->floatingCursor; - taosArrayPush(pArray, &pItem); - } else { - ASSERT(0); - } - } - } - - if (numOfMsgs > 0) { - // set code and other msg - rpcSendResponse(*pRsp); - } else { - // most recent data has been fetched - - // enable timer for blocking wait - // once new data written when waiting, launch query and rsp - } - - // fetched a num of msgs, rpc response - for (int i = 0; i < pArray->size; i++) { - STqMsgItem* pItem = taosArrayGet(pArray, i); - - // read from wal - void* raw = NULL; - /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ - /*int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset);*/ - /*if (code < 0) {*/ - // TODO: error - /*}*/ - // get msgType - // if submitblk - pItem->executor->assign(pItem->executor->runtimeEnv, raw); - SSDataBlock* content = pItem->executor->exec(pItem->executor->runtimeEnv); - pItem->content = content; - // if other type, send just put into buffer - /*pItem->content = raw;*/ - - int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_PROCESS, TQ_ITEM_READY); - ASSERT(old == TQ_ITEM_PROCESS); - } - taosArrayDestroy(pArray); - - return 0; -} - -#if 0 -int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { - if (!tqProtoCheck((STqMsgHead*)pMsg)) { - // proto version invalid - return -1; - } - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - // client not connect - return -1; - } - if (pMsg->acks.ackNum != 0) { - if (tqAck(pGroup, &pMsg->acks) != 0) { - // ack not success - return -1; - } - } - - STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - - if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) { - // fetch error - return -1; - } - - // judge and launch new query - /*if (tqSendLaunchQuery(gHandle)) {*/ - // launch query error - /*return -1;*/ - /*}*/ - return 0; -} -#endif - int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) { int32_t num = taosArrayGetSize(pConsumer->topics); - int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN); + int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + + num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN); if (sz > (*ppHead)->ssize) { void* tmpPtr = realloc(*ppHead, sz); if (tmpPtr == NULL) { @@ -511,13 +111,13 @@ int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** *(int64_t*)ptr = pTopic->committedOffset; POINTER_SHIFT(ptr, sizeof(int64_t)); } - + return 0; } const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle** ppConsumer) { STqConsumerHandle* pConsumer = *ppConsumer; - const void* ptr = pHead->content; + const void* ptr = pHead->content; pConsumer->consumerId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); pConsumer->epoch = *(int64_t*)ptr; @@ -534,165 +134,55 @@ const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHan return NULL; } -#if 0 -int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { - // calculate size - int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); - if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sz); - if (tmpPtr == NULL) { - free(*ppHead); - // TODO: memory err - return -1; - } - *ppHead = tmpPtr; - (*ppHead)->ssize = sz; - } - void* ptr = (*ppHead)->content; - // do serialization - *(int64_t*)ptr = pGroup->clientId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pGroup->cgId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = pGroup->topicNum; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if (pGroup->topicNum > 0) { - tqSerializeListHandle(pGroup->head, ptr); - } - return 0; -} - -void* tqSerializeListHandle(STqList* listHandle, void* ptr) { - STqList* node = listHandle; - ASSERT(node != NULL); - while (node) { - ptr = tqSerializeTopic(&node->topic, ptr); - node = node->next; - } - return ptr; -} - -void* tqSerializeTopic(STqTopic* pTopic, void* ptr) { - *(int64_t*)ptr = pTopic->nextConsumeOffset; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pTopic->topicId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - /**(int32_t*)ptr = pTopic->head;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - /**(int32_t*)ptr = pTopic->tail;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqSerializeItem(&pTopic->buffer[i], ptr); - } - return ptr; -} - -void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) { - // TODO: do we need serialize this? - // mainly for executor - return ptr; -} - -const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) { - STqGroup* gHandle = *ppGroup; - const void* ptr = pHead->content; - gHandle->clientId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - gHandle->cgId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - gHandle->ahandle = NULL; - gHandle->topicNum = *(int32_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - gHandle->head = NULL; - STqList* node = gHandle->head; - for (int i = 0; i < gHandle->topicNum; i++) { - if (gHandle->head == NULL) { - if ((node = malloc(sizeof(STqList))) == NULL) { - // TODO: error - return NULL; - } - node->next = NULL; - ptr = tqDeserializeTopic(ptr, &node->topic); - gHandle->head = node; - } else { - node->next = malloc(sizeof(STqList)); - if (node->next == NULL) { - // TODO: error - return NULL; - } - node->next->next = NULL; - ptr = tqDeserializeTopic(ptr, &node->next->topic); - node = node->next; - } - } - return ptr; -} - -const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) { - const void* ptr = pBytes; - topic->nextConsumeOffset = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - topic->topicId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - /*topic->head = *(int32_t*)ptr;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - /*topic->tail = *(int32_t*)ptr;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqDeserializeItem(ptr, &topic->buffer[i]); - } - return ptr; -} - -const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; } - -// TODO: make this a macro -int tqGroupSSize(const STqGroup* gHandle) { - return sizeof(int64_t) * 2 // cId + cgId - + sizeof(int32_t) // topicNum - + gHandle->topicNum * tqTopicSSize(); -} - -// TODO: make this a macro -int tqTopicSSize() { - return sizeof(int64_t) * 2 // nextConsumeOffset + topicId - + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqItemSSize(); -} - -int tqItemSSize() { - // TODO: do this need serialization? - // mainly for executor - return 0; -} -#endif int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { - SMqConsumeReq* pReq = pMsg->pCont; - SRpcMsg rpcMsg; - int64_t reqId = pReq->reqId; - int64_t consumerId = pReq->consumerId; - int64_t reqOffset = pReq->offset; - int64_t fetchOffset = reqOffset; - int64_t blockingTime = pReq->blockingTime; + SMqConsumeReq* pReq = pMsg->pCont; + int64_t reqId = pReq->reqId; + int64_t consumerId = pReq->consumerId; + int64_t fetchOffset = pReq->offset; + int64_t blockingTime = pReq->blockingTime; - int rspLen = 0; + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - int sz = taosArrayGetSize(pConsumer->topics); + if (pConsumer == NULL) { + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = -1; + rpcSendResponse(pMsg); + return 0; + } + int sz = taosArrayGetSize(pConsumer->topics); for (int i = 0; i < sz; i++) { STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); - //TODO: support multiple topic in one req + // TODO: support multiple topic in one req if (strcmp(pTopic->topicName, pReq->topic) != 0) { continue; } - if (fetchOffset == -1) { - fetchOffset = pTopic->committedOffset + 1; - } - int8_t pos; - int8_t skip = 0; + if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { + pTopic->committedOffset = pReq->offset; + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = 0; + rpcSendResponse(pMsg); + return 0; + } + + if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { + pTopic->committedOffset = pReq->offset-1; + } + + rsp.committedOffset = pTopic->committedOffset; + rsp.reqOffset = pReq->offset; + rsp.skipLogNum = 0; + + if (fetchOffset <= pTopic->committedOffset) { + fetchOffset = pTopic->committedOffset + 1; + } + int8_t pos; + int8_t skip = 0; SWalHead* pHead; while (1) { pos = fetchOffset % TQ_BUFFER_SIZE; @@ -712,6 +202,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (pHead->head.msgType == TDMT_VND_SUBMIT) { break; } + rsp.skipLogNum++; if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { atomic_store_8(&pTopic->buffer.output[pos].status, 0); skip = 1; @@ -726,7 +217,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { qSetStreamInput(task, pCont); - //SArray + // SArray SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); while (1) { SSDataBlock* pDataBlock; @@ -735,28 +226,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { break; } if (pDataBlock != NULL) { - SMqTbData tbData = { - .uid = pDataBlock->info.uid, - .numOfCols = pDataBlock->info.numOfCols, - .numOfRows = pDataBlock->info.rows, - }; - for (int i = 0; i < pDataBlock->info.numOfCols; i++) { - SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, i); - int32_t sz = pColData->info.bytes * pDataBlock->info.rows; - SMqColData colData = { - .bytes = pColData->info.bytes, - .colId = pColData->info.colId, - .type = pColData->info.type, - }; - memcpy(colData.data, pColData->pData, colData.bytes * pDataBlock->info.rows); - memcpy(&tbData.colData[i], &colData, sz); - } - /*pDataBlock->info.*/ taosArrayPush(pRes, pDataBlock); } else { break; } } + //TODO copy + rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; + rsp.rspOffset = fetchOffset; atomic_store_8(&pTopic->buffer.output[pos].status, 0); @@ -764,8 +241,13 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { taosArrayDestroy(pRes); fetchOffset++; continue; + } else { + rsp.numOfTopics++; } + rsp.pBlockData = pRes; + +#if 0 pTopic->buffer.output[pos].dst = pRes; if (pTopic->buffer.firstOffset == -1 || pReq->offset < pTopic->buffer.firstOffset) { pTopic->buffer.firstOffset = pReq->offset; @@ -773,25 +255,54 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (pTopic->buffer.lastOffset == -1 || pReq->offset > pTopic->buffer.lastOffset) { pTopic->buffer.lastOffset = pReq->offset; } - // put output into rsp - SMqConsumeRsp rsp = { - .consumerId = consumerId, - .numOfTopics = 1 - }; +#endif } - + int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + return -1; + } + void* abuf = buf; + tEncodeSMqConsumeRsp(&abuf, &rsp); + if (rsp.pBlockData) { + taosArrayDestroyEx(rsp.pBlockData, (void(*)(void*))tDeleteSSDataBlock); + rsp.pBlockData = NULL; + /*for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) {*/ + /*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/ + /*tDeleteSSDataBlock(pBlock);*/ + /*}*/ + /*taosArrayDestroy(rsp.pBlockData);*/ + } + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + rpcSendResponse(pMsg); return 0; } int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { - SMqSetCVgReq req; + SMqSetCVgReq req = {0}; tDecodeSMqSetCVgReq(msg, &req); - STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); + + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); if (pConsumer == NULL) { - return -1; + pConsumer = calloc(sizeof(STqConsumerHandle), 1); + if (pConsumer == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + } else { + tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.newConsumerId); + tqHandlePurge(pTq->tqMeta, req.oldConsumerId); + terrno = TSDB_CODE_SUCCESS; + return 0; } strcpy(pConsumer->cgroup, req.cgroup); pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); + pConsumer->consumerId = req.newConsumerId; + pConsumer->epoch = 0; STqTopicHandle* pTopic = calloc(sizeof(STqTopicHandle), 1); if (pTopic == NULL) { @@ -799,9 +310,11 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { return -1; } strcpy(pTopic->topicName, req.topicName); - pTopic->sql = strdup(req.sql); - pTopic->logicalPlan = strdup(req.logicalPlan); - pTopic->physicalPlan = strdup(req.physicalPlan); + pTopic->sql = req.sql; + pTopic->logicalPlan = req.logicalPlan; + pTopic->physicalPlan = req.physicalPlan; + pTopic->committedOffset = -1; + pTopic->currentOffset = -1; pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; @@ -811,9 +324,13 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, pReadHandle); + SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pMeta }; + pTopic->buffer.output[i].pReadHandle = pReadHandle; + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); } taosArrayPush(pConsumer->topics, pTopic); + tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.newConsumerId); terrno = TSDB_CODE_SUCCESS; return 0; } @@ -823,23 +340,44 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { if (pReadHandle == NULL) { return NULL; } - pReadHandle->pMeta = pMeta; + pReadHandle->pVnodeMeta = pMeta; pReadHandle->pMsg = NULL; pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; + pReadHandle->sver = -1; + pReadHandle->pSchema = NULL; + pReadHandle->pSchemaWrapper = NULL; return pReadHandle; } void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { pReadHandle->pMsg = pMsg; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); pReadHandle->ver = ver; memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); } bool tqNextDataBlock(STqReadHandle* pHandle) { - while (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) >= 0) { - if (pHandle->tbUid == pHandle->pBlock->uid) return true; + while (1) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + return false; + } + if (pHandle->pBlock == NULL) return false; + + pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); + /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ + ASSERT(pHandle->tbIdHash); + void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); + if (ret != NULL) { + pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); + pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); + pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); + pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); + pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); + return true; + } } return false; } @@ -854,31 +392,77 @@ int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) } SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { - int32_t sversion = pHandle->pBlock->sversion; - SSchemaWrapper* pSchemaWrapper = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, true); - STSchema* pTschema = metaGetTbTSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion); - SArray* pArray = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); + /*int32_t sversion = pHandle->pBlock->sversion;*/ + // TODO set to real sversion + int32_t sversion = 0; + if (pHandle->sver != sversion) { + pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion); + + tb_uid_t quid; + STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid); + if (pTbCfg->type == META_CHILD_TABLE) { + quid = pTbCfg->ctbCfg.suid; + } else { + quid = pHandle->pBlock->uid; + } + pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); + pHandle->sver = sversion; + } + + STSchema* pTschema = pHandle->pSchema; + SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; + + int32_t numOfRows = pHandle->pBlock->numOfRows; + int32_t numOfCols = pHandle->pSchema->numOfCols; + int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + + //TODO: stable case + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); if (pArray == NULL) { return NULL; } - SColumnInfoData colInfo; - int sz = pSchemaWrapper->nCols * pSchemaWrapper->pSchema->bytes; - colInfo.pData = malloc(sz); - if (colInfo.pData == NULL) { - return NULL; + + int j = 0; + for (int32_t i = 0; i < colNumNeed; i++) { + int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); + while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { + j++; + } + SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; + SColumnInfoData colInfo = {0}; + int sz = numOfRows * pColSchema->bytes; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = colId; + colInfo.info.type = pColSchema->type; + + colInfo.pData = calloc(1, sz); + if (colInfo.pData == NULL) { + // TODO free + taosArrayDestroy(pArray); + return NULL; + } + taosArrayPush(pArray, &colInfo); } SMemRow row; - int32_t kvIdx; + int32_t kvIdx = 0; + int32_t curRow = 0; + tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { - for (int i = 0; i < pTschema->numOfCols && kvIdx < pTschema->numOfCols; i++) { - // TODO: filter out unused column - STColumn* pCol = schemaColAt(pTschema, i); + // get all wanted col of that block + for (int32_t i = 0; i < colNumNeed; i++) { + SColumnInfoData* pColData = taosArrayGet(pArray, i); + STColumn* pCol = schemaColAt(pTschema, i); + // TODO + ASSERT(pCol->colId == pColData->info.colId); void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - // TODO: handle varlen - memcpy(POINTER_SHIFT(colInfo.pData, pCol->offset), val, pCol->bytes); + memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), val, pCol->bytes); } + curRow++; } - taosArrayPush(pArray, &colInfo); return pArray; } diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 57e20010e3..d220966ba6 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -584,12 +584,30 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - if (pNode->handle.valueInTxn) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn) { + pMeta->pDeleter(pNode->handle.valueInTxn); + } - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; + pNode->handle.valueInTxn = TQ_DELETE_TOKEN; + tqLinkUnpersist(pMeta, pNode); + return 0; + } + } else { + pNode = pNode->next; + } + } + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; +} + +int32_t tqHandlePurge(STqMetaStore* pMeta, int64_t key) { + int64_t bucketKey = key & TQ_BUCKET_MASK; + STqMetaList* pNode = pMeta->bucket[bucketKey]; + while (pNode) { + if (pNode->handle.key == key) { + pNode->handle.valueInUse = TQ_DELETE_TOKEN; tqLinkUnpersist(pMeta, pNode); return 0; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ce06fb089e..3b4057a6e4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -81,7 +81,6 @@ enum { CHECKINFO_CHOSEN_BOTH = 2 //for update=2(merge case) }; - typedef struct STableCheckInfo { uint64_t tableId; TSKEY lastKey; @@ -160,7 +159,7 @@ static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGro static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); //static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); -static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle); +static void changeQueryHandleForInterpQuery(tsdbReaderT pHandle); static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); @@ -168,7 +167,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); //static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef); //static void* doFreeColumnInfoData(SArray* pColumnInfoData); //static void* destroyTableCheckInfo(SArray* pTableCheckInfo); -static bool tsdbGetExternalRow(tsdbReadHandleT pHandle); +static bool tsdbGetExternalRow(tsdbReaderT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -209,32 +208,7 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) { -// assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL); -// -// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; -// if (pTsdbReadHandle->pMemRef->ref++ == 0) { -// tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable); -// } -// -// taosArrayDestroy(psTable); -} - -static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { -// assert(pTsdbReadHandle != NULL); -// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; -// if (pMemRef == NULL) { // it has been freed -// return; -// } -// -// if (--pMemRef->ref == 0) { -// tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot)); -// } -// -// pTsdbReadHandle->pMemRef = NULL; -} - -//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { +//int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { // STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // // int64_t rows = 0; @@ -263,9 +237,9 @@ static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { // return rows; //} -static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) { - size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); - assert(sizeOfGroup >= 1); +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList) { + size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); + assert(numOfGroup >= 1); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo)); @@ -273,14 +247,8 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S return NULL; } - SArray* pTable = taosArrayInit(4, sizeof(STable*)); - if (pTable == NULL) { - taosArrayDestroy(pTableCheckInfo); - return NULL; - } - // todo apply the lastkey of table check to avoid to load header file - for (int32_t i = 0; i < sizeOfGroup; ++i) { + for (int32_t i = 0; i < numOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(pGroupList->pGroupList, i); size_t gsize = taosArrayGetSize(group); @@ -289,12 +257,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S for (int32_t j = 0; j < gsize; ++j) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); - STableCheckInfo info = { .lastKey = pKeyInfo->lastKey}; -// assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || -// info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); - - info.tableId = pKeyInfo->uid; - + STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid}; if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) { info.lastKey = pTsdbReadHandle->window.skey; @@ -310,13 +273,8 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S } } -// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); - size_t gsize = taosArrayGetSize(pTableCheckInfo); -// for (int32_t i = 0; i < gsize; ++i) { -// STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); -// } - - *psTable = pTable; + // TODO group table according to the tag value. + taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); return pTableCheckInfo; } @@ -466,40 +424,39 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); - return (tsdbReadHandleT)pReadHandle; + return (tsdbReaderT)pReadHandle; _end: - tsdbCleanupQueryHandle(pReadHandle); + tsdbCleanupReadHandle(pReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } -tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } if (emptyQueryTimewindow(pTsdbReadHandle)) { - return (tsdbReadHandleT*) pTsdbReadHandle; + return (tsdbReaderT*) pTsdbReadHandle; } - SArray* psTable = NULL; - // todo apply the lastkey of table check to avoid to load header file - pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList); if (pTsdbReadHandle->pTableCheckInfo == NULL) { -// tsdbCleanupQueryHandle(pTsdbReadHandle); - taosArrayDestroy(psTable); +// tsdbCleanupReadHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - tsdbDebug("%p total numOfTable:%" PRIzu " in query, %s", pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->idStr); - return (tsdbReadHandleT) pTsdbReadHandle; + tsdbDebug("%p total numOfTable:%" PRIzu " in this query, group %"PRIzu" %s", pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), + taosArrayGetSize(groupList->pGroupList), pTsdbReadHandle->idStr); + + return (tsdbReaderT) pTsdbReadHandle; } -void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { +void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond *pCond) { STsdbReadHandle* pTsdbReadHandle = queryHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { @@ -536,7 +493,7 @@ void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { +void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { STsdbReadHandle* pTsdbReadHandle = queryHandle; pTsdbReadHandle->order = pCond->order; @@ -568,7 +525,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable); if (pTsdbReadHandle->pTableCheckInfo == NULL) { -// tsdbCleanupQueryHandle(pTsdbReadHandle); +// tsdbCleanupReadHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; } @@ -576,7 +533,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -604,7 +561,7 @@ tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup } #if 0 -tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); if (pTsdbReadHandle == NULL) { return NULL; @@ -624,7 +581,7 @@ tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGro } #endif -SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) { +SArray* tsdbGetQueriedTableList(tsdbReaderT *pHandle) { assert(pHandle != NULL); STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle; @@ -667,7 +624,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -2384,7 +2341,7 @@ static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { cur->blockCompleted = false; } #if 0 -int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTableBlockInfo) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; @@ -2540,7 +2497,7 @@ static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { } //todo not unref yet, since it is not support multi-group interpolation query -static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) { // filter the queried time stamp in the first place STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; @@ -2643,7 +2600,7 @@ static int32_t getAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.pTable = NULL, .lastKey = TSKEY_INITIAL_VAL, uid = id}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; taosArrayPush(list, &info); } @@ -2681,7 +2638,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { } if (exists) { - tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL); + tsdbRetrieveDataBlock((tsdbReaderT*) pTsdbReadHandle, NULL); if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); @@ -2937,7 +2894,7 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { } // handle data in cache situation -bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { +bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { @@ -3097,11 +3054,11 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { // } // //out_of_memory: -// tsdbCleanupQueryHandle(pSecQueryHandle); +// tsdbCleanupReadHandle(pSecQueryHandle); // return terrno; //} -bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { +bool tsdbGetExternalRow(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; SQueryFilePos* cur = &pTsdbReadHandle->cur; @@ -3158,7 +3115,7 @@ bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { // return code; //} -bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) { +bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle) { return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } @@ -3234,7 +3191,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { if (key < lastKey) { key = lastKey; - keyInfo.pTable = pInfo->pTable; +// keyInfo.pTable = pInfo->pTable; keyInfo.lastKey = key; pInfo->lastKey = key; @@ -3248,29 +3205,19 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { } } - // clear current group, unref unused table - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); - - // keyInfo.pTable may be NULL here. - if (pInfo->pTable != keyInfo.pTable) { -// tsdbUnRefTable(pInfo->pTable); - } - } - // more than one table in each group, only one table left for each group - if (keyInfo.pTable != NULL) { - totalNumOfTable++; - if (taosArrayGetSize(pGroup) == 1) { - // do nothing - } else { - taosArrayClear(pGroup); - taosArrayPush(pGroup, &keyInfo); - } - } else { // mark all the empty groups, and remove it later - taosArrayDestroy(pGroup); - taosArrayPush(emptyGroup, &j); - } +// if (keyInfo.pTable != NULL) { +// totalNumOfTable++; +// if (taosArrayGetSize(pGroup) == 1) { +// // do nothing +// } else { +// taosArrayClear(pGroup); +// taosArrayPush(pGroup, &keyInfo); +// } +// } else { // mark all the empty groups, and remove it later +// taosArrayDestroy(pGroup); +// taosArrayPush(emptyGroup, &j); +// } } // window does not being updated, so set the original @@ -3286,7 +3233,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { return window; } -void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { +void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; SQueryFilePos* cur = &pHandle->cur; @@ -3310,7 +3257,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) { +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStatis** pBlockStatis) { STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle; SQueryFilePos* c = &pHandle->cur; @@ -3365,7 +3312,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataS return TSDB_CODE_SUCCESS; } -SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) { +SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range @@ -3460,11 +3407,13 @@ void filterPrepare(void* expr, void* param) { } } +#endif static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { +#if 0 STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; - STable* pTable1 = ((STableKeyInfo*) p1)->pTable; - STable* pTable2 = ((STableKeyInfo*) p2)->pTable; + STable* pTable1 = ((STableKeyInfo*) p1)->uid; + STable* pTable2 = ((STableKeyInfo*) p2)->uid; for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) { SColIndex* pColIndex = &pTableGroupSupp->pCols[i]; @@ -3512,10 +3461,9 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa return ret; } } - +#endif return 0; } -#endif static int tsdbCheckInfoCompar(const void* key1, const void* key2) { if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { @@ -3531,10 +3479,9 @@ static int tsdbCheckInfoCompar(const void* key1, const void* key2) { void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); - SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; + STableKeyInfo info = {.lastKey = skey}; taosArrayPush(g, &info); for (int32_t i = 1; i < numOfTables; ++i) { @@ -3545,13 +3492,13 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable assert(ret == 0 || ret == -1); if (ret == 0) { - STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + STableKeyInfo info1 = {.lastKey = skey}; taosArrayPush(g, &info1); } else { taosArrayPush(pGroups, &g); // current group is ended, start a new group g = taosArrayInit(16, sizeof(STableKeyInfo)); - STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + STableKeyInfo info1 = {.lastKey = skey}; taosArrayPush(g, &info1); } } @@ -3584,8 +3531,8 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd sup.pTagSchema = pTagSchema->pSchema; sup.pCols = pCols; -// taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); -// createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; @@ -3691,29 +3638,29 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd // return TSDB_CODE_SUCCESS; //} -int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId) { - STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid); + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) { + STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); if (pTbCfg == NULL) { - tsdbError("%p failed to get stable, uid:%"PRIu64", reqId:0x%"PRIx64, tsdb, uid, reqId); +// tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; } if (pTbCfg->type != META_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", reId:0x%"PRIx64, tsdb, uid, reqId); +// tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client goto _error; } //NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - SSchemaWrapper* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); + SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { - int32_t ret = getAllTableList(tsdb->pMeta, uid, res); + int32_t ret = getAllTableList(pMeta, uid, res); if (ret != TSDB_CODE_SUCCESS) { goto _error; } @@ -3721,8 +3668,8 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu", tsdb, - pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); +// tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, +// pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); taosArrayDestroy(res); return ret; @@ -3778,26 +3725,19 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch return terrno; } -#if 0 -int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { - if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; - - STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - if (pTable == NULL) { +int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { + STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); + if (pTbCfg == NULL) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - tsdbUnlockRepoMeta(tsdb); goto _error; } - assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); - if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; - pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = pTable, .lastKey = startKey}; + STableKeyInfo info = {.lastKey = startKey, .uid = uid}; taosArrayPush(group, &info); taosArrayPush(pGroupInfo->pGroupList, &group); @@ -3807,6 +3747,7 @@ int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGr return terrno; } +#if 0 int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; @@ -3882,7 +3823,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { } -void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { +void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; if (pTsdbReadHandle == NULL) { return; @@ -3895,7 +3836,7 @@ void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { tfree(pTsdbReadHandle->statis); if (!emptyQueryTimewindow(pTsdbReadHandle)) { - tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); +// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); } else { assert(pTsdbReadHandle->pTableCheckInfo == NULL); } diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index d762844120..477deed8c8 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -26,6 +26,7 @@ int vnodeInit(const SVnodeOpt *pOption) { vnodeMgr.stop = false; vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; + vnodeMgr.sendReqToDnodeFp = pOption->sendReqToDnodeFp; // Start commit handers if (pOption->nthreads > 0) { @@ -96,6 +97,10 @@ int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); } +void vnodeSendReqToDnode(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { + (*vnodeMgr.sendReqToDnodeFp)(pVnode->pDnode, epSet, pReq); +} + /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { setThreadName("vnode-commit"); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index b211894281..a45a9d5a72 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -21,17 +21,19 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, - (putReqToQueryQFp)vnodePutReqToVQueryQ); + (putReqToQueryQFp)vnodePutReqToVQueryQ, (sendReqToDnodeFp)vnodeSendReqToDnode); } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in query queue is processing"); + SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta}; switch (pMsg->msgType) { - case TDMT_VND_QUERY: - return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + case TDMT_VND_QUERY:{ + return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); + } case TDMT_VND_QUERY_CONTINUE: - return qWorkerProcessCQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; @@ -43,6 +45,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_VND_FETCH: return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg); + case TDMT_VND_FETCH_RSP: + return qWorkerProcessFetchRsp(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_RES_READY: return qWorkerProcessReadyMsg(pVnode, pVnode->pQuery, pMsg); case TDMT_VND_TASKS_STATUS: @@ -80,7 +84,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { int msgLen = 0; int32_t code = TSDB_CODE_VND_APP_ERROR; - pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); + pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tbName, &uid); if (pTbCfg == NULL) { code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; @@ -115,13 +119,13 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - memcpy(pTbMetaMsg->dbFname, pReq->dbFname, sizeof(pTbMetaMsg->dbFname)); - strcpy(pTbMetaMsg->tbFname, pTbCfg->name); + memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); + strcpy(pTbMetaMsg->tbName, pReq->tbName); if (pTbCfg->type == META_CHILD_TABLE) { - strcpy(pTbMetaMsg->stbFname, pStbCfg->name); + strcpy(pTbMetaMsg->stbName, pStbCfg->name); pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid); } else if (pTbCfg->type == META_SUPER_TABLE) { - strcpy(pTbMetaMsg->stbFname, pTbCfg->name); + strcpy(pTbMetaMsg->stbName, pTbCfg->name); pTbMetaMsg->suid = htobe64(uid); } pTbMetaMsg->numOfTags = htonl(nTagCols); diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 326d99ddbb..40cb02176b 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -16,6 +16,7 @@ #include "tq.h" #include "vnd.h" +#if 0 int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_VND_MQ_SET_CUR: @@ -26,6 +27,7 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { } return 0; } +#endif int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { SRpcMsg *pMsg; @@ -103,7 +105,12 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayDestroy(vCreateTbBatchReq.pArray); break; + case TDMT_VND_ALTER_STB: + vTrace("vgId:%d, process drop stb req", pVnode->vgId); + break; case TDMT_VND_DROP_STB: + vTrace("vgId:%d, process drop stb req", pVnode->vgId); + break; case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { // // TODO: handle error diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 9f1ea754c2..9c041d76c7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -27,11 +27,11 @@ extern "C" { #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_DB_NUMBER 20 -#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000 +#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 10000 #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 -#define CTG_RENT_SLOT_SECOND 2 +#define CTG_RENT_SLOT_SECOND 1.5 #define CTG_DEFAULT_INVALID_VERSION (-1) @@ -47,55 +47,52 @@ enum { CTG_RENT_STABLE, }; -typedef struct SCTGDebug { +typedef struct SCtgDebug { int32_t lockDebug; -} SCTGDebug; +} SCtgDebug; -typedef struct SVgroupListCache { - int32_t vgroupVersion; - SHashObj *cache; // key:vgId, value:SVgroupInfo -} SVgroupListCache; +typedef struct SCtgTbMetaCache { + SRWLatch stbLock; + SHashObj *cache; //key:tbname, value:STableMeta + SHashObj *stbCache; //key:suid, value:STableMeta* +} SCtgTbMetaCache; -typedef struct SDBVgroupCache { - SHashObj *cache; //key:dbname, value:SDBVgroupInfo -} SDBVgroupCache; +typedef struct SCtgDBCache { + SRWLatch vgLock; + int8_t deleted; + SDBVgroupInfo *vgInfo; + SCtgTbMetaCache tbCache; +} SCtgDBCache; -typedef struct STableMetaCache { - SRWLatch stableLock; - SHashObj *cache; //key:fulltablename, value:STableMeta - SHashObj *stableCache; //key:suid, value:STableMeta* -} STableMetaCache; - -typedef struct SRentSlotInfo { +typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; SArray *meta; // element is SDbVgVersion or SSTableMetaVersion -} SRentSlotInfo; +} SCtgRentSlot; -typedef struct SMetaRentMgmt { +typedef struct SCtgRentMgmt { int8_t type; uint16_t slotNum; uint16_t slotRIdx; int64_t lastReadMsec; - SRentSlotInfo *slots; -} SMetaRentMgmt; + SCtgRentSlot *slots; +} SCtgRentMgmt; typedef struct SCatalog { - uint64_t clusterId; - SDBVgroupCache dbCache; - STableMetaCache tableCache; - SMetaRentMgmt dbRent; - SMetaRentMgmt stableRent; + uint64_t clusterId; + SHashObj *dbCache; //key:dbname, value:SCtgDBCache + SCtgRentMgmt dbRent; + SCtgRentMgmt stbRent; } SCatalog; typedef struct SCtgApiStat { } SCtgApiStat; -typedef struct SCtgResourceStat { +typedef struct SCtgRuntimeStat { -} SCtgResourceStat; +} SCtgRuntimeStat; typedef struct SCtgCacheStat { @@ -103,7 +100,7 @@ typedef struct SCtgCacheStat { typedef struct SCatalogStat { SCtgApiStat api; - SCtgResourceStat resource; + SCtgRuntimeStat runtime; SCtgCacheStat cache; } SCatalogStat; @@ -115,7 +112,7 @@ typedef struct SCatalogMgmt { typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); -#define CTG_IS_META_NONE(type) ((type) == META_TYPE_NON_TABLE) +#define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) #define CTG_IS_META_TABLE(type) ((type) == META_TYPE_TABLE) #define CTG_IS_META_BOTH(type) ((type) == META_TYPE_BOTH_TABLE) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 2197fdfd62..02773fe533 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -20,31 +20,106 @@ SCatalogMgmt ctgMgmt = {0}; -SCTGDebug gCTGDebug = {0}; +SCtgDebug gCTGDebug = {0}; -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, bool *inCache) { - if (NULL == pCatalog->dbCache.cache) { +void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { + if (NULL == mgmt->slots) { + return; + } + + for (int32_t i = 0; i < mgmt->slotNum; ++i) { + SCtgRentSlot *slot = &mgmt->slots[i]; + if (slot->meta) { + taosArrayDestroy(slot->meta); + slot->meta = NULL; + } + } + + tfree(mgmt->slots); +} + + +void ctgFreeTableMetaCache(SCtgTbMetaCache *table) { + CTG_LOCK(CTG_WRITE, &table->stbLock); + if (table->stbCache) { + taosHashCleanup(table->stbCache); + table->stbCache = NULL; + } + CTG_UNLOCK(CTG_WRITE, &table->stbLock); + + if (table->cache) { + taosHashCleanup(table->cache); + table->cache = NULL; + } +} + +void ctgFreeDbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache) { + return; + } + + atomic_store_8(&dbCache->deleted, 1); + + SDBVgroupInfo *dbInfo = NULL; + if (dbCache->vgInfo) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (dbCache->vgInfo->vgHash) { + taosHashCleanup(dbCache->vgInfo->vgHash); + dbCache->vgInfo->vgHash = NULL; + } + + tfree(dbCache->vgInfo); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + } + + ctgFreeTableMetaCache(&dbCache->tbCache); +} + + +void ctgFreeHandle(struct SCatalog* pCatalog) { + ctgFreeMetaRent(&pCatalog->dbRent); + ctgFreeMetaRent(&pCatalog->stbRent); + if (pCatalog->dbCache) { + void *pIter = taosHashIterate(pCatalog->dbCache, NULL); + while (pIter) { + SCtgDBCache *dbCache = pIter; + + ctgFreeDbCache(dbCache); + + pIter = taosHashIterate(pCatalog->dbCache, pIter); + } + + taosHashCleanup(pCatalog->dbCache); + } + + free(pCatalog); +} + + +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SCtgDBCache **dbCache, bool *inCache) { + if (NULL == pCatalog->dbCache) { *inCache = false; ctgWarn("empty db cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - SDBVgroupInfo *info = NULL; + SCtgDBCache *cache = NULL; while (true) { - info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); + cache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == info) { + if (NULL == cache) { *inCache = false; ctgWarn("not in db vgroup cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &info->lock); - if (NULL == info->vgInfo) { - CTG_UNLOCK(CTG_READ, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); + CTG_LOCK(CTG_READ, &cache->vgLock); + if (NULL == cache->vgInfo) { + CTG_UNLOCK(CTG_READ, &cache->vgLock); + taosHashRelease(pCatalog->dbCache, cache); ctgWarn("db cache vgInfo is NULL, dbName:%s", dbName); continue; @@ -53,7 +128,7 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S break; } - *dbInfo = info; + *dbCache = cache; *inCache = true; ctgDebug("Got db vgroup from cache, dbName:%s", dbName); @@ -65,7 +140,6 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, SBuildUseDBInput *input, SUseDbOutput *out) { char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; ctgDebug("try to get db vgroup from mnode, db:%s", input->db); @@ -99,48 +173,73 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp return TSDB_CODE_SUCCESS; } -int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, const char* tbFullName, int32_t *exist) { - if (NULL == pCatalog->tableCache.cache) { +int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCatalog->dbCache) { *exist = 0; - ctgWarn("empty tablemeta cache, tbName:%s", tbFullName); + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - size_t sz = 0; - STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName)); - - if (NULL == tbMeta) { + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + if (NULL == dbCache) { *exist = 0; - ctgDebug("tablemeta not in cache, tbName:%s", tbFullName); + ctgWarn("db not exist in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + + size_t sz = 0; + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.cache, tbName, strlen(tbName)); + if (NULL == tbMeta) { + taosHashRelease(pCatalog->dbCache, dbCache); + + *exist = 0; + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } *exist = 1; + + taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("tablemeta is in cache, tbName:%s", tbFullName); + ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { - if (NULL == pCatalog->tableCache.cache) { + if (NULL == pCatalog->dbCache) { *exist = 0; - ctgWarn("empty tablemeta cache, tbName:%s", pTableName->tname); + ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); *pTableMeta = NULL; - size_t sz = 0; - STableMeta *tbMeta = taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)pTableMeta, &sz); + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, db, strlen(db)); + if (NULL == dbCache) { + *exist = 0; + ctgWarn("no db cache, dbFName:%s, tbName:%s", db, pTableName->tname); + return TSDB_CODE_SUCCESS; + } + if (NULL == dbCache->tbCache.cache) { + *exist = 0; + taosHashRelease(pCatalog->dbCache, dbCache); + ctgWarn("empty tbmeta cache, dbFName:%s, tbName:%s", db, pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + size_t sz = 0; + STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.cache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz); if (NULL == *pTableMeta) { *exist = 0; - ctgDebug("tablemeta not in cache, tbName:%s", tbFullName); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -149,91 +248,91 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgDebug("Got tablemeta from cache, tbName:%s", tbFullName); - + taosHashRelease(pCatalog->dbCache, dbCache); + ctgDebug("Got tbmeta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, db, pTableName->tname); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid)); + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); - ctgError("stable not in stableCache, suid:%"PRIx64, tbMeta->suid); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgError("stable not in stbCache, suid:%"PRIx64, tbMeta->suid); tfree(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); tfree(*pTableMeta); - ctgError("stable suid in stableCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); *pTableMeta = realloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); ctgError("realloc size[%d] failed", metaSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgDebug("Got tablemeta from cache, tbName:%s", tbFullName); + taosHashRelease(pCatalog->dbCache, dbCache); + + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { - if (NULL == pCatalog->tableCache.cache) { - ctgWarn("empty tablemeta cache, tbName:%s", pTableName->tname); + if (NULL == pCatalog->dbCache) { + ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbName); - size_t sz = 0; - STableMeta *pTableMeta = NULL; - - taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)&pTableMeta, &sz); + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not in cache, dbFName:%s", dbName); + return TSDB_CODE_SUCCESS; + } + STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.cache, pTableName->tname, strlen(pTableName->tname)); if (NULL == pTableMeta) { - ctgWarn("tablemeta not in cache, tbName:%s", tbFullName); - + ctgWarn("tbmeta not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); + taosHashRelease(pCatalog->dbCache, dbCache); + return TSDB_CODE_SUCCESS; } - *tbType = pTableMeta->tableType; + *tbType = atomic_load_8(&pTableMeta->tableType); - ctgDebug("Got tabletype from cache, tbName:%s, type:%d", tbFullName, *tbType); + taosHashRelease(dbCache->tbCache.cache, dbCache); + taosHashRelease(pCatalog->dbCache, dbCache); + + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbName, pTableName->tname, *tbType); return TSDB_CODE_SUCCESS; } - -void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { - epSet->inUse = 0; - epSet->numOfEps = vgroupInfo->numOfEps; - - for (int32_t i = 0; i < vgroupInfo->numOfEps; ++i) { - memcpy(&epSet->port[i], &vgroupInfo->epAddr[i].port, sizeof(epSet->port[i])); - memcpy(&epSet->fqdn[i], &vgroupInfo->epAddr[i].fqdn, sizeof(epSet->fqdn[i])); - } -} - -int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) { - SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; +int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { + SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - ctgDebug("try to get table meta from mnode, tbName:%s", tbFullName); + ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen); if (code) { @@ -253,31 +352,31 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - SET_META_TYPE_NONE(output->metaType); - ctgDebug("stablemeta not exist in mnode, tbName:%s", tbFullName); + SET_META_TYPE_NULL(output->metaType); + ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - ctgError("error rsp for stablemeta from mnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tbFullName); + ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tbName); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process mnode stablemeta rsp failed, code:%x, tbName:%s", code, tbFullName); + ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); CTG_ERR_RET(code); } - ctgDebug("Got table meta from mnode, tbName:%s", tbFullName); + ctgDebug("Got table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, tbFullName, output); + return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); } int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { @@ -285,19 +384,18 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - char dbFullName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFullName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); - ctgDebug("try to get table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); + ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)tNameGetTableName(pTableName)}; + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; char *msg = NULL; - SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName)); + ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } @@ -308,29 +406,26 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, }; SRpcMsg rpcRsp = {0}; - SEpSet epSet; - - ctgGenEpSet(&epSet, vgroupInfo); - rpcSendRecv(pTransporter, &epSet, &rpcMsg, &rpcRsp); + rpcSendRecv(pTransporter, &vgroupInfo->epset, &rpcMsg, &rpcRsp); if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { - SET_META_TYPE_NONE(output->metaType); - ctgDebug("tablemeta not exist in vnode, tbName:%s", tNameGetTableName(pTableName)); + SET_META_TYPE_NULL(output->metaType); + ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } - ctgError("error rsp for table meta from vnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tNameGetTableName(pTableName)); + ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%s, tbName:%s", tstrerror(code), tNameGetTableName(pTableName)); + ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } - ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); + ctgDebug("Got table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } @@ -350,7 +445,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { @@ -358,7 +453,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; @@ -367,7 +462,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(dbInfo->vgInfo, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } @@ -390,7 +485,7 @@ _return: int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -402,40 +497,39 @@ int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbIn tableNameHashFp fp = NULL; SVgroupInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); char tbFullName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFullName); uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); - void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgInfo, pIter); + taosHashCancelIterate(dbInfo->vgHash, pIter); break; } - pIter = taosHashIterate(dbInfo->vgInfo, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgInfo)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } *pVgroup = *vgInfo; -_return: CTG_RET(code); } int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { return 1; } else { return 0; @@ -443,9 +537,9 @@ int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { } int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { - if (((SDbVgVersion*)key1)->dbId < ((SDbVgVersion*)key2)->dbId) { + if (*(int64_t *)key1 < ((SDbVgVersion*)key2)->dbId) { return -1; - } else if (((SDbVgVersion*)key1)->dbId > ((SDbVgVersion*)key2)->dbId) { + } else if (*(int64_t *)key1 > ((SDbVgVersion*)key2)->dbId) { return 1; } else { return 0; @@ -453,17 +547,17 @@ int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { } -int32_t ctgMetaRentInit(SMetaRentMgmt *mgmt, uint32_t rentSec, int8_t type) { +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; mgmt->type = type; - size_t msgSize = sizeof(SRentSlotInfo) * mgmt->slotNum; + size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; mgmt->slots = calloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); @@ -472,10 +566,10 @@ int32_t ctgMetaRentInit(SMetaRentMgmt *mgmt, uint32_t rentSec, int8_t type) { } -int32_t ctgMetaRentAdd(SMetaRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { +int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); @@ -502,22 +596,22 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentUpdate(SMetaRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t compare) { +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t compare) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("meta in slot is empty, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (slot->needSort) { taosArraySort(slot->meta, compare); slot->needSort = false; - qDebug("slot meta sorted, slot idx:%d, type:%d", widx, mgmt->type); + qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type); } void *orig = taosArraySearch(slot->meta, &id, compare, TD_EQ); @@ -542,14 +636,50 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentGetImpl(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { +int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t compare) { + int16_t widx = abs(id % mgmt->slotNum); + + SCtgRentSlot *slot = &mgmt->slots[widx]; + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &slot->lock); + if (NULL == slot->meta) { + qError("empty meta slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (slot->needSort) { + taosArraySort(slot->meta, compare); + slot->needSort = false; + qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type); + } + + int32_t idx = taosArraySearchIdx(slot->meta, &id, compare, TD_EQ); + if (idx < 0) { + qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + taosArrayRemove(slot->meta, idx); + + qDebug("meta in rent removed, id:%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + +_return: + + CTG_UNLOCK(CTG_WRITE, &slot->lock); + + CTG_RET(code); +} + + +int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); if (ridx >= mgmt->slotNum) { ridx %= mgmt->slotNum; atomic_store_16(&mgmt->slotRIdx, ridx); } - SRentSlotInfo *slot = &mgmt->slots[ridx]; + SCtgRentSlot *slot = &mgmt->slots[ridx]; int32_t code = 0; CTG_LOCK(CTG_READ, &slot->lock); @@ -588,7 +718,7 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentGet(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { +int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { while (true) { int64_t msec = taosGetTimestampMs(); int64_t lsec = atomic_load_64(&mgmt->lastReadMsec); @@ -615,52 +745,79 @@ int32_t ctgMetaRentGet(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t s int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { int32_t code = 0; + SCtgDBCache *dbCache = NULL; - if (NULL == output->tbMeta) { - ctgError("no valid table meta got from meta rsp, tbName:%s", output->tbFname); + if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (NULL == pCatalog->tableCache.cache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->dbCache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == cache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.cache, NULL, cache)) { + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { taosHashCleanup(cache); } } - if (NULL == pCatalog->tableCache.stableCache) { + while (true) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, output->dbFName, strlen(output->dbFName)); + if (dbCache) { + break; + } + + SCtgDBCache newDbCache = {0}; + + if (taosHashPut(pCatalog->dbCache, output->dbFName, strlen(output->dbFName), &newDbCache, sizeof(newDbCache))) { + ctgError("taosHashPut db to cache failed, db:%s", output->dbFName); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + } + + if (NULL == dbCache->tbCache.cache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == cache) { + ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.cache, NULL, cache)) { + taosHashCleanup(cache); + } + } + + if (NULL == dbCache->tbCache.stbCache) { SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == cache) { ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.stableCache, NULL, cache)) { + if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.stbCache, NULL, cache)) { taosHashCleanup(cache); } } if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { - ctgError("taosHashPut ctablemeta to cache failed, ctbName:%s", output->ctbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + if (taosHashPut(dbCache->tbCache.cache, output->ctbName, strlen(output->ctbName), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { + ctgError("taosHashPut ctbmeta to cache failed, ctbName:%s", output->ctbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - ctgDebug("update child tablemeta to cache, tbName:%s", output->ctbFname); + ctgDebug("ctbmeta updated to cache, ctbName:%s", output->ctbName); } if (CTG_IS_META_CTABLE(output->metaType)) { - return TSDB_CODE_SUCCESS; + goto _return; } if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); @@ -668,45 +825,53 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { bool newAdded = false; SSTableMetaVersion metaRent = {.suid = output->tbMeta->suid, .sversion = output->tbMeta->sversion, .tversion = output->tbMeta->tversion}; + strcpy(metaRent.dbFName, output->dbFName); + strcpy(metaRent.stbName, output->tbName); - CTG_LOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - ctgError("taosHashPut tablemeta to cache failed, tbName:%s", output->tbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashPut(dbCache->tbCache.cache, output->tbName, strlen(output->tbName), output->tbMeta, tbSize) != 0) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgError("taosHashPut tablemeta to cache failed, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname)); - if (taosHashPutExt(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.cache, output->tbName, strlen(output->tbName)); + if (taosHashPutExt(dbCache->tbCache.stbCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, output->tbMeta->suid); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgDebug("update stable to cache, suid:%"PRIx64, output->tbMeta->suid); if (newAdded) { - CTG_ERR_RET(ctgMetaRentAdd(&pCatalog->stableRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); } else { - CTG_ERR_RET(ctgMetaRentUpdate(&pCatalog->stableRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); + CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); } } else { - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { - ctgError("taosHashPut tablemeta to cache failed, tbName:%s", output->tbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + if (taosHashPut(dbCache->tbCache.cache, output->tbName, strlen(output->tbName), output->tbMeta, tbSize) != 0) { + ctgError("taosHashPut tablemeta to cache failed, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } - ctgDebug("update tablemeta to cache, tbName:%s", output->tbFname); + ctgDebug("update tablemeta to cache, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + +_return: + + if (dbCache) { + taosHashRelease(pCatalog->dbCache, dbCache); + } CTG_RET(code); } -int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, bool forceUpdate, SDBVgroupInfo** dbInfo) { +int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, bool forceUpdate, SCtgDBCache** dbCache) { bool inCache = false; if (!forceUpdate) { - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbCache, &inCache)); if (inCache) { return TSDB_CODE_SUCCESS; } @@ -722,8 +887,8 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm while (true) { CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup)); - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, DbOut.dbVgroup)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbCache, &inCache)); if (!inCache) { ctgWarn("can't get db vgroup from cache, will retry, db:%s", dbName); @@ -737,32 +902,91 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm } -int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { - SDBVgroupInfo *oldInfo = (SDBVgroupInfo *)taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (oldInfo) { - CTG_LOCK(CTG_WRITE, &oldInfo->lock); - if (dbInfo->vgVersion <= oldInfo->vgVersion) { - ctgInfo("db vgVersion is not new, db:%s, vgVersion:%d, current:%d", dbName, dbInfo->vgVersion, oldInfo->vgVersion); - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); - taosHashRelease(pCatalog->dbCache.cache, oldInfo); - - return TSDB_CODE_SUCCESS; - } - - if (oldInfo->vgInfo) { - ctgInfo("cleanup db vgInfo, db:%s", dbName); - taosHashCleanup(oldInfo->vgInfo); - oldInfo->vgInfo = NULL; - } - - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); +int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, bool *removed) { + *removed = false; + + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); + return TSDB_CODE_SUCCESS; + } - taosHashRelease(pCatalog->dbCache.cache, oldInfo); + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (NULL == dbCache->vgInfo) { + ctgInfo("db vgInfo not in dbCache, may be removed, db:%s, dbId:%"PRIx64, dbName, dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->vgInfo->dbId != dbId) { + ctgInfo("db id already updated, db:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, dbName, dbCache->vgInfo->dbId, dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->vgInfo->vgHash) { + ctgInfo("cleanup db vgInfo, db:%s, dbId:%"PRIx64, dbName, dbId); + taosHashCleanup(dbCache->vgInfo->vgHash); + tfree(dbCache->vgInfo); } + if (taosHashRemove(pCatalog->dbCache, dbName, strlen(dbName))) { + ctgError("taosHashRemove from dbCache failed, db:%s", dbName); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + dbCache->deleted = true; + + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + + ctgFreeTableMetaCache(&dbCache->tbCache); + + taosHashRelease(pCatalog->dbCache, dbCache); + + *removed = true; + return TSDB_CODE_SUCCESS; } +int32_t ctgValidateAndRemoveStbMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid, bool *removed) { + *removed = false; + + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &suid, sizeof(suid))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgInfo("stb not exist in stbCache, may be removed, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); + return TSDB_CODE_SUCCESS; + } + + if (taosHashRemove(dbCache->tbCache.cache, stbName, strlen(stbName))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgError("stb not exist in cache, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + taosHashRelease(pCatalog->dbCache, dbCache); + + *removed = true; + + return TSDB_CODE_SUCCESS; +} + + + int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); @@ -778,50 +1002,54 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con STableMetaOutput *output = &voutput; if (CTG_IS_STABLE(isSTable)) { - ctgDebug("will renew table meta, supposed to be stable, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will renew tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); - if (CTG_IS_META_NONE(moutput.metaType)) { + if (CTG_IS_META_NULL(moutput.metaType)) { CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); } else { output = &moutput; } } else { - ctgDebug("will renew table meta, not supposed to be stable, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will renew tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - ctgDebug("will continue to renew table meta since got stable, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); + ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); + voutput.metaType = moutput.metaType; + tfree(voutput.tbMeta); voutput.tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else if (CTG_IS_META_BOTH(voutput.metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.tbFname, &exist)); + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.dbFName, voutput.tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); - if (CTG_IS_META_NONE(moutput.metaType)) { - SET_META_TYPE_NONE(voutput.metaType); + if (CTG_IS_META_NULL(moutput.metaType)) { + SET_META_TYPE_NULL(voutput.metaType); } tfree(voutput.tbMeta); voutput.tbMeta = moutput.tbMeta; moutput.tbMeta = NULL; } else { + tfree(voutput.tbMeta); + SET_META_TYPE_CTABLE(voutput.metaType); } } } - if (CTG_IS_META_NONE(output->metaType)) { + if (CTG_IS_META_NULL(output->metaType)) { ctgError("no tablemeta got, tbNmae:%s", tNameGetTableName(pTableName)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -869,68 +1097,10 @@ int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMg return TSDB_CODE_SUCCESS; } -void ctgFreeMetaRent(SMetaRentMgmt *mgmt) { - if (NULL == mgmt->slots) { - return; - } - - for (int32_t i = 0; i < mgmt->slotNum; ++i) { - SRentSlotInfo *slot = &mgmt->slots[i]; - if (slot->meta) { - taosArrayDestroy(slot->meta); - slot->meta = NULL; - } - } - - tfree(mgmt->slots); -} - -void ctgFreeDbCache(SDBVgroupCache *db) { - if (NULL == db->cache) { - return; - } - - SDBVgroupInfo *dbInfo = NULL; - void *pIter = taosHashIterate(db->cache, NULL); - while (pIter) { - dbInfo = pIter; - - if (dbInfo->vgInfo) { - taosHashCleanup(dbInfo->vgInfo); - dbInfo->vgInfo = NULL; - } - - pIter = taosHashIterate(db->cache, pIter); - } - - taosHashCleanup(db->cache); - db->cache = NULL; -} - -void ctgFreeTableMetaCache(STableMetaCache *table) { - if (table->stableCache) { - taosHashCleanup(table->stableCache); - table->stableCache = NULL; - } - - if (table->cache) { - taosHashCleanup(table->cache); - table->cache = NULL; - } -} - -void ctgFreeHandle(struct SCatalog* pCatalog) { - ctgFreeMetaRent(&pCatalog->dbRent); - ctgFreeMetaRent(&pCatalog->stableRent); - ctgFreeDbCache(&pCatalog->dbCache); - ctgFreeTableMetaCache(&pCatalog->tableCache); - - free(pCatalog); -} int32_t catalogInit(SCatalogCfg *cfg) { if (ctgMgmt.pCluster) { - qError("catalog already init"); + qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -949,14 +1119,14 @@ int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; } - if (ctgMgmt.cfg.stableRentSec == 0) { - ctgMgmt.cfg.stableRentSec = CTG_DEFAULT_RENT_SECOND; + if (ctgMgmt.cfg.stbRentSec == 0) { + ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } } else { ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; - ctgMgmt.cfg.stableRentSec = CTG_DEFAULT_RENT_SECOND; + ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); @@ -965,7 +1135,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stableRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stableRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } @@ -976,7 +1146,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { } if (NULL == ctgMgmt.pCluster) { - qError("cluster cache are not ready, clusterId:%"PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } @@ -1001,7 +1171,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { clusterCtg->clusterId = clusterId; CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, ctgMgmt.cfg.dbRentSec, CTG_RENT_DB)); - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stableRent, ctgMgmt.cfg.stableRentSec, CTG_RENT_STABLE)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, ctgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); code = taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { @@ -1052,21 +1222,33 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCatalog->dbCache.cache) { + if (NULL == pCatalog->dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; ctgInfo("empty db cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - SDBVgroupInfo * dbInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (NULL == dbInfo) { + SCtgDBCache *db = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == db) { *version = CTG_DEFAULT_INVALID_VERSION; ctgInfo("db not in cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - *version = dbInfo->vgVersion; - taosHashRelease(pCatalog->dbCache.cache, dbInfo); + CTG_LOCK(CTG_READ, &db->vgLock); + + if (NULL == db->vgInfo) { + CTG_UNLOCK(CTG_READ, &db->vgLock); + + *version = CTG_DEFAULT_INVALID_VERSION; + ctgInfo("db not in cache, dbName:%s", dbName); + return TSDB_CODE_SUCCESS; + } + + *version = db->vgInfo->vgVersion; + CTG_UNLOCK(CTG_READ, &db->vgLock); + + taosHashRelease(pCatalog->dbCache, db); ctgDebug("Got db vgVersion from cache, dbName:%s, vgVersion:%d", dbName, *version); @@ -1078,29 +1260,31 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - SDBVgroupInfo* db = NULL; + SCtgDBCache* dbCache = NULL; SVgroupInfo *vgInfo = NULL; int32_t code = 0; SArray *vgList = NULL; - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbName, forceUpdate, &db)); + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbName, forceUpdate, &dbCache)); - vgList = taosArrayInit(taosHashGetSize(db->vgInfo), sizeof(SVgroupInfo)); + int32_t vgNum = (int32_t)taosHashGetSize(dbCache->vgInfo->vgHash); + vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { - ctgError("taosArrayInit %d failed", taosHashGetSize(db->vgInfo)); + ctgError("taosArrayInit %d failed", vgNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(db->vgInfo, NULL); + void *pIter = taosHashIterate(dbCache->vgInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(dbCache->vgInfo->vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(db->vgInfo, pIter); + pIter = taosHashIterate(dbCache->vgInfo->vgHash, pIter); vgInfo = NULL; } @@ -1108,9 +1292,10 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* vgList = NULL; _return: - if (db) { - CTG_UNLOCK(CTG_READ, &db->lock); - taosHashRelease(pCatalog->dbCache.cache, db); + + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } if (vgList) { @@ -1129,66 +1314,148 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == dbInfo->vgInfo || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgInfo) <= 0) { - ctgError("invalid db vgInfo, dbName:%s, vgInfo:%p, vgVersion:%d", dbName, dbInfo->vgInfo, dbInfo->vgVersion); + if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbName:%s, vgHash:%p, vgVersion:%d", dbName, dbInfo->vgHash, dbInfo->vgVersion); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (dbInfo->vgVersion < 0) { - ctgWarn("db vgVersion less than 0, dbName:%s, vgVersion:%d", dbName, dbInfo->vgVersion); - - if (pCatalog->dbCache.cache) { - CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo)); - - CTG_ERR_JRET(taosHashRemove(pCatalog->dbCache.cache, dbName, strlen(dbName))); - } - - ctgWarn("db removed from cache, db:%s", dbName); - goto _return; - } - - if (NULL == pCatalog->dbCache.cache) { + if (NULL == pCatalog->dbCache) { SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == cache) { ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache.cache, NULL, cache)) { + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { taosHashCleanup(cache); } - } else { - CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo)); } bool newAdded = false; - if (taosHashPutExt(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo), &newAdded) != 0) { - ctgError("taosHashPutExt db vgroup to cache failed, db:%s", dbName); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + SDbVgVersion vgVersion = {.dbId = dbInfo->dbId, .vgVersion = dbInfo->vgVersion}; + + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (NULL == dbCache->vgInfo) { + newAdded = true; + + dbCache->vgInfo = dbInfo; + } else { + if (dbCache->vgInfo->dbId != dbInfo->dbId) { + ctgMetaRentRemove(&pCatalog->dbRent, dbCache->vgInfo->dbId, ctgDbVgVersionCompare); + newAdded = true; + } else if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { + ctgInfo("db vgVersion is old, db:%s, vgVersion:%d, current:%d", dbName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + + goto _return; + } + + if (dbCache->vgInfo->vgHash) { + ctgInfo("cleanup db vgHash, db:%s", dbName); + taosHashCleanup(dbCache->vgInfo->vgHash); + dbCache->vgInfo->vgHash = NULL; + } + + tfree(dbCache->vgInfo); + dbCache->vgInfo = dbInfo; + } + + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + } else { + SCtgDBCache newDBCache = {0}; + newDBCache.vgInfo = dbInfo; + + if (taosHashPut(pCatalog->dbCache, dbName, strlen(dbName), &newDBCache, sizeof(newDBCache)) != 0) { + ctgError("taosHashPut db & db vgroup to cache failed, db:%s", dbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + newAdded = true; } - dbInfo->vgInfo = NULL; + dbInfo = NULL; - SDbVgVersion vgVersion = {.dbId = dbInfo->dbId, .vgVersion = dbInfo->vgVersion}; + strncpy(vgVersion.dbFName, dbName, sizeof(vgVersion.dbFName)); + if (newAdded) { - CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, dbInfo->dbId, sizeof(SDbVgVersion))); + CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion))); } else { - CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, dbInfo->dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); + CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); } - ctgDebug("dbName:%s vgroup updated, vgVersion:%d", dbName, dbInfo->vgVersion); + ctgDebug("dbName:%s vgroup updated, vgVersion:%d", dbName, vgVersion.vgVersion); _return: - if (dbInfo && dbInfo->vgInfo) { - taosHashCleanup(dbInfo->vgInfo); - dbInfo->vgInfo = NULL; + if (dbInfo) { + taosHashCleanup(dbInfo->vgHash); + dbInfo->vgHash = NULL; + tfree(dbInfo); } CTG_RET(code); } + +int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId) { + int32_t code = 0; + bool removed = false; + + if (NULL == pCatalog || NULL == dbName) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + if (NULL == pCatalog->dbCache) { + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgValidateAndRemoveDb(pCatalog, dbName, dbId, &removed)); + if (!removed) { + return TSDB_CODE_SUCCESS; + } + + ctgInfo("db removed from cache, db:%s, uid:%"PRIx64, dbName, dbId); + + CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbId, ctgDbVgVersionCompare)); + + ctgDebug("db removed from rent, db:%s, uid:%"PRIx64, dbName, dbId); + + CTG_RET(code); +} + +int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid) { + int32_t code = 0; + bool removed = false; + + if (NULL == pCatalog || NULL == dbName || NULL == stbName) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + if (NULL == pCatalog->dbCache) { + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgValidateAndRemoveStbMeta(pCatalog, dbName, stbName, suid, &removed)); + if (!removed) { + return TSDB_CODE_SUCCESS; + } + + ctgInfo("stb removed from cache, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + + CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->stbRent, suid, ctgSTableVersionCompare)); + + ctgDebug("stb removed from rent, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + + CTG_RET(code); +} + + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { return ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1); } @@ -1217,7 +1484,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; - SDBVgroupInfo* dbVgroup = NULL; + SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; *pVgroupList = NULL; @@ -1226,7 +1493,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbVgroup)); + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbCache)); // REMOEV THIS .... if (0 == tbMeta->vgId) { @@ -1239,10 +1506,10 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S // REMOVE THIS .... if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbVgroup, pVgroupList)); + CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbCache->vgInfo, pVgroupList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(dbVgroup->vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { + if (NULL == taosHashGetClone(dbCache->vgInfo->vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1265,9 +1532,9 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S _return: tfree(tbMeta); - if (dbVgroup) { - CTG_UNLOCK(CTG_READ, &dbVgroup->lock); - taosHashRelease(pCatalog->dbCache.cache, dbVgroup); + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } if (vgList) { @@ -1280,20 +1547,21 @@ _return: int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { - SDBVgroupInfo* dbInfo = NULL; + SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_RET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbInfo)); + CTG_ERR_RET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbCache)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbCache->vgInfo, pTableName, pVgroup)); _return: - if (dbInfo) { - CTG_UNLOCK(CTG_READ, &dbInfo->lock); - taosHashRelease(pCatalog->dbCache.cache, dbInfo); + + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } CTG_RET(code); @@ -1337,6 +1605,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const S return TSDB_CODE_SUCCESS; _return: + if (pRsp->pTableMeta) { int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); for (int32_t i = 0; i < aSize; ++i) { @@ -1366,7 +1635,7 @@ int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion * CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_RET(ctgMetaRentGet(&pCatalog->stableRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + CTG_RET(ctgMetaRentGet(&pCatalog->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index a01c3bcf5d..751fa72347 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -59,7 +59,7 @@ int32_t ctgTestTagNum = 1; int32_t ctgTestSVersion = 1; int32_t ctgTestTVersion = 1; int32_t ctgTestSuid = 2; -int64_t ctgTestDbId = 33; +uint64_t ctgTestDbId = 33; uint64_t ctgTestClusterId = 0x1; char *ctgTestDbname = "1.db1"; @@ -128,15 +128,14 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(sn.dbname, "db1"); strcpy(sn.tname, ctgTestSTablename); - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&cn, tbFullName); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&cn, db); + strcpy(output->dbFName, db); SET_META_TYPE_BOTH_TABLE(output->metaType); - strcpy(output->ctbFname, tbFullName); - - tNameExtractFullName(&cn, tbFullName); - strcpy(output->tbFname, tbFullName); + strcpy(output->ctbName, cn.tname); + strcpy(output->tbName, sn.tname); output->ctbMeta.vgId = 9; output->ctbMeta.tableType = TSDB_CHILD_TABLE; @@ -175,10 +174,11 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(s->name, "tag1s"); } -void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { +void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; + SDBVgroupInfo *dbVgroup = (SDBVgroupInfo *)calloc(1, sizeof(SDBVgroupInfo)); dbVgroup->vgVersion = vgVersion++; @@ -186,7 +186,7 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { dbVgroup->hashMethod = 0; dbVgroup->dbId = ctgTestDbId; - dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + dbVgroup->vgHash = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion); uint32_t hashUnit = UINT32_MAX / vgNum; @@ -195,16 +195,18 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { vgInfo.vgId = i + 1; vgInfo.hashBegin = i * hashUnit; vgInfo.hashEnd = hashUnit * (i + 1) - 1; - vgInfo.numOfEps = i % TSDB_MAX_REPLICA + 1; - vgInfo.inUse = i % vgInfo.numOfEps; - for (int32_t n = 0; n < vgInfo.numOfEps; ++n) { - SEpAddr *addr = &vgInfo.epAddr[n]; + vgInfo.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; + vgInfo.epset.inUse = i % vgInfo.epset.numOfEps; + for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) { + SEp *addr = &vgInfo.epset.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = htons(n + 22); } - taosHashPut(dbVgroup->vgInfo, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); + taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); } + + *pdbVgroup = dbVgroup; } void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { @@ -229,10 +231,10 @@ void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM vg->vgId = htonl(i + 1); vg->hashBegin = htonl(i * hashUnit); vg->hashEnd = htonl(hashUnit * (i + 1) - 1); - vg->numOfEps = i % TSDB_MAX_REPLICA + 1; - vg->inUse = i % vg->numOfEps; - for (int32_t n = 0; n < vg->numOfEps; ++n) { - SEpAddr *addr = &vg->epAddr[n]; + vg->epset.numOfEps = i % TSDB_MAX_REPLICA + 1; + vg->epset.inUse = i % vg->epset.numOfEps; + for (int32_t n = 0; n < vg->epset.numOfEps; ++n) { + SEp *addr = &vg->epset.eps[n]; strcpy(addr->fqdn, "a0"); addr->port = htons(n + 22); } @@ -250,7 +252,8 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestTablename); rspMsg->numOfTags = 0; rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -285,8 +288,9 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestCTablename); - sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestCTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -327,8 +331,9 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); - sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestSTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -370,8 +375,9 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); - sprintf(rspMsg->stbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); + strcpy(rspMsg->dbFName, ctgTestDbname); + sprintf(rspMsg->tbName, "%s_%d", ctgTestSTablename, idx); + sprintf(rspMsg->stbName, "%s_%d", ctgTestSTablename, idx); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -589,12 +595,12 @@ void *ctgTestGetDbVgroupThread(void *param) { void *ctgTestSetDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; + SDBVgroupInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, dbVgroup); if (code) { assert(0); } @@ -669,6 +675,7 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } + TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -693,7 +700,7 @@ TEST(tableMeta, normalTable) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.numOfEps, 3); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); ctgTestSetPrepareTableMeta(); @@ -741,7 +748,7 @@ TEST(tableMeta, normalTable) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -837,7 +844,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -938,7 +945,8 @@ TEST(tableMeta, superTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + free(stb); stb = NULL; } else { @@ -983,7 +991,7 @@ TEST(tableDistVgroup, normalTable) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 8); - ASSERT_EQ(vgInfo->numOfEps, 3); + ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); } @@ -1015,7 +1023,7 @@ TEST(tableDistVgroup, childTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 9); - ASSERT_EQ(vgInfo->numOfEps, 4); + ASSERT_EQ(vgInfo->epset.numOfEps, 4); catalogDestroy(); } @@ -1046,13 +1054,13 @@ TEST(tableDistVgroup, superTableCase) { ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(vgInfo->vgId, 1); - ASSERT_EQ(vgInfo->numOfEps, 1); + ASSERT_EQ(vgInfo->epset.numOfEps, 1); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1); ASSERT_EQ(vgInfo->vgId, 2); - ASSERT_EQ(vgInfo->numOfEps, 2); + ASSERT_EQ(vgInfo->epset.numOfEps, 2); vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2); ASSERT_EQ(vgInfo->vgId, 3); - ASSERT_EQ(vgInfo->numOfEps, 3); + ASSERT_EQ(vgInfo->epset.numOfEps, 3); catalogDestroy(); } @@ -1062,9 +1070,11 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgroupInfo *dbVgroup = NULL; SArray *vgList = NULL; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndNormalMeta(); initQueryModuleMsgHandle(); @@ -1088,31 +1098,31 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 8); - ASSERT_EQ(vgInfo.numOfEps, 3); + ASSERT_EQ(vgInfo.epset.numOfEps, 3); code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->numOfEps, 3); + ASSERT_EQ(pvgInfo->epset.numOfEps, 3); taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, dbVgroup); ASSERT_EQ(code, 0); code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); ASSERT_EQ(code, 0); ASSERT_EQ(vgInfo.vgId, 7); - ASSERT_EQ(vgInfo.numOfEps, 2); + ASSERT_EQ(vgInfo.epset.numOfEps, 2); code = catalogGetTableDistVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1); pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0); ASSERT_EQ(pvgInfo->vgId, 8); - ASSERT_EQ(pvgInfo->numOfEps, 3); + ASSERT_EQ(pvgInfo->epset.numOfEps, 3); taosArrayDestroy(vgList); catalogDestroy(); @@ -1169,6 +1179,7 @@ TEST(multiThread, getSetDbVgroupCase) { catalogDestroy(); } + TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1178,6 +1189,8 @@ TEST(multiThread, ctableMeta) { SArray *vgList = NULL; ctgTestStop = false; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -1212,11 +1225,13 @@ TEST(multiThread, ctableMeta) { } ctgTestStop = true; - sleep(1); + sleep(2); catalogDestroy(); } + + TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1229,6 +1244,8 @@ TEST(rentTest, allRent) { SSTableMetaVersion *stable = NULL; uint32_t num = 0; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); initQueryModuleMsgHandle(); @@ -1273,7 +1290,7 @@ TEST(rentTest, allRent) { printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%" PRId64 ", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; @@ -1291,4 +1308,4 @@ int main(int argc, char **argv) { return RUN_ALL_TESTS(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 702581edb9..130e46fc4c 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -38,7 +38,7 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.idstr) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index f36071f46c..41ed1739b9 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -224,12 +224,12 @@ typedef struct STaskAttr { // SFilterInfo *pFilters; void* tsdb; -// SMemRef memRef; STableGroupInfo tableGroupInfo; // table list SArray int32_t vgId; SArray *pUdfInfo; // no need to free } STaskAttr; +typedef int32_t (*__optr_prepare_fn_t)(void* param); typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); @@ -239,8 +239,7 @@ typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id uint64_t subplanId; uint64_t templateId; - uint64_t taskId; // this is a subplan id - char *idstr; + char *str; } STaskIdInfo; typedef struct SExecTaskInfo { @@ -314,8 +313,9 @@ typedef struct SOperatorInfo { struct SOperatorInfo **pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + __optr_prepare_fn_t prepareFn; __operator_fn_t exec; - __optr_cleanup_fn_t cleanup; + __optr_cleanup_fn_t cleanupFn; } SOperatorInfo; enum { @@ -396,7 +396,7 @@ typedef struct STableScanInfo { int32_t current; int32_t reverseTimes; // 0 by default - SQLFunctionCtx *pCtx; // next operator query context + SqlFunctionCtx *pCtx; // next operator query context SResultRowInfo *pResultRowInfo; int32_t *rowCellInfoOffset; SExprInfo *pExpr; @@ -426,7 +426,7 @@ typedef struct SStreamBlockScanInfo { typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; int32_t *rowCellInfoOffset; // offset value for each row result cell info - SQLFunctionCtx *pCtx; + SqlFunctionCtx *pCtx; SSDataBlock *pRes; } SOptrBasicInfo; @@ -565,7 +565,6 @@ typedef struct SOrderOperatorInfo { SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); @@ -608,11 +607,11 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); +void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); void freeParam(STaskParam *param); int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, @@ -639,7 +638,6 @@ int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); bool isTaskKilled(SExecTaskInfo *pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); -bool doBuildResCheck(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); bool onlyQueryTags(STaskAttr* pQueryAttr); @@ -661,8 +659,8 @@ void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId); +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 97a0557748..3623f5947d 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -186,6 +186,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; + *pQueryEnd = pDispatcher->queryEnd; } static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a2eb5dc339..c47c83ba29 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -18,20 +18,20 @@ #include "executorimpl.h" #include "planner.h" -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, uint64_t reqId) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != OP_StreamScan) { if (pOperator->numOfDownstream == 0) { - qError("failed to find stream scan operator to set the input data block, reqId:0x%" PRIx64, reqId); + qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query - qError("join not supported for stream block scan, reqId:0x%" PRIx64, reqId); + qError("join not supported for stream block scan, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - return doSetStreamBlock(pOperator->pDownstream[0], input, reqId); + return doSetStreamBlock(pOperator->pDownstream[0], input, id); } else { SStreamBlockScanInfo* pInfo = pOperator->info; tqReadHandleSetMsg(pInfo->readerHandle, input, 0); @@ -39,7 +39,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, uint64_t } } -int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input) { if (tinfo == NULL) { return TSDB_CODE_QRY_APP_ERROR; } @@ -50,7 +50,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, pTaskInfo->id.queryId); + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*) input, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 8635b2d5c8..f632a84c1f 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -51,25 +51,7 @@ static void freeqinfoFn(void *qhandle) { qDestroyTask(*handle); } -void freeParam(STaskParam *param) { - tfree(param->sql); - tfree(param->tagCond); - tfree(param->tbnameCond); - tfree(param->pTableIdList); - taosArrayDestroy(param->pOperator); - tfree(param->pExprs); - tfree(param->pSecExprs); - - tfree(param->pExpr); - tfree(param->pSecExpr); - - tfree(param->pGroupColIndex); - tfree(param->pTagColumnInfo); - tfree(param->pGroupbyExpr); - tfree(param->prevResult); -} - -int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { assert(readHandle != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 16487fe747..2d3085c82b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -173,13 +173,13 @@ static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, +static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset); -void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx); -static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); +static void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); static bool hasMainOutput(STaskAttr *pQueryAttr); @@ -219,14 +219,14 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); -static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); +static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); -static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); -static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); static void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, - SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); + SqlFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); SArray* getOrderCheckColumns(STaskAttr* pQuery); @@ -354,7 +354,7 @@ void* destroyOutputBuf(SSDataBlock* pBlock) { return NULL; } -static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static bool isSelectivityWithTagsQuery(SqlFunctionCtx *pCtx, int32_t numOfOutput) { return true; // bool hasTags = false; // int32_t numOfSelectivity = 0; @@ -754,14 +754,14 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf } static bool chkWindowOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + bool masterscan, SResultRow **pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -940,7 +940,7 @@ static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlock return num; } -static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, +static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; bool hasAggregates = pCtx[0].isAggSet; @@ -1073,7 +1073,7 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr *pQueryAttr, STimeWindow *p return ekey; } -static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { +static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { if (type == RESULT_ROW_START_INTERP) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].start.key = INT64_MIN; @@ -1112,9 +1112,9 @@ static TSKEY getStartTsKey(STaskAttr* pQueryAttr, STimeWindow* win, const TSKEY* return ts; } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1124,7 +1124,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { // if (pCtx[0].functionId == FUNCTION_ARITHM) { // SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz; // if (pSupport->colList == NULL) { @@ -1141,7 +1141,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo // } } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1205,7 +1205,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, } } -static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { +static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { if (functionNeedToExecute(NULL, &pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct @@ -1214,7 +1214,7 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction } } -static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx, int32_t numOfOutput) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -1242,7 +1242,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; SExprInfo* pExpr = pOperator->pExpr; - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { int32_t functionId = pCtx[k].functionId; @@ -1302,7 +1302,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } } -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1333,7 +1333,7 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLF return true; } -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { STaskRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1365,7 +1365,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFun return true; } -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1775,7 +1775,7 @@ static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicI int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; - SQLFunctionCtx *pCtx = binfo->pCtx; + SqlFunctionCtx *pCtx = binfo->pCtx; // not assign result buffer yet, add new result buffer, TODO remove it char* d = pData; @@ -1824,7 +1824,7 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // in case of timestamp column, always generated results. @@ -1855,7 +1855,7 @@ static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx * return true; } -void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { +void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { SColumnDataAgg *pAgg = NULL; if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColumn->flag)) { @@ -1879,7 +1879,7 @@ void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* } // set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { return TSDB_CODE_SUCCESS; } @@ -1887,8 +1887,8 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { int32_t num = 0; int16_t tagLen = 0; - SQLFunctionCtx* p = NULL; - SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + SqlFunctionCtx* p = NULL; + SqlFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); if (pTagCtx == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -1920,11 +1920,11 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { return TSDB_CODE_SUCCESS; } -static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } @@ -1937,7 +1937,7 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI for (int32_t i = 0; i < numOfOutput; ++i) { SSqlExpr *pSqlExpr = &pExpr[i].base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; + SqlFunctionCtx* pCtx = &pFuncCtx[i]; #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2024,10 +2024,10 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } -static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { +static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { size_t numOfOutput = taosArrayGetSize(pExprInfo); - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } @@ -2042,7 +2042,7 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); SSqlExpr *pSqlExpr = &pExpr->base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; + SqlFunctionCtx* pCtx = &pFuncCtx[i]; #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2128,7 +2128,7 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC return pFuncCtx; } -static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } @@ -2371,7 +2371,7 @@ _clean: static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); +// tsdbCleanupReadHandle(pRuntimeEnv->pTsdbReadHandle); pRuntimeEnv->pTsdbReadHandle = NULL; // SMemRef* pMemRef = &pQueryAttr->memRef; @@ -2773,7 +2773,7 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SqlFunctionCtx *pCtx, int32_t numOfRows) { // STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // // if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { @@ -3053,7 +3053,7 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes); static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { - SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; + SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; uint32_t status = BLK_DATA_NO_NEEDED; int32_t numOfOutput = pTableScanInfo->numOfOutput; @@ -3334,7 +3334,7 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } /* - * set tag value in SQLFunctionCtx + * set tag value in SqlFunctionCtx * e.g.,tag information into input buffer */ static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes) { @@ -3375,7 +3375,7 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num return NULL; } -void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; SExprInfo *pExpr = pOperatorInfo->pExpr; @@ -3509,7 +3509,7 @@ static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { } -void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); } @@ -3531,7 +3531,7 @@ int32_t initResultRow(SResultRow *pResultRow) { * offset[0] offset[1] offset[2] */ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; @@ -3568,7 +3568,7 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t stage, SExecTaskInfo* pTaskInfo) { SOptrBasicInfo *pInfo = &pAggInfo->binfo; - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; @@ -3637,7 +3637,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf } } -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) { bool needCopyTs = false; int32_t tsNum = 0; char *src = NULL; @@ -3680,7 +3680,7 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { } } -void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { +void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (isRowEntryInitialized(pResInfo)) { @@ -3701,7 +3701,7 @@ void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) { } } -static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { // if (pRuntimeEnv->pTsBuf) { // SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); // bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); @@ -3722,7 +3722,7 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt pTableScanInfo->reverseTimes = 0; } -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { +void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -3832,7 +3832,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3865,7 +3865,7 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pRes } } -void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, +void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SqlFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; @@ -3905,7 +3905,7 @@ void setExecutionContext(STaskRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, in pRuntimeEnv->prevGroupId = tableGroupId; } -void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3928,7 +3928,7 @@ void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFu } } -void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { +void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SSqlExpr* pExpr = &pExprInfo->base; @@ -3996,7 +3996,7 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S } // TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { +void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { #if 0 STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -4031,7 +4031,7 @@ void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, #endif } -void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { +void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 int32_t numOfExprs = pQueryAttr->numOfOutput; @@ -4199,7 +4199,7 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv* pRuntim } } -static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, +static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -4953,6 +4953,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->pTsdbReadHandle == NULL) { return NULL; } @@ -5070,6 +5071,8 @@ static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { SStreamBlockScanInfo* pInfo = pOperator->info; SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + pBlockInfo->rows = 0; + while (tqNextDataBlock(pInfo->readerHandle)) { pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { @@ -5115,7 +5118,7 @@ static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { tfree(pMsgBody); } -void processRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { +void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SMsgSendInfo *pSendInfo = (SMsgSendInfo *) pMsg->ahandle; assert(pMsg->ahandle != NULL); @@ -5146,7 +5149,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); if (pExchangeInfo->current >= totalSources) { - qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", pTaskInfo->id.idstr, totalSources, + qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); return NULL; } @@ -5163,14 +5166,9 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); - SEpSet epSet = {0}; - epSet.numOfEps = pSource->addr.numOfEps; - epSet.port[0] = pSource->addr.epAddr[0].port; - tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); - int64_t startTs = taosGetTimestampUs(); qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources); + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, pExchangeInfo->current, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); pMsg->sId = htobe64(pSource->schedId); @@ -5192,7 +5190,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { pMsgSendInfo->fp = loadRemoteDataCallback; int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &epSet, &transporterId, pMsgSendInfo); + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo); tsem_wait(&pExchangeInfo->ready); SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp; @@ -5208,7 +5206,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { int64_t el = taosGetTimestampUs() - startTs; pExchangeInfo->totalElapsed += el; - qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", pTaskInfo->id.idstr, totalSources, + qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); return NULL; } else { @@ -5296,13 +5294,14 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pOperator->exec = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; +#if 1 { // todo refactor SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; rpcInit.label = "EX"; rpcInit.numOfThreads = 1; - rpcInit.cfp = processRspMsg; + rpcInit.cfp = qProcessFetchRsp; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.user = (char *)"root"; @@ -5316,7 +5315,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* return NULL; // todo } } - +#endif return pOperator; } @@ -5345,37 +5344,6 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { return pResBlock; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { - assert(repeatTime > 0 && numOfOutput > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pInfo->pTsdbReadHandle = pTsdbReadHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - - pOperator->name = "TableScanOperator"; - pOperator->operatorType = OP_TableScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->exec = doTableScan; - pOperator->pTaskInfo = pTaskInfo; - - return pOperator; -} - SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); @@ -5450,14 +5418,13 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; +// pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; pOperator->exec = doBlockInfoScan; return pOperator; } -SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, uint64_t uid, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5467,6 +5434,9 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp return NULL; } + // todo dynamic set the value of 4096 + pInfo->pRes = createOutputBuf_rv(pExprInfo, 4096); + int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprInfo); SArray* pColList = taosArrayInit(numOfOutput, sizeof(int32_t)); for(int32_t i = 0; i < numOfOutput; ++i) { @@ -5477,7 +5447,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); - tqReadHandleSetTbUid(streamReadHandle, uid); + tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); pInfo->readerHandle = streamReadHandle; @@ -5701,9 +5671,8 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - // pOperator->exec = doGlobalAggregate; - pOperator->cleanup = destroyGlobalAggOperatorInfo; + pOperator->cleanupFn = destroyGlobalAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -5746,7 +5715,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; // pOperator->exec = doMultiwayMergeSort; - pOperator->cleanup = destroyGlobalAggOperatorInfo; + pOperator->cleanupFn = destroyGlobalAggOperatorInfo; return pOperator; } @@ -5850,7 +5819,7 @@ SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->exec = doSort; - pOperator->cleanup = destroyOrderOperatorInfo; + pOperator->cleanupFn = destroyOrderOperatorInfo; pOperator->pRuntimeEnv = pRuntimeEnv; appendDownstream(pOperator, downstream); @@ -5897,7 +5866,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { finalizeQueryResult(pOperator, pInfo->pCtx, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); pInfo->pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput); - return pInfo->pRes; + return (pInfo->pRes->info.rows != 0)? pInfo->pRes:NULL; } static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { @@ -6759,8 +6728,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { return; } - if (pOperator->cleanup != NULL) { - pOperator->cleanup(pOperator->info, pOperator->numOfOutput); + if (pOperator->cleanupFn != NULL) { + pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput); } if (pOperator->pDownstream != NULL) { @@ -6814,7 +6783,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; - pOperator->cleanup = destroyAggOperatorInfo; + pOperator->cleanupFn = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6909,7 +6878,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOp pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableAggregate; - pOperator->cleanup = destroyAggOperatorInfo; + pOperator->cleanupFn = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6939,7 +6908,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doProjectOperation; - pOperator->cleanup = destroyProjectOperatorInfo; + pOperator->cleanupFn = destroyProjectOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6997,7 +6966,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->exec = doFilter; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyConditionOperatorInfo; + pOperator->cleanupFn = destroyConditionOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7039,7 +7008,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7064,7 +7033,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAllIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7088,7 +7057,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doStateWindowAgg; - pOperator->cleanup = destroyStateWindowOperatorInfo; + pOperator->cleanupFn = destroyStateWindowOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7113,7 +7082,7 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSessionWindowAgg; - pOperator->cleanup = destroySWindowOperatorInfo; + pOperator->cleanupFn = destroySWindowOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7137,7 +7106,7 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7161,7 +7130,7 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAllSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); @@ -7194,7 +7163,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = hashGroupbyAggregate; - pOperator->cleanup = destroyGroupbyOperatorInfo; + pOperator->cleanupFn = destroyGroupbyOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7233,7 +7202,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doFill; - pOperator->cleanup = destroySFillOperatorInfo; + pOperator->cleanupFn = destroySFillOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7281,7 +7250,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI // pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroySlimitOperatorInfo; + pOperator->cleanupFn = destroySlimitOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7431,7 +7400,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SeqTableTagScan"; -// pOperator->operatorType = OP_TagScan; + pOperator->operatorType = OP_TagScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -7439,7 +7408,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyTagScanOperatorInfo; + pOperator->cleanupFn = destroyTagScanOperatorInfo; return pOperator; } @@ -7579,7 +7548,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = hashDistinct; pOperator->pExpr = pExpr; - pOperator->cleanup = destroyDistinctOperatorInfo; + pOperator->cleanupFn = destroyDistinctOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7741,40 +7710,51 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; - pTaskInfo->id.taskId = taskId; char* p = calloc(1, 128); snprintf(p, 128, "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, queryId); - pTaskInfo->id.idstr = strdup(p); + pTaskInfo->id.str = strdup(p); return pTaskInfo; } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId); +static tsdbReaderT doCreateDataReader(STableScanPhyNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId); +static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); -SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { - if (pPhyNode->info.type == OP_TableScan) { + if (pPhyNode->info.type == OP_DataBlocksOptScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + tsdbReaderT pDataReader = doCreateDataReader((STableScanPhyNode*) pPhyNode, pHandle, (uint64_t) queryId, taskId); - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); - - return createTableScanOperatorInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); - } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); - - return createDataBlocksOptScanInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + return createDataBlocksOptScanInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (pPhyNode->info.type == OP_Exchange) { SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); } else if (pPhyNode->info.type == OP_StreamScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. - return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pScanPhyNode->uid, pTaskInfo); + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. + STableGroupInfo groupInfo = {0}; + + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + + SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); + ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); + + // Transfer the Array of STableKeyInfo into uid list. + size_t numOfTables = taosArrayGetSize(pa); + SArray* idList = taosArrayInit(numOfTables, sizeof(uint64_t)); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pa, i); + taosArrayPush(idList, &pkeyInfo->uid); + } + + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pPhyNode->pTargets, idList, pTaskInfo); + taosArrayDestroy(idList); + + //TODO destroy groupInfo + return pOperator; } } @@ -7784,18 +7764,23 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask for (int32_t i = 0; i < size; ++i) { SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i); - SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId, taskId); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId); return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo); } } } -static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { +static tsdbReaderT createDataReaderImpl(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readHandle, uint64_t queryId, uint64_t taskId) { STsdbQueryCond cond = {.loadExternalRows = false}; cond.order = pTableScanNode->scan.order; cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets); cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + if (cond.colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + cond.twindow = pTableScanNode->window; cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; @@ -7809,32 +7794,27 @@ static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, S cond.colList[i].colId = pSchema->colId; } - return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, taskId); + return tsdbQueryTables(readHandle, &cond, pGroupInfo, queryId, taskId); } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId) { - int32_t code = 0; +static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { + int32_t code = 0; + if (tableType == TSDB_SUPER_TABLE) { + code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); + } else { // Create one table group. + code = tsdbGetOneTableGroup(metaHandle, tableUid, 0, pGroupInfo); + } + + return code; +} + +static tsdbReaderT doCreateDataReader(STableScanPhyNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { STableGroupInfo groupInfo = {0}; uint64_t uid = pTableScanNode->scan.uid; - STimeWindow window = pTableScanNode->window; - int32_t tableType = pTableScanNode->scan.tableType; - - if (tableType == TSDB_SUPER_TABLE) { - code = - tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } else { // Create one table group. - groupInfo.numOfTables = 1; - groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES); - - SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - - STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; - taosArrayPush(pa, &info); - taosArrayPush(groupInfo.pGroupList, &pa); + int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId); + if (code != TSDB_CODE_SUCCESS) { + goto _error; } if (groupInfo.numOfTables == 0) { @@ -7843,14 +7823,14 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, goto _error; } - return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId, taskId); + return createDataReaderImpl(pTableScanNode, &groupInfo, pHandle->reader, queryId, taskId); _error: terrno = code; return NULL; } -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId) { +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; @@ -7860,7 +7840,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* goto _complete; } - (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId, taskId); + (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId); if ((*pTaskInfo)->pRoot == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; @@ -8827,35 +8807,17 @@ void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { } void doDestroyTask(SExecTaskInfo *pTaskInfo) { - qDebug("%s start to free execTask", GET_TASKID(pTaskInfo)); + qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); + doDestroyTableQueryInfo(&pTaskInfo->tableqinfoGroupInfo); // taosArrayDestroy(pTaskInfo->summary.queryProfEvents); // taosHashCleanup(pTaskInfo->summary.operatorProfResults); - qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); + tfree(pTaskInfo->sql); + tfree(pTaskInfo->id.str); tfree(pTaskInfo); } -bool doBuildResCheck(SQInfo* pQInfo) { - bool buildRes = false; - - pthread_mutex_lock(&pQInfo->lock); - - pQInfo->dataReady = QUERY_RESULT_READY; - buildRes = needBuildResAfterQueryComplete(pQInfo); - - // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is - // put into task to be executed. - assert(pQInfo->owner == taosGetSelfPthreadId()); - pQInfo->owner = 0; - - pthread_mutex_unlock(&pQInfo->lock); - - // used in retrieve blocking model. - tsem_post(&pQInfo->ready); - return buildRes; -} - static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { if (val == NULL) { setNull(output, type, bytes); diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 1814e1724b..8381a7c585 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -219,7 +219,7 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - int32_t code = qCreateExecTask((void*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); + int32_t code = qCreateExecTask((SReadHandle*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index 8de66584c6..8b83dbe8cd 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -73,7 +73,7 @@ typedef struct STwaInfo { extern int32_t functionCompatList[]; // compatible check array list -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval); +bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** * the numOfRes should be kept, since it may be used later diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index ff2fb95328..694a1c2ca5 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -59,7 +59,7 @@ #define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ __ctx->tag.i = (ts); \ __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ @@ -71,14 +71,14 @@ #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ aggFunc[FUNCTION_TAG].addInput(__ctx); \ } \ } while (0); -void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} +void noop1(SqlFunctionCtx *UNUSED_PARAM(pCtx)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } +void doFinalizer(SqlFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } typedef struct tValuePair { SVariant v; @@ -200,7 +200,7 @@ void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { pCell->initialized = false; } -int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num) { int32_t maxOutput = 0; for (int32_t j = 0; j < num; ++j) { int32_t id = pCtx[j].functionId; @@ -223,7 +223,7 @@ int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { return maxOutput; } -void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num) { +void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num) { for (int32_t j = 0; j < num; ++j) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); pResInfo->numOfRes = 0; @@ -473,7 +473,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { return false; } @@ -490,7 +490,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf * * @param pCtx */ -static void function_finalizer(SQLFunctionCtx *pCtx) { +static void function_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); @@ -503,7 +503,7 @@ static void function_finalizer(SQLFunctionCtx *pCtx) { * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ -static void count_function(SQLFunctionCtx *pCtx) { +static void count_function(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; /* @@ -537,7 +537,7 @@ static void count_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void count_func_merge(SQLFunctionCtx *pCtx) { +static void count_func_merge(SqlFunctionCtx *pCtx) { int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->pOutput) += pData[i]; @@ -555,7 +555,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { * @param filterCols * @return */ -int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +int32_t countRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { return BLK_DATA_NO_NEEDED; } else { @@ -563,7 +563,7 @@ int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { } } -int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +int32_t noDataRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_NO_NEEDED; } #define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ @@ -635,7 +635,7 @@ int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ } while (0) -static void do_sum(SQLFunctionCtx *pCtx) { +static void do_sum(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; // Only the pre-computing information loaded and actual data does not loaded @@ -698,7 +698,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { } } -static void sum_function(SQLFunctionCtx *pCtx) { +static void sum_function(SqlFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer @@ -710,7 +710,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { } } -static void sum_func_merge(SQLFunctionCtx *pCtx) { +static void sum_func_merge(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); @@ -742,16 +742,16 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { } } -static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t statisRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_STATIS_NEEDED; } -static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t dataBlockRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_ALL_NEEDED; } // todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -764,7 +764,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t c } } -static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i) { return BLK_DATA_NO_NEEDED; } @@ -776,7 +776,7 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t co } } -static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -796,7 +796,7 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32 } } -static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i) { return BLK_DATA_NO_NEEDED; } @@ -822,7 +822,7 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_ * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate * result into output buffer. */ -static void avg_function(SQLFunctionCtx *pCtx) { +static void avg_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf @@ -885,7 +885,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } } -static void avg_func_merge(SQLFunctionCtx *pCtx) { +static void avg_func_merge(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->pOutput; @@ -907,7 +907,7 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { /* * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ -static void avg_finalizer(SQLFunctionCtx *pCtx) { +static void avg_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == MERGE_STAGE) { @@ -938,7 +938,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { ///////////////////////////////////////////////////////////////////////////////////////////// -static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { +static void minMax_function(SqlFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { // data in current data block are qualified to the query if (pCtx->isAggSet) { *notNullElems = pCtx->size - pCtx->agg.numOfNull; @@ -994,7 +994,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*data < val) ^ isMin) { *data = (int32_t)val; for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; + SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -1089,7 +1089,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } } -static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1135,7 +1135,7 @@ static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf return true; } -static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1184,7 +1184,7 @@ static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf /* * the output result of min/max function is the final output buffer, not the intermediate result buffer */ -static void min_function(SQLFunctionCtx *pCtx) { +static void min_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); @@ -1201,7 +1201,7 @@ static void min_function(SQLFunctionCtx *pCtx) { } } -static void max_function(SQLFunctionCtx *pCtx) { +static void max_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); @@ -1218,7 +1218,7 @@ static void max_function(SQLFunctionCtx *pCtx) { } } -static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { +static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); @@ -1247,7 +1247,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp *(int32_t *)output = v; for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; + SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; aggFunc[FUNCTION_TAG].addInput(__ctx); } @@ -1303,7 +1303,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp return notNullElems; } -static void min_func_merge(SQLFunctionCtx *pCtx) { +static void min_func_merge(SqlFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1314,7 +1314,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { } } -static void max_func_merge(SQLFunctionCtx *pCtx) { +static void max_func_merge(SqlFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1334,7 +1334,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { (r) += TPOW2(((type *)d)[i] - (delta)); \ } -static void stddev_function(SQLFunctionCtx *pCtx) { +static void stddev_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); @@ -1419,7 +1419,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { } } -static void stddev_finalizer(SQLFunctionCtx *pCtx) { +static void stddev_finalizer(SqlFunctionCtx *pCtx) { SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { @@ -1445,7 +1445,7 @@ int32_t tsCompare(const void* p1, const void* p2) { } } -static void stddev_dst_function(SQLFunctionCtx *pCtx) { +static void stddev_dst_function(SqlFunctionCtx *pCtx) { SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // the second stage to calculate standard deviation @@ -1536,7 +1536,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); } -static void stddev_dst_merge(SQLFunctionCtx *pCtx) { +static void stddev_dst_merge(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -1553,7 +1553,7 @@ static void stddev_dst_merge(SQLFunctionCtx *pCtx) { } } -static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { +static void stddev_dst_finalizer(SqlFunctionCtx *pCtx) { SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { @@ -1568,7 +1568,7 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -1581,7 +1581,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* } // todo opt for null block -static void first_function(SQLFunctionCtx *pCtx) { +static void first_function(SqlFunctionCtx *pCtx) { if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1612,7 +1612,7 @@ static void first_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); @@ -1630,7 +1630,7 @@ static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t in * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) * to decide if the value is earlier than current intermediate result */ -static void first_dist_function(SQLFunctionCtx *pCtx) { +static void first_dist_function(SqlFunctionCtx *pCtx) { /* * do not to check data in the following cases: * 1. data block that are not loaded @@ -1661,7 +1661,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_dist_func_merge(SQLFunctionCtx *pCtx) { +static void first_dist_func_merge(SqlFunctionCtx *pCtx) { assert(pCtx->stableQuery); char * pData = GET_INPUT_DATA_LIST(pCtx); @@ -1691,7 +1691,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at * least one data in this block that is not null.(TODO opt for this case) */ -static void last_function(SQLFunctionCtx *pCtx) { +static void last_function(SqlFunctionCtx *pCtx) { if (pCtx->order != pCtx->param[0].i) { return; } @@ -1743,7 +1743,7 @@ static void last_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); @@ -1761,7 +1761,7 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind } } -static void last_dist_function(SQLFunctionCtx *pCtx) { +static void last_dist_function(SqlFunctionCtx *pCtx) { /* * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data @@ -1796,7 +1796,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { +static void last_dist_func_merge(SqlFunctionCtx *pCtx) { char *pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); @@ -1824,7 +1824,7 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { /* * NOTE: last_row does not use the interResultBuf to keep the result */ -static void last_row_function(SQLFunctionCtx *pCtx) { +static void last_row_function(SqlFunctionCtx *pCtx) { assert(pCtx->size >= 1); char *pData = GET_INPUT_DATA_LIST(pCtx); @@ -1849,7 +1849,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); } -static void last_row_finalizer(SQLFunctionCtx *pCtx) { +static void last_row_finalizer(SqlFunctionCtx *pCtx) { // do nothing at the first stage SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { @@ -1873,7 +1873,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { - SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; + SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; if (ctx->functionId == FUNCTION_TS_DUMMY) { ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; ctx->tag.i = tsKey; @@ -2023,7 +2023,7 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } -static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { +static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -2118,7 +2118,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * * top/bottom use the intermediate result buffer to keep the intermediate result */ -static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { +static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer @@ -2136,7 +2136,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { +static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; tmp += POINTER_BYTES * pCtx->param[0].i; @@ -2150,7 +2150,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { } } -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { +bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; @@ -2206,7 +2206,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha } } -static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool top_bottom_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2216,7 +2216,7 @@ static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* return true; } -static void top_function(SQLFunctionCtx *pCtx) { +static void top_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; STopBotInfo *pRes = getTopBotOutputInfo(pCtx); @@ -2252,7 +2252,7 @@ static void top_function(SQLFunctionCtx *pCtx) { } } -static void top_func_merge(SQLFunctionCtx *pCtx) { +static void top_func_merge(SqlFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data @@ -2275,7 +2275,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { } } -static void bottom_function(SQLFunctionCtx *pCtx) { +static void bottom_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; STopBotInfo *pRes = getTopBotOutputInfo(pCtx); @@ -2309,7 +2309,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } } -static void bottom_func_merge(SQLFunctionCtx *pCtx) { +static void bottom_func_merge(SqlFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data @@ -2332,7 +2332,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { } } -static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { +static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results @@ -2361,7 +2361,7 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { } /////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool percentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2375,7 +2375,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* return true; } -static void percentile_function(SQLFunctionCtx *pCtx) { +static void percentile_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -2460,7 +2460,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void percentile_finalizer(SQLFunctionCtx *pCtx) { +static void percentile_finalizer(SqlFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i : pCtx->param[0].d; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -2484,7 +2484,7 @@ static void buildHistogramInfo(SAPercentileInfo* pInfo) { pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); } -static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { +static SAPercentileInfo *getAPerctInfo(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo* pInfo = NULL; @@ -2498,7 +2498,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { return pInfo; } -static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool apercentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2510,7 +2510,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo return true; } -static void apercentile_function(SQLFunctionCtx *pCtx) { +static void apercentile_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); @@ -2542,7 +2542,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } } -static void apercentile_func_merge(SQLFunctionCtx *pCtx) { +static void apercentile_func_merge(SqlFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); @@ -2572,7 +2572,7 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); } -static void apercentile_finalizer(SQLFunctionCtx *pCtx) { +static void apercentile_finalizer(SqlFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i : pCtx->param[0].d; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); @@ -2608,7 +2608,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2638,7 +2638,7 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInf LEASTSQR_CAL(param, x, y, i, step); \ } -static void leastsquares_function(SQLFunctionCtx *pCtx) { +static void leastsquares_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2724,7 +2724,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { +static void leastsquares_finalizer(SqlFunctionCtx *pCtx) { // no data in query SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2756,12 +2756,12 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static void date_col_output_function(SQLFunctionCtx *pCtx) { +static void date_col_output_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); *(int64_t *)(pCtx->pOutput) = pCtx->startTs; } -static void col_project_function(SQLFunctionCtx *pCtx) { +static void col_project_function(SqlFunctionCtx *pCtx) { // the number of output rows should not affect the final number of rows, so set it to be 0 if (pCtx->numOfParams == 2) { return; @@ -2791,7 +2791,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { * @param pCtx * @return */ -static void tag_project_function(SQLFunctionCtx *pCtx) { +static void tag_project_function(SqlFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); assert(pCtx->inputBytes == pCtx->resDataInfo.bytes); @@ -2814,9 +2814,9 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { * @param pCtx * @return */ -static void copy_function(SQLFunctionCtx *pCtx); +static void copy_function(SqlFunctionCtx *pCtx); -static void tag_function(SQLFunctionCtx *pCtx) { +static void tag_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); if (pCtx->currentStage == MERGE_STAGE) { copy_function(pCtx); @@ -2825,7 +2825,7 @@ static void tag_function(SQLFunctionCtx *pCtx) { } } -static void copy_function(SQLFunctionCtx *pCtx) { +static void copy_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); char *pData = GET_INPUT_DATA_LIST(pCtx); @@ -2836,7 +2836,7 @@ enum { INITIAL_VALUE_NOT_ASSIGNED = 0, }; -static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2846,7 +2846,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResI return false; } -static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2861,7 +2861,7 @@ static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pRes return false; } -static void deriv_function(SQLFunctionCtx *pCtx) { +static void deriv_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3056,7 +3056,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { } while (0); // TODO difference in date column -static void diff_function(SQLFunctionCtx *pCtx) { +static void diff_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); @@ -3236,7 +3236,7 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } -static void arithmetic_function(SQLFunctionCtx *pCtx) { +static void arithmetic_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; @@ -3264,7 +3264,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3283,7 +3283,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pRe return true; } -static void spread_function(SQLFunctionCtx *pCtx) { +static void spread_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3368,7 +3368,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer */ -void spread_func_merge(SQLFunctionCtx *pCtx) { +void spread_func_merge(SqlFunctionCtx *pCtx) { SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); if (pData->hasResult != DATA_SET_FLAG) { return; @@ -3385,7 +3385,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } -void spread_function_finalizer(SQLFunctionCtx *pCtx) { +void spread_function_finalizer(SqlFunctionCtx *pCtx) { /* * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary @@ -3423,7 +3423,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * param[2]: end time * @param pCtx */ -static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool twa_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3444,7 +3444,7 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { +static int32_t twa_function_impl(SqlFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3685,7 +3685,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si return notNullElems; } -static void twa_function(SQLFunctionCtx *pCtx) { +static void twa_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3719,7 +3719,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { * by next input data. The TWA function only applies to each table, so no merge procedure * is required, we simply copy to the resut ot interResBuffer. */ -void twa_function_copy(SQLFunctionCtx *pCtx) { +void twa_function_copy(SqlFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3727,7 +3727,7 @@ void twa_function_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; } -void twa_function_finalizer(SQLFunctionCtx *pCtx) { +void twa_function_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -3752,7 +3752,7 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { * @param pCtx */ -static void interp_function_impl(SQLFunctionCtx *pCtx) { +static void interp_function_impl(SqlFunctionCtx *pCtx) { int32_t type = (int32_t) pCtx->param[2].i; if (type == TSDB_FILL_NONE) { return; @@ -3875,7 +3875,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); } -static void interp_function(SQLFunctionCtx *pCtx) { +static void interp_function(SqlFunctionCtx *pCtx) { // at this point, the value is existed, return directly if (pCtx->size > 0) { bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); @@ -3918,7 +3918,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } -static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool ts_comp_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; // not initialized since it has been initialized } @@ -3929,7 +3929,7 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pR return true; } -static void ts_comp_function(SQLFunctionCtx *pCtx) { +static void ts_comp_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; @@ -3949,7 +3949,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void ts_comp_finalize(SQLFunctionCtx *pCtx) { +static void ts_comp_finalize(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -4006,7 +4006,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; } -static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool rate_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -4023,7 +4023,7 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResI return true; } -static void rate_function(SQLFunctionCtx *pCtx) { +static void rate_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; @@ -4076,7 +4076,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { } } -static void rate_func_copy(SQLFunctionCtx *pCtx) { +static void rate_func_copy(SqlFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4084,7 +4084,7 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; } -static void rate_finalizer(SQLFunctionCtx *pCtx) { +static void rate_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4102,7 +4102,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static void irate_function(SQLFunctionCtx *pCtx) { +static void irate_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; @@ -4184,7 +4184,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi } } -static void blockInfo_func(SQLFunctionCtx* pCtx) { +static void blockInfo_func(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -4232,7 +4232,7 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock } } -void block_func_merge(SQLFunctionCtx* pCtx) { +void block_func_merge(SqlFunctionCtx* pCtx) { STableBlockDist info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); @@ -4338,7 +4338,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { UNUSED(sz); } -void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { +void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -4380,12 +4380,12 @@ int32_t functionCompatList[] = { }; //typedef struct SFunctionFpSet { -// bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment -// void (*addInput)(struct SQLFunctionCtx *pCtx); +// bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment +// void (*addInput)(struct SqlFunctionCtx *pCtx); // // // finalizer must be called after all exec has been executed to generated final result. -// void (*finalize)(struct SQLFunctionCtx *pCtx); -// void (*combine)(struct SQLFunctionCtx *pCtx); +// void (*finalize)(struct SqlFunctionCtx *pCtx); +// void (*combine)(struct SqlFunctionCtx *pCtx); //} SFunctionFpSet; SFunctionFpSet fpSet[1] = { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 2771a9ecf8..b65e637a57 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -121,7 +121,7 @@ void destroyUdfInfo(SUdfInfo* pUdfInfo) { tfree(pUdfInfo); } -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type) { int32_t output = 0; if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 7f77876c4c..417c56aba1 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( parser - PRIVATE os util catalog function transport qcom + PRIVATE os util nodes catalog function transport qcom ) if(${BUILD_TEST}) diff --git a/source/libs/parser/inc/astCreateContext.h b/source/libs/parser/inc/astCreateContext.h new file mode 100644 index 0000000000..5458500a82 --- /dev/null +++ b/source/libs/parser/inc/astCreateContext.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_AST_CREATER_H_ +#define _TD_AST_CREATER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "nodes.h" +#include "parser.h" + +typedef struct SAstCreateContext { + SParseContext* pQueryCxt; + bool notSupport; + bool valid; + SNode* pRootNode; + SHashObj* pResourceHash; +} SAstCreateContext; + +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt); +int32_t destroyAstCreateContext(SAstCreateContext* pCxt); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_AST_CREATER_H_*/ diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h new file mode 100644 index 0000000000..15f0792d5c --- /dev/null +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" +#include "nodesShowStmts.h" +#include "astCreateContext.h" +#include "ttoken.h" + +#ifndef _TD_AST_CREATE_FUNCS_H_ +#define _TD_AST_CREATE_FUNCS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +extern SToken nil_token; + +SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableAlias, const SToken* pColumnName); +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull); +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias); +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); +SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal); +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow); +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList); +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving); +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); +SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); +SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); + +SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type); + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_AST_CREATE_FUNCS_H_*/ diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y new file mode 100644 index 0000000000..6616b80d17 --- /dev/null +++ b/source/libs/parser/inc/new_sql.y @@ -0,0 +1,345 @@ +//lemon parser file to generate sql parse by using finite-state-machine code used to parse sql +//usage: lemon sql.y + +%name NewParse + +%token_prefix NEW_TK_ +%token_type { SToken } +%default_type { SNode* } +%default_destructor { PARSER_DESTRUCTOR_TRACE; nodesDestroyNode($$); } + +%extra_argument { SAstCreateContext* pCxt } + +%include { +#include +#include +#include +#include +#include + +#include "nodes.h" +#include "ttoken.h" +#include "ttokendef.h" +#include "astCreateFuncs.h" + +#if 0 +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif +} + +%syntax_error { + if(TOKEN.z) { + char msg[] = "syntax error near \"%s\""; + int32_t sqlLen = strlen(&TOKEN.z[0]); + + if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > pCxt->pQueryCxt->msgLen) { + char tmpstr[128] = {0}; + memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); + sprintf(pCxt->pQueryCxt->pMsg, msg, tmpstr); + } else { + sprintf(pCxt->pQueryCxt->pMsg, msg, &TOKEN.z[0]); + } + } else { + sprintf(pCxt->pQueryCxt->pMsg, "Incomplete SQL statement"); + } + pCxt->valid = false; +} + +%parse_accept { PARSER_COMPLETE; } + +%left OR. +%left AND. +%left UNION ALL MINUS EXCEPT INTERSECT. +//%left BITAND BITOR LSHIFT RSHIFT. +%left NK_PLUS NK_MINUS. +//%left DIVIDE TIMES. +%left NK_STAR NK_SLASH NK_REM. +//%left CONCAT. +//%right UMINUS UPLUS BITNOT. + +cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } +cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } + +/************************************************ literal *************************************************************/ +literal(A) ::= NK_INTEGER(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } +literal(A) ::= NK_FLOAT(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +literal(A) ::= NK_STRING(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } +literal(A) ::= NK_BOOL(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } +literal(A) ::= TIMESTAMP NK_STRING(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); } +literal(A) ::= duration_literal(B). { PARSER_TRACE; A = B; } + +duration_literal(A) ::= NK_VARIABLE(B). { PARSER_TRACE; A = createDurationValueNode(pCxt, &B); } + +%type literal_list { SNodeList* } +%destructor literal_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +literal_list(A) ::= literal(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +/************************************************ names and identifiers ***********************************************/ +%type db_name { SToken } +%destructor db_name { PARSER_DESTRUCTOR_TRACE; } +db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type table_name { SToken } +%destructor table_name { PARSER_DESTRUCTOR_TRACE; } +table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type column_name { SToken } +%destructor column_name { PARSER_DESTRUCTOR_TRACE; } +column_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type function_name { SToken } +%destructor function_name { PARSER_DESTRUCTOR_TRACE; } +function_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type table_alias { SToken } +%destructor table_alias { PARSER_DESTRUCTOR_TRACE; } +table_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +%type column_alias { SToken } +%destructor column_alias { PARSER_DESTRUCTOR_TRACE; } +column_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } + +/************************************************ expression **********************************************************/ +expression(A) ::= literal(B). { PARSER_TRACE; A = B; } +//expression(A) ::= NK_QUESTION(B). { PARSER_TRACE; A = B; } +//expression(A) ::= pseudo_column(B). { PARSER_TRACE; A = B; } +expression(A) ::= column_reference(B). { PARSER_TRACE; A = B; } +expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { PARSER_TRACE; A = createFunctionNode(pCxt, &B, C); } +//expression(A) ::= cast_expression(B). { PARSER_TRACE; A = B; } +//expression(A) ::= case_expression(B). { PARSER_TRACE; A = B; } +expression(A) ::= subquery(B). { PARSER_TRACE; A = B; } +expression(A) ::= NK_LP expression(B) NK_RP. { PARSER_TRACE; A = B; } +expression(A) ::= NK_PLUS expression(B). { PARSER_TRACE; A = B; } +expression(A) ::= NK_MINUS expression(B). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, NULL); } +expression(A) ::= expression(B) NK_PLUS expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_ADD, B, C); } +expression(A) ::= expression(B) NK_MINUS expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, C); } +expression(A) ::= expression(B) NK_STAR expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MULTI, B, C); } +expression(A) ::= expression(B) NK_SLASH expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_DIV, B, C); } +expression(A) ::= expression(B) NK_REM expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MOD, B, C); } + +%type expression_list { SNodeList* } +%destructor expression_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +expression_list(A) ::= expression(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +column_reference(A) ::= column_name(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } +column_reference(A) ::= table_name(B) NK_DOT column_name(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } + +//pseudo_column(A) ::= NK_NOW. { PARSER_TRACE; A = createFunctionNode(pCxt, NULL, NULL); } + +/************************************************ predicate ***********************************************************/ +predicate(A) ::= expression(B) compare_op(C) expression(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); } +//predicate(A) ::= expression(B) compare_op sub_type expression(B). +predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createBetweenAnd(pCxt, B, C, D); } +predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createNotBetweenAnd(pCxt, C, B, D); } +predicate(A) ::= expression(B) IS NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, true); } +predicate(A) ::= expression(B) IS NOT NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, false); } +predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); } + +%type compare_op { EOperatorType } +%destructor compare_op { PARSER_DESTRUCTOR_TRACE; } +compare_op(A) ::= NK_LT. { PARSER_TRACE; A = OP_TYPE_LOWER_THAN; } +compare_op(A) ::= NK_GT. { PARSER_TRACE; A = OP_TYPE_GREATER_THAN; } +compare_op(A) ::= NK_LE. { PARSER_TRACE; A = OP_TYPE_LOWER_EQUAL; } +compare_op(A) ::= NK_GE. { PARSER_TRACE; A = OP_TYPE_GREATER_EQUAL; } +compare_op(A) ::= NK_NE. { PARSER_TRACE; A = OP_TYPE_NOT_EQUAL; } +compare_op(A) ::= NK_EQ. { PARSER_TRACE; A = OP_TYPE_EQUAL; } +compare_op(A) ::= LIKE. { PARSER_TRACE; A = OP_TYPE_LIKE; } +compare_op(A) ::= NOT LIKE. { PARSER_TRACE; A = OP_TYPE_NOT_LIKE; } +compare_op(A) ::= MATCH. { PARSER_TRACE; A = OP_TYPE_MATCH; } +compare_op(A) ::= NMATCH. { PARSER_TRACE; A = OP_TYPE_NMATCH; } + +%type in_op { EOperatorType } +%destructor in_op { PARSER_DESTRUCTOR_TRACE; } +in_op(A) ::= IN. { PARSER_TRACE; A = OP_TYPE_IN; } +in_op(A) ::= NOT IN. { PARSER_TRACE; A = OP_TYPE_NOT_IN; } + +in_predicate_value(A) ::= NK_LP expression_list(B) NK_RP. { PARSER_TRACE; A = createNodeListNode(pCxt, B); } + +/************************************************ boolean_value_expression ********************************************/ +boolean_value_expression(A) ::= boolean_primary(B). { PARSER_TRACE; A = B; } +boolean_value_expression(A) ::= NOT boolean_primary(B). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, B, NULL); } +boolean_value_expression(A) ::= + boolean_value_expression(B) OR boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, B, C); } +boolean_value_expression(A) ::= + boolean_value_expression(B) AND boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, B, C); } + +boolean_primary(A) ::= predicate(B). { PARSER_TRACE; A = B; } +boolean_primary(A) ::= NK_LP boolean_value_expression(B) NK_RP. { PARSER_TRACE; A = B; } + +/************************************************ from_clause *********************************************************/ +from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } + +table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } +table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } + +/************************************************ table_reference *****************************************************/ +table_reference(A) ::= table_primary(B). { PARSER_TRACE; A = B; } +table_reference(A) ::= joined_table(B). { PARSER_TRACE; A = B; } + +table_primary(A) ::= table_name(B) alias_opt(C). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B, &C); } +table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C, &D); } +table_primary(A) ::= subquery(B) alias_opt(C). { PARSER_TRACE; A = createTempTableNode(pCxt, B, &C); } +table_primary(A) ::= parenthesized_joined_table(B). { PARSER_TRACE; A = B; } + +%type alias_opt { SToken } +%destructor alias_opt { PARSER_DESTRUCTOR_TRACE; } +alias_opt(A) ::= . { PARSER_TRACE; A = nil_token; } +alias_opt(A) ::= table_alias(B). { PARSER_TRACE; A = B; } +alias_opt(A) ::= AS table_alias(B). { PARSER_TRACE; A = B; } + +parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { PARSER_TRACE; A = B; } +parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { PARSER_TRACE; A = B; } + +/************************************************ joined_table ********************************************************/ +joined_table(A) ::= + table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { PARSER_TRACE; A = createJoinTableNode(pCxt, C, B, D, E); } + +%type join_type { EJoinType } +%destructor join_type { PARSER_DESTRUCTOR_TRACE; } +join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; } + +/************************************************ query_specification *************************************************/ +query_specification(A) ::= + SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) + partition_by_clause_opt(F) twindow_clause_opt(G) + group_by_clause_opt(H) having_clause_opt(I). { + PARSER_TRACE; + A = createSelectStmt(pCxt, B, C, D); + A = addWhereClause(pCxt, A, E); + A = addPartitionByClause(pCxt, A, F); + A = addWindowClauseClause(pCxt, A, G); + A = addGroupByClause(pCxt, A, H); + A = addHavingClause(pCxt, A, I); + } + +%type set_quantifier_opt { bool } +%destructor set_quantifier_opt { PARSER_DESTRUCTOR_TRACE; } +set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } +set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } +set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } + +%type select_list { SNodeList* } +%destructor select_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } +select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } + +%type select_sublist { SNodeList* } +%destructor select_sublist { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +select_item(A) ::= expression(B). { PARSER_TRACE; A = B; } +select_item(A) ::= expression(B) column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= expression(B) AS column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } + +where_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +where_clause_opt(A) ::= WHERE search_condition(B). { PARSER_TRACE; A = B; } + +%type partition_by_clause_opt { SNodeList* } +%destructor partition_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +partition_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { PARSER_TRACE; A = B; } + +twindow_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +twindow_clause_opt(A) ::= + SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { PARSER_TRACE; A = createSessionWindowNode(pCxt, B, &C); } +twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { PARSER_TRACE; A = createStateWindowNode(pCxt, B); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, NULL, C, D); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP + sliding_opt(D) fill_opt(E). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, C, D, E); } + +sliding_opt(A) ::= . { PARSER_TRACE; A = NULL; } +sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { PARSER_TRACE; A = B; } + +fill_opt(A) ::= . { PARSER_TRACE; A = NULL; } +fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, B, NULL); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } + +%type fill_mode { EFillMode } +%destructor fill_mode { PARSER_DESTRUCTOR_TRACE; } +fill_mode(A) ::= NONE. { PARSER_TRACE; A = FILL_MODE_NONE; } +fill_mode(A) ::= PREV. { PARSER_TRACE; A = FILL_MODE_PREV; } +fill_mode(A) ::= NULL. { PARSER_TRACE; A = FILL_MODE_NULL; } +fill_mode(A) ::= LINEAR. { PARSER_TRACE; A = FILL_MODE_LINEAR; } +fill_mode(A) ::= NEXT. { PARSER_TRACE; A = FILL_MODE_NEXT; } + +%type group_by_clause_opt { SNodeList* } +%destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +group_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +group_by_clause_opt(A) ::= GROUP BY expression_list(B). { PARSER_TRACE; A = B; } + +having_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +having_clause_opt(A) ::= HAVING search_condition(B). { PARSER_TRACE; A = B; } + +/************************************************ query_expression ****************************************************/ +query_expression(A) ::= + query_expression_body(B) + order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { + PARSER_TRACE; + A = addOrderByClause(pCxt, B, C); + A = addSlimitClause(pCxt, A, D); + A = addLimitClause(pCxt, A, E); + } + +query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } +query_expression_body(A) ::= + query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } + +query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } +//query_primary(A) ::= +// NK_LP query_expression_body(B) +// order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP. { PARSER_TRACE; A = B;} + +%type order_by_clause_opt { SNodeList* } +%destructor order_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } + +slimit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } + +limit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } + +/************************************************ subquery ************************************************************/ +subquery(A) ::= NK_LP query_expression(B) NK_RP. { PARSER_TRACE; A = B; } + +/************************************************ search_condition ****************************************************/ +search_condition(A) ::= boolean_value_expression(B). { PARSER_TRACE; A = B; } + +/************************************************ sort_specification_list *********************************************/ +%type sort_specification_list { SNodeList* } +%destructor sort_specification_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +sort_specification_list(A) ::= + sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +sort_specification(A) ::= + expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } + +%type ordering_specification_opt EOrder +%destructor ordering_specification_opt { PARSER_DESTRUCTOR_TRACE; } +ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } + +%type null_ordering_opt ENullOrder +%destructor null_ordering_opt { PARSER_DESTRUCTOR_TRACE; } +null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } +null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } +null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } diff --git a/source/nodes/src/nodesUtil.c b/source/libs/parser/inc/parserImpl.h similarity index 64% rename from source/nodes/src/nodesUtil.c rename to source/libs/parser/inc/parserImpl.h index fe5883d809..183075d465 100644 --- a/source/nodes/src/nodesUtil.c +++ b/source/libs/parser/inc/parserImpl.h @@ -14,11 +14,24 @@ */ #include "nodes.h" +#include "parser.h" -bool isTimeorderQuery(const SNode* pQuery) { +#ifndef _TD_AST_CREATE_FUNCS_H_ +#define _TD_AST_CREATE_FUNCS_H_ +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SQuery { + SNode* pRoot; +} SQuery; + +int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery); +int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery); + +#ifdef __cplusplus } +#endif -bool isTimelineQuery(const SNode* pQuery) { - -} +#endif /*_TD_AST_CREATE_FUNCS_H_*/ diff --git a/source/libs/parser/src/astCreateContext.c b/source/libs/parser/src/astCreateContext.c new file mode 100644 index 0000000000..a8c82780ca --- /dev/null +++ b/source/libs/parser/src/astCreateContext.c @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "ttoken.h" +#include "astCreateContext.h" + +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt) { + pCxt->pQueryCxt = pQueryCxt; + pCxt->notSupport = false; + pCxt->valid = true; + pCxt->pRootNode = NULL; + return TSDB_CODE_SUCCESS; +} + +int32_t destroyAstCreateContext(SAstCreateContext* pCxt) { + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c new file mode 100644 index 0000000000..6091961ed5 --- /dev/null +++ b/source/libs/parser/src/astCreateFuncs.c @@ -0,0 +1,320 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "astCreateFuncs.h" + +#define CHECK_OUT_OF_MEM(p) \ + do { \ + if (NULL == (p)) { \ + pCxt->valid = false; \ + return NULL; \ + } \ + } while (0) + +SToken nil_token = { .type = TK_NIL, .n = 0, .z = NULL }; + +static bool checkDbName(SAstCreateContext* pCxt, const SToken* pDbName) { + if (NULL == pDbName) { + return true; + } + pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false; + return pCxt->valid; +} + +static bool checkTableName(SAstCreateContext* pCxt, const SToken* pTableName) { + if (NULL == pTableName) { + return true; + } + pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; + return pCxt->valid; +} + +static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName) { + if (NULL == pColumnName) { + return true; + } + pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false; + return pCxt->valid; +} + +SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { + SNodeList* list = nodesMakeList(); + CHECK_OUT_OF_MEM(list); + return nodesListAppend(list, pNode); +} + +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { + return nodesListAppend(pList, pNode); +} + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableAlias, const SToken* pColumnName) { + if (!checkTableName(pCxt, pTableAlias) || !checkColumnName(pCxt, pColumnName)) { + return NULL; + } + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + CHECK_OUT_OF_MEM(col); + if (NULL != pTableAlias) { + strncpy(col->tableAlias, pTableAlias->z, pTableAlias->n); + } + strncpy(col->colName, pColumnName->z, pColumnName->n); + return (SNode*)col; +} + +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + // todo + return (SNode*)val; +} + +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + // todo + return (SNode*)val; +} + +SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode) { + // todo +} + +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2) { + SLogicConditionNode* cond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->condType = type; + cond->pParameterList = nodesMakeList(); + nodesListAppend(cond->pParameterList, pParam1); + nodesListAppend(cond->pParameterList, pParam2); + return (SNode*)cond; +} + +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight) { + SOperatorNode* op = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + CHECK_OUT_OF_MEM(op); + op->opType = type; + op->pLeft = pLeft; + op->pRight = pRight; + return (SNode*)op; +} + +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, + createOperatorNode(pCxt, OP_TYPE_GREATER_EQUAL, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_LOWER_EQUAL, pExpr, pRight)); +} + +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, + createOperatorNode(pCxt, OP_TYPE_LOWER_THAN, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, pExpr, pRight)); +} + +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull) { + SIsNullCondNode* cond = (SIsNullCondNode*)nodesMakeNode(QUERY_NODE_IS_NULL_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->pExpr = pExpr; + cond->isNull = isNull; + return (SNode*)cond; +} + +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { + SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + CHECK_OUT_OF_MEM(func); + strncpy(func->functionName, pFuncName->z, pFuncName->n); + func->pParameterList = pParameterList; + return (SNode*)func; +} + +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList) { + SNodeListNode* list = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); + CHECK_OUT_OF_MEM(list); + list->pNodeList = pList; + return (SNode*)list; +} + +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias) { + if (!checkDbName(pCxt, pDbName) || !checkTableName(pCxt, pTableName)) { + return NULL; + } + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + CHECK_OUT_OF_MEM(realTable); + if (NULL != pDbName) { + strncpy(realTable->table.dbName, pDbName->z, pDbName->n); + } else { + strcpy(realTable->table.dbName, pCxt->pQueryCxt->db); + } + if (NULL != pTableAlias && TK_NIL != pTableAlias->type) { + strncpy(realTable->table.tableAlias, pTableAlias->z, pTableAlias->n); + } else { + strncpy(realTable->table.tableAlias, pTableName->z, pTableName->n); + } + strncpy(realTable->table.tableName, pTableName->z, pTableName->n); + return (SNode*)realTable; +} + +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias) { + STempTableNode* tempTable = (STempTableNode*)nodesMakeNode(QUERY_NODE_TEMP_TABLE); + CHECK_OUT_OF_MEM(tempTable); + tempTable->pSubquery = pSubquery; + if (NULL != pTableAlias && TK_NIL != pTableAlias->type) { + strncpy(tempTable->table.tableAlias, pTableAlias->z, pTableAlias->n); + } + return (SNode*)tempTable; +} + +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond) { + SJoinTableNode* joinTable = (SJoinTableNode*)nodesMakeNode(QUERY_NODE_JOIN_TABLE); + CHECK_OUT_OF_MEM(joinTable); + joinTable->joinType = type; + joinTable->pLeft = pLeft; + joinTable->pRight = pRight; + joinTable->pOnCond = pJoinCond; + return (SNode*)joinTable; +} + +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { + SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + CHECK_OUT_OF_MEM(limitNode); + // limitNode->limit = limit; + // limitNode->offset = offset; + return (SNode*)limitNode; +} + +SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) { + SOrderByExprNode* orderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + CHECK_OUT_OF_MEM(orderByExpr); + orderByExpr->pExpr = pExpr; + orderByExpr->order = order; + orderByExpr->nullOrder = nullOrder; + return (SNode*)orderByExpr; +} + +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal) { + SSessionWindowNode* session = (SSessionWindowNode*)nodesMakeNode(QUERY_NODE_SESSION_WINDOW); + CHECK_OUT_OF_MEM(session); + session->pCol = pCol; + // session->gap = getInteger(pVal); + return (SNode*)session; +} + +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { + SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); + CHECK_OUT_OF_MEM(state); + state->pCol = pCol; + return (SNode*)state; +} + +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { + SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); + CHECK_OUT_OF_MEM(interval); + interval->pInterval = pInterval; + interval->pOffset = pOffset; + interval->pSliding = pSliding; + interval->pFill = pFill; + return (SNode*)interval; +} + +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { + SFillNode* fill = (SFillNode*)nodesMakeNode(QUERY_NODE_FILL); + CHECK_OUT_OF_MEM(fill); + fill->mode = mode; + fill->pValues = pValues; + return (SNode*)fill; +} + +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { + strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n); + return pNode; +} + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWhere = pWhere; + } + return pStmt; +} + +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pPartitionByList = pPartitionByList; + } + return pStmt; +} + +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWindow = pWindow; + } + return pStmt; +} + +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pGroupByList = pGroupByList; + } + return pStmt; +} + +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pHaving = pHaving; + } + return pStmt; +} + +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pOrderByList = pOrderByList; + } + return pStmt; +} + +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pSlimit = pSlimit; + } + return pStmt; +} + +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pLimit = pLimit; + } + return pStmt; +} + +SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { + SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + CHECK_OUT_OF_MEM(select); + select->isDistinct = isDistinct; + select->pProjectionList = pProjectionList; + select->pFromTable = pTable; + return (SNode*)select; +} + +SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight) { + SSetOperator* setOp = (SSetOperator*)nodesMakeNode(QUERY_NODE_SET_OPERATOR); + CHECK_OUT_OF_MEM(setOp); + setOp->opType = type; + setOp->pLeft = pLeft; + setOp->pRight = pRight; + return (SNode*)setOp; +} + +SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type) { + SShowStmt* show = (SShowStmt*)nodesMakeNode(QUERY_NODE_SHOW_STMT); + CHECK_OUT_OF_MEM(show); + show->showType = type; + return (SNode*)show; +} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 0a2f2b20f2..a56a6524fc 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3644,6 +3644,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) return TSDB_CODE_SUCCESS; } +//TODO remove it int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); @@ -3651,21 +3652,17 @@ int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgL return code; } - int32_t vgroupNum = taosArrayGetSize(vgroupList); + size_t vgroupNum = taosArrayGetSize(vgroupList); - SVgroupsInfo *vgList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupNum); - + SVgroupsInfo *vgList = calloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * vgroupNum); vgList->numOfVgroups = vgroupNum; for (int32_t i = 0; i < vgroupNum; ++i) { SVgroupInfo *vg = taosArrayGet(vgroupList, i); - vgList->vgroups[i].vgId = vg->vgId; - vgList->vgroups[i].numOfEps = vg->numOfEps; - memcpy(vgList->vgroups[i].epAddr, vg->epAddr, sizeof(vgList->vgroups[i].epAddr)); + vgList->vgroups[i] = *vg; } *pVgList = vgList; - taosArrayDestroy(vgroupList); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 06a9a3d16e..3ae89bca0a 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -58,13 +58,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out SVgroupInfo* info = taosArrayGet(array, 0); pShowReq->head.vgId = htonl(info->vgId); - pEpSet->numOfEps = info->numOfEps; - pEpSet->inUse = info->inUse; - - for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { - strncpy(pEpSet->fqdn[i], info->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); - pEpSet->port[i] = info->epAddr[i].port; - } + *pEpSet = info->epset; *outputLen = sizeof(SVShowTablesReq); *output = pShowReq; diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c new file mode 100644 index 0000000000..8ce36a6425 --- /dev/null +++ b/source/libs/parser/src/new_sql.c @@ -0,0 +1,2184 @@ +/* +** 2000-05-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Driver template for the LEMON parser generator. +** +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: +*/ +#include +#include +/************ Begin %include sections from the grammar ************************/ + +#include +#include +#include +#include +#include + +#include "nodes.h" +#include "ttoken.h" +#include "ttokendef.h" +#include "astCreateFuncs.h" + +#if 0 +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** NewParseTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is NewParseTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** NewParseARG_SDECL A static variable declaration for the %extra_argument +** NewParseARG_PDECL A parameter declaration for the %extra_argument +** NewParseARG_PARAM Code to pass %extra_argument as a subroutine parameter +** NewParseARG_STORE Code to store %extra_argument into yypParser +** NewParseARG_FETCH Code to extract %extra_argument from yypParser +** NewParseCTX_* As NewParseARG_ except for %extra_context +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +#define YYCODETYPE unsigned char +#define YYNOCODE 124 +#define YYACTIONTYPE unsigned short int +#define NewParseTOKENTYPE SToken +typedef union { + int yyinit; + NewParseTOKENTYPE yy0; + EOrder yy10; + EFillMode yy14; + SNode* yy168; + ENullOrder yy177; + SNodeList* yy192; + bool yy209; + EOperatorType yy228; + EJoinType yy229; + SToken yy241; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define NewParseARG_SDECL SAstCreateContext* pCxt ; +#define NewParseARG_PDECL , SAstCreateContext* pCxt +#define NewParseARG_PARAM ,pCxt +#define NewParseARG_FETCH SAstCreateContext* pCxt =yypParser->pCxt ; +#define NewParseARG_STORE yypParser->pCxt =pCxt ; +#define NewParseCTX_SDECL +#define NewParseCTX_PDECL +#define NewParseCTX_PARAM +#define NewParseCTX_FETCH +#define NewParseCTX_STORE +#define YYNSTATE 143 +#define YYNRULE 134 +#define YYNTOKEN 72 +#define YY_MAX_SHIFT 142 +#define YY_MIN_SHIFTREDUCE 238 +#define YY_MAX_SHIFTREDUCE 371 +#define YY_ERROR_ACTION 372 +#define YY_ACCEPT_ACTION 373 +#define YY_NO_ACTION 374 +#define YY_MIN_REDUCE 375 +#define YY_MAX_REDUCE 508 +/************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +#define YY_ACTTAB_COUNT (705) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 384, 382, 89, 22, 64, 385, 382, 458, 29, 27, + /* 10 */ 25, 24, 23, 392, 382, 137, 406, 126, 406, 138, + /* 20 */ 71, 109, 44, 393, 259, 395, 21, 84, 93, 142, + /* 30 */ 277, 278, 279, 280, 281, 282, 283, 285, 286, 287, + /* 40 */ 29, 27, 25, 24, 23, 25, 24, 23, 125, 9, + /* 50 */ 456, 240, 241, 242, 243, 139, 246, 106, 21, 84, + /* 60 */ 93, 51, 277, 278, 279, 280, 281, 282, 283, 285, + /* 70 */ 286, 287, 127, 392, 382, 137, 406, 137, 406, 138, + /* 80 */ 315, 113, 38, 393, 114, 395, 431, 253, 28, 26, + /* 90 */ 83, 427, 487, 443, 131, 240, 241, 242, 243, 139, + /* 100 */ 246, 487, 98, 1, 340, 486, 67, 10, 51, 485, + /* 110 */ 440, 392, 382, 60, 50, 137, 406, 138, 485, 123, + /* 120 */ 39, 393, 314, 395, 431, 51, 125, 9, 91, 427, + /* 130 */ 105, 338, 339, 341, 342, 392, 382, 132, 462, 137, + /* 140 */ 406, 138, 4, 311, 39, 393, 407, 395, 431, 51, + /* 150 */ 443, 443, 91, 427, 29, 27, 25, 24, 23, 323, + /* 160 */ 392, 382, 483, 73, 137, 406, 138, 439, 438, 39, + /* 170 */ 393, 251, 395, 431, 389, 18, 387, 91, 427, 7, + /* 180 */ 6, 29, 27, 25, 24, 23, 134, 447, 127, 392, + /* 190 */ 382, 7, 6, 137, 406, 138, 28, 26, 77, 393, + /* 200 */ 8, 395, 295, 240, 241, 242, 243, 139, 246, 107, + /* 210 */ 98, 1, 28, 26, 469, 10, 104, 487, 52, 240, + /* 220 */ 241, 242, 243, 139, 246, 246, 98, 5, 102, 40, + /* 230 */ 50, 135, 337, 19, 485, 392, 382, 130, 468, 137, + /* 240 */ 406, 138, 103, 284, 39, 393, 288, 395, 431, 51, + /* 250 */ 56, 54, 430, 427, 57, 370, 371, 29, 27, 25, + /* 260 */ 24, 23, 30, 392, 382, 289, 90, 137, 406, 138, + /* 270 */ 3, 254, 39, 393, 117, 395, 431, 28, 26, 316, + /* 280 */ 118, 427, 47, 449, 240, 241, 242, 243, 139, 246, + /* 290 */ 70, 98, 5, 101, 392, 382, 129, 127, 126, 406, + /* 300 */ 138, 122, 43, 44, 393, 119, 395, 274, 30, 59, + /* 310 */ 41, 257, 2, 29, 27, 25, 24, 23, 61, 65, + /* 320 */ 436, 121, 15, 120, 69, 311, 487, 20, 413, 250, + /* 330 */ 99, 455, 42, 29, 27, 25, 24, 23, 253, 50, + /* 340 */ 28, 26, 444, 485, 31, 62, 254, 240, 241, 242, + /* 350 */ 243, 139, 246, 459, 98, 1, 392, 382, 94, 502, + /* 360 */ 137, 406, 138, 136, 484, 39, 393, 72, 395, 431, + /* 370 */ 28, 26, 367, 368, 428, 133, 251, 240, 241, 242, + /* 380 */ 243, 139, 246, 13, 98, 5, 29, 27, 25, 24, + /* 390 */ 23, 115, 110, 108, 392, 382, 12, 30, 137, 406, + /* 400 */ 138, 53, 259, 45, 393, 334, 395, 55, 34, 336, + /* 410 */ 330, 46, 58, 35, 392, 382, 373, 140, 137, 406, + /* 420 */ 138, 329, 111, 81, 393, 100, 395, 392, 382, 112, + /* 430 */ 387, 137, 406, 138, 36, 6, 81, 393, 116, 395, + /* 440 */ 128, 500, 14, 392, 382, 275, 487, 137, 406, 138, + /* 450 */ 309, 308, 81, 393, 92, 395, 392, 382, 33, 50, + /* 460 */ 137, 406, 138, 485, 66, 45, 393, 32, 395, 392, + /* 470 */ 382, 386, 49, 137, 406, 138, 361, 16, 81, 393, + /* 480 */ 97, 395, 37, 11, 356, 355, 74, 95, 360, 392, + /* 490 */ 382, 359, 96, 137, 406, 138, 244, 17, 78, 393, + /* 500 */ 376, 395, 375, 501, 374, 141, 392, 382, 374, 374, + /* 510 */ 137, 406, 138, 374, 374, 75, 393, 374, 395, 392, + /* 520 */ 382, 374, 374, 137, 406, 138, 374, 374, 79, 393, + /* 530 */ 374, 395, 392, 382, 374, 374, 137, 406, 138, 374, + /* 540 */ 374, 76, 393, 374, 395, 392, 382, 374, 374, 137, + /* 550 */ 406, 138, 374, 374, 80, 393, 374, 395, 374, 374, + /* 560 */ 374, 374, 374, 374, 392, 382, 374, 374, 137, 406, + /* 570 */ 138, 374, 374, 403, 393, 374, 395, 374, 392, 382, + /* 580 */ 374, 374, 137, 406, 138, 374, 374, 402, 393, 374, + /* 590 */ 395, 392, 382, 374, 374, 137, 406, 138, 374, 374, + /* 600 */ 401, 393, 374, 395, 392, 382, 374, 374, 137, 406, + /* 610 */ 138, 374, 374, 87, 393, 374, 395, 392, 382, 374, + /* 620 */ 374, 137, 406, 138, 374, 374, 86, 393, 374, 395, + /* 630 */ 392, 382, 374, 374, 137, 406, 138, 374, 374, 88, + /* 640 */ 393, 374, 395, 392, 382, 374, 374, 137, 406, 138, + /* 650 */ 374, 374, 85, 393, 374, 395, 374, 392, 382, 374, + /* 660 */ 374, 137, 406, 138, 374, 374, 82, 393, 374, 395, + /* 670 */ 122, 43, 374, 374, 374, 122, 43, 374, 374, 41, + /* 680 */ 374, 374, 122, 43, 41, 374, 374, 124, 63, 436, + /* 690 */ 437, 41, 441, 48, 436, 437, 374, 441, 374, 374, + /* 700 */ 68, 436, 437, 374, 441, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 74, 75, 76, 88, 89, 74, 75, 82, 8, 9, + /* 10 */ 10, 11, 12, 74, 75, 78, 79, 78, 79, 80, + /* 20 */ 122, 84, 83, 84, 24, 86, 26, 27, 28, 13, + /* 30 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + /* 40 */ 8, 9, 10, 11, 12, 10, 11, 12, 22, 23, + /* 50 */ 111, 15, 16, 17, 18, 19, 20, 114, 26, 27, + /* 60 */ 28, 45, 30, 31, 32, 33, 34, 35, 36, 37, + /* 70 */ 38, 39, 73, 74, 75, 78, 79, 78, 79, 80, + /* 80 */ 4, 84, 83, 84, 22, 86, 87, 22, 8, 9, + /* 90 */ 91, 92, 102, 81, 21, 15, 16, 17, 18, 19, + /* 100 */ 20, 102, 22, 23, 29, 115, 41, 27, 45, 119, + /* 110 */ 98, 74, 75, 107, 115, 78, 79, 80, 119, 100, + /* 120 */ 83, 84, 46, 86, 87, 45, 22, 23, 91, 92, + /* 130 */ 55, 56, 57, 58, 59, 74, 75, 64, 101, 78, + /* 140 */ 79, 80, 43, 44, 83, 84, 79, 86, 87, 45, + /* 150 */ 81, 81, 91, 92, 8, 9, 10, 11, 12, 10, + /* 160 */ 74, 75, 101, 116, 78, 79, 80, 98, 98, 83, + /* 170 */ 84, 22, 86, 87, 23, 2, 25, 91, 92, 1, + /* 180 */ 2, 8, 9, 10, 11, 12, 21, 101, 73, 74, + /* 190 */ 75, 1, 2, 78, 79, 80, 8, 9, 83, 84, + /* 200 */ 103, 86, 24, 15, 16, 17, 18, 19, 20, 54, + /* 210 */ 22, 23, 8, 9, 113, 27, 53, 102, 112, 15, + /* 220 */ 16, 17, 18, 19, 20, 20, 22, 23, 75, 21, + /* 230 */ 115, 66, 24, 26, 119, 74, 75, 3, 113, 78, + /* 240 */ 79, 80, 75, 36, 83, 84, 39, 86, 87, 45, + /* 250 */ 112, 21, 91, 92, 24, 70, 71, 8, 9, 10, + /* 260 */ 11, 12, 21, 74, 75, 24, 75, 78, 79, 80, + /* 270 */ 61, 22, 83, 84, 60, 86, 87, 8, 9, 10, + /* 280 */ 91, 92, 106, 109, 15, 16, 17, 18, 19, 20, + /* 290 */ 41, 22, 23, 48, 74, 75, 62, 73, 78, 79, + /* 300 */ 80, 77, 78, 83, 84, 27, 86, 29, 21, 108, + /* 310 */ 86, 24, 47, 8, 9, 10, 11, 12, 105, 95, + /* 320 */ 96, 97, 23, 99, 104, 44, 102, 2, 90, 22, + /* 330 */ 110, 111, 78, 8, 9, 10, 11, 12, 22, 115, + /* 340 */ 8, 9, 81, 119, 40, 93, 22, 15, 16, 17, + /* 350 */ 18, 19, 20, 82, 22, 23, 74, 75, 69, 123, + /* 360 */ 78, 79, 80, 65, 118, 83, 84, 117, 86, 87, + /* 370 */ 8, 9, 67, 68, 92, 63, 22, 15, 16, 17, + /* 380 */ 18, 19, 20, 49, 22, 23, 8, 9, 10, 11, + /* 390 */ 12, 50, 51, 52, 74, 75, 21, 21, 78, 79, + /* 400 */ 80, 24, 24, 83, 84, 24, 86, 23, 21, 24, + /* 410 */ 24, 23, 23, 23, 74, 75, 72, 73, 78, 79, + /* 420 */ 80, 24, 15, 83, 84, 85, 86, 74, 75, 21, + /* 430 */ 25, 78, 79, 80, 23, 2, 83, 84, 85, 86, + /* 440 */ 120, 121, 49, 74, 75, 29, 102, 78, 79, 80, + /* 450 */ 24, 24, 83, 84, 85, 86, 74, 75, 21, 115, + /* 460 */ 78, 79, 80, 119, 25, 83, 84, 42, 86, 74, + /* 470 */ 75, 25, 25, 78, 79, 80, 24, 21, 83, 84, + /* 480 */ 85, 86, 4, 49, 15, 15, 25, 15, 15, 74, + /* 490 */ 75, 15, 15, 78, 79, 80, 17, 23, 83, 84, + /* 500 */ 0, 86, 0, 121, 124, 14, 74, 75, 124, 124, + /* 510 */ 78, 79, 80, 124, 124, 83, 84, 124, 86, 74, + /* 520 */ 75, 124, 124, 78, 79, 80, 124, 124, 83, 84, + /* 530 */ 124, 86, 74, 75, 124, 124, 78, 79, 80, 124, + /* 540 */ 124, 83, 84, 124, 86, 74, 75, 124, 124, 78, + /* 550 */ 79, 80, 124, 124, 83, 84, 124, 86, 124, 124, + /* 560 */ 124, 124, 124, 124, 74, 75, 124, 124, 78, 79, + /* 570 */ 80, 124, 124, 83, 84, 124, 86, 124, 74, 75, + /* 580 */ 124, 124, 78, 79, 80, 124, 124, 83, 84, 124, + /* 590 */ 86, 74, 75, 124, 124, 78, 79, 80, 124, 124, + /* 600 */ 83, 84, 124, 86, 74, 75, 124, 124, 78, 79, + /* 610 */ 80, 124, 124, 83, 84, 124, 86, 74, 75, 124, + /* 620 */ 124, 78, 79, 80, 124, 124, 83, 84, 124, 86, + /* 630 */ 74, 75, 124, 124, 78, 79, 80, 124, 124, 83, + /* 640 */ 84, 124, 86, 74, 75, 124, 124, 78, 79, 80, + /* 650 */ 124, 124, 83, 84, 124, 86, 124, 74, 75, 124, + /* 660 */ 124, 78, 79, 80, 124, 124, 83, 84, 124, 86, + /* 670 */ 77, 78, 124, 124, 124, 77, 78, 124, 124, 86, + /* 680 */ 124, 124, 77, 78, 86, 124, 124, 94, 95, 96, + /* 690 */ 97, 86, 99, 95, 96, 97, 124, 99, 124, 124, + /* 700 */ 95, 96, 97, 124, 99, +}; +#define YY_SHIFT_COUNT (142) +#define YY_SHIFT_MIN (0) +#define YY_SHIFT_MAX (502) +static const unsigned short int yy_shift_ofst[] = { + /* 0 */ 16, 80, 188, 188, 188, 204, 188, 188, 269, 104, + /* 10 */ 332, 362, 362, 362, 362, 362, 362, 362, 362, 362, + /* 20 */ 362, 362, 362, 362, 362, 362, 362, 362, 362, 362, + /* 30 */ 362, 26, 26, 26, 36, 62, 62, 63, 0, 32, + /* 40 */ 36, 65, 65, 65, 249, 305, 75, 341, 99, 149, + /* 50 */ 234, 76, 155, 163, 205, 205, 155, 163, 205, 209, + /* 60 */ 214, 245, 265, 281, 299, 281, 307, 316, 281, 304, + /* 70 */ 324, 289, 298, 312, 354, 173, 325, 378, 146, 146, + /* 80 */ 146, 146, 146, 178, 207, 35, 35, 35, 35, 208, + /* 90 */ 230, 190, 241, 278, 185, 73, 165, 287, 151, 375, + /* 100 */ 376, 334, 377, 381, 384, 387, 385, 388, 389, 386, + /* 110 */ 390, 397, 407, 408, 405, 411, 376, 393, 433, 416, + /* 120 */ 426, 427, 439, 425, 437, 446, 447, 452, 456, 434, + /* 130 */ 478, 469, 470, 472, 473, 476, 477, 461, 474, 479, + /* 140 */ 500, 502, 491, +}; +#define YY_REDUCE_COUNT (74) +#define YY_REDUCE_MIN (-102) +#define YY_REDUCE_MAX (605) +static const short yy_reduce_ofst[] = { + /* 0 */ 344, -1, 37, 61, 86, 115, 161, 189, 220, 224, + /* 10 */ 282, 320, -61, 340, 353, 369, 382, 395, 415, 432, + /* 20 */ 445, 458, 471, 490, 504, 517, 530, 543, 556, 569, + /* 30 */ 583, 593, 598, 605, -74, -63, -3, -10, -85, -85, + /* 40 */ -69, 12, 69, 70, -75, -102, -57, 6, 19, 67, + /* 50 */ 47, 97, 101, 106, 153, 167, 125, 138, 191, 174, + /* 60 */ 201, 176, 213, 19, 238, 19, 254, 261, 19, 252, + /* 70 */ 271, 236, 246, 250, 67, +}; +static const YYACTIONTYPE yy_default[] = { + /* 0 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 10 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 20 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 30 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 40 */ 372, 442, 442, 442, 457, 503, 372, 465, 372, 372, + /* 50 */ 488, 450, 472, 470, 372, 372, 472, 470, 372, 482, + /* 60 */ 480, 463, 461, 434, 372, 372, 372, 372, 435, 372, + /* 70 */ 372, 506, 494, 490, 372, 372, 372, 372, 410, 409, + /* 80 */ 408, 404, 405, 372, 372, 399, 400, 398, 397, 372, + /* 90 */ 372, 499, 372, 372, 372, 491, 495, 372, 388, 454, + /* 100 */ 464, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 110 */ 372, 372, 372, 372, 388, 372, 481, 372, 429, 372, + /* 120 */ 441, 437, 372, 372, 433, 387, 372, 372, 489, 372, + /* 130 */ 372, 372, 372, 372, 372, 372, 372, 372, 372, 372, + /* 140 */ 372, 372, 372, +}; +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + yyStackEntry *yytos; /* Pointer to top element of the stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyhwm; /* High-water mark of the stack */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + NewParseARG_SDECL /* A place to hold %extra_argument */ + NewParseCTX_SDECL /* A place to hold %extra_context */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ +#endif +}; +typedef struct yyParser yyParser; + +#ifndef NDEBUG +#include +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
    +**
  • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
  • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
+** +** Outputs: +** None. +*/ +void NewParseTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "OR", + /* 2 */ "AND", + /* 3 */ "UNION", + /* 4 */ "ALL", + /* 5 */ "MINUS", + /* 6 */ "EXCEPT", + /* 7 */ "INTERSECT", + /* 8 */ "NK_PLUS", + /* 9 */ "NK_MINUS", + /* 10 */ "NK_STAR", + /* 11 */ "NK_SLASH", + /* 12 */ "NK_REM", + /* 13 */ "SHOW", + /* 14 */ "DATABASES", + /* 15 */ "NK_INTEGER", + /* 16 */ "NK_FLOAT", + /* 17 */ "NK_STRING", + /* 18 */ "NK_BOOL", + /* 19 */ "TIMESTAMP", + /* 20 */ "NK_VARIABLE", + /* 21 */ "NK_COMMA", + /* 22 */ "NK_ID", + /* 23 */ "NK_LP", + /* 24 */ "NK_RP", + /* 25 */ "NK_DOT", + /* 26 */ "BETWEEN", + /* 27 */ "NOT", + /* 28 */ "IS", + /* 29 */ "NULL", + /* 30 */ "NK_LT", + /* 31 */ "NK_GT", + /* 32 */ "NK_LE", + /* 33 */ "NK_GE", + /* 34 */ "NK_NE", + /* 35 */ "NK_EQ", + /* 36 */ "LIKE", + /* 37 */ "MATCH", + /* 38 */ "NMATCH", + /* 39 */ "IN", + /* 40 */ "FROM", + /* 41 */ "AS", + /* 42 */ "JOIN", + /* 43 */ "ON", + /* 44 */ "INNER", + /* 45 */ "SELECT", + /* 46 */ "DISTINCT", + /* 47 */ "WHERE", + /* 48 */ "PARTITION", + /* 49 */ "BY", + /* 50 */ "SESSION", + /* 51 */ "STATE_WINDOW", + /* 52 */ "INTERVAL", + /* 53 */ "SLIDING", + /* 54 */ "FILL", + /* 55 */ "VALUE", + /* 56 */ "NONE", + /* 57 */ "PREV", + /* 58 */ "LINEAR", + /* 59 */ "NEXT", + /* 60 */ "GROUP", + /* 61 */ "HAVING", + /* 62 */ "ORDER", + /* 63 */ "SLIMIT", + /* 64 */ "SOFFSET", + /* 65 */ "LIMIT", + /* 66 */ "OFFSET", + /* 67 */ "ASC", + /* 68 */ "DESC", + /* 69 */ "NULLS", + /* 70 */ "FIRST", + /* 71 */ "LAST", + /* 72 */ "cmd", + /* 73 */ "query_expression", + /* 74 */ "literal", + /* 75 */ "duration_literal", + /* 76 */ "literal_list", + /* 77 */ "db_name", + /* 78 */ "table_name", + /* 79 */ "column_name", + /* 80 */ "function_name", + /* 81 */ "table_alias", + /* 82 */ "column_alias", + /* 83 */ "expression", + /* 84 */ "column_reference", + /* 85 */ "expression_list", + /* 86 */ "subquery", + /* 87 */ "predicate", + /* 88 */ "compare_op", + /* 89 */ "in_op", + /* 90 */ "in_predicate_value", + /* 91 */ "boolean_value_expression", + /* 92 */ "boolean_primary", + /* 93 */ "from_clause", + /* 94 */ "table_reference_list", + /* 95 */ "table_reference", + /* 96 */ "table_primary", + /* 97 */ "joined_table", + /* 98 */ "alias_opt", + /* 99 */ "parenthesized_joined_table", + /* 100 */ "join_type", + /* 101 */ "search_condition", + /* 102 */ "query_specification", + /* 103 */ "set_quantifier_opt", + /* 104 */ "select_list", + /* 105 */ "where_clause_opt", + /* 106 */ "partition_by_clause_opt", + /* 107 */ "twindow_clause_opt", + /* 108 */ "group_by_clause_opt", + /* 109 */ "having_clause_opt", + /* 110 */ "select_sublist", + /* 111 */ "select_item", + /* 112 */ "sliding_opt", + /* 113 */ "fill_opt", + /* 114 */ "fill_mode", + /* 115 */ "query_expression_body", + /* 116 */ "order_by_clause_opt", + /* 117 */ "slimit_clause_opt", + /* 118 */ "limit_clause_opt", + /* 119 */ "query_primary", + /* 120 */ "sort_specification_list", + /* 121 */ "sort_specification", + /* 122 */ "ordering_specification_opt", + /* 123 */ "null_ordering_opt", +}; +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { + /* 0 */ "cmd ::= SHOW DATABASES", + /* 1 */ "cmd ::= query_expression", + /* 2 */ "literal ::= NK_INTEGER", + /* 3 */ "literal ::= NK_FLOAT", + /* 4 */ "literal ::= NK_STRING", + /* 5 */ "literal ::= NK_BOOL", + /* 6 */ "literal ::= TIMESTAMP NK_STRING", + /* 7 */ "literal ::= duration_literal", + /* 8 */ "duration_literal ::= NK_VARIABLE", + /* 9 */ "literal_list ::= literal", + /* 10 */ "literal_list ::= literal_list NK_COMMA literal", + /* 11 */ "db_name ::= NK_ID", + /* 12 */ "table_name ::= NK_ID", + /* 13 */ "column_name ::= NK_ID", + /* 14 */ "function_name ::= NK_ID", + /* 15 */ "table_alias ::= NK_ID", + /* 16 */ "column_alias ::= NK_ID", + /* 17 */ "expression ::= literal", + /* 18 */ "expression ::= column_reference", + /* 19 */ "expression ::= function_name NK_LP expression_list NK_RP", + /* 20 */ "expression ::= subquery", + /* 21 */ "expression ::= NK_LP expression NK_RP", + /* 22 */ "expression ::= NK_PLUS expression", + /* 23 */ "expression ::= NK_MINUS expression", + /* 24 */ "expression ::= expression NK_PLUS expression", + /* 25 */ "expression ::= expression NK_MINUS expression", + /* 26 */ "expression ::= expression NK_STAR expression", + /* 27 */ "expression ::= expression NK_SLASH expression", + /* 28 */ "expression ::= expression NK_REM expression", + /* 29 */ "expression_list ::= expression", + /* 30 */ "expression_list ::= expression_list NK_COMMA expression", + /* 31 */ "column_reference ::= column_name", + /* 32 */ "column_reference ::= table_name NK_DOT column_name", + /* 33 */ "predicate ::= expression compare_op expression", + /* 34 */ "predicate ::= expression BETWEEN expression AND expression", + /* 35 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 36 */ "predicate ::= expression IS NULL", + /* 37 */ "predicate ::= expression IS NOT NULL", + /* 38 */ "predicate ::= expression in_op in_predicate_value", + /* 39 */ "compare_op ::= NK_LT", + /* 40 */ "compare_op ::= NK_GT", + /* 41 */ "compare_op ::= NK_LE", + /* 42 */ "compare_op ::= NK_GE", + /* 43 */ "compare_op ::= NK_NE", + /* 44 */ "compare_op ::= NK_EQ", + /* 45 */ "compare_op ::= LIKE", + /* 46 */ "compare_op ::= NOT LIKE", + /* 47 */ "compare_op ::= MATCH", + /* 48 */ "compare_op ::= NMATCH", + /* 49 */ "in_op ::= IN", + /* 50 */ "in_op ::= NOT IN", + /* 51 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 52 */ "boolean_value_expression ::= boolean_primary", + /* 53 */ "boolean_value_expression ::= NOT boolean_primary", + /* 54 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 55 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 56 */ "boolean_primary ::= predicate", + /* 57 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 58 */ "from_clause ::= FROM table_reference_list", + /* 59 */ "table_reference_list ::= table_reference", + /* 60 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 61 */ "table_reference ::= table_primary", + /* 62 */ "table_reference ::= joined_table", + /* 63 */ "table_primary ::= table_name alias_opt", + /* 64 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 65 */ "table_primary ::= subquery alias_opt", + /* 66 */ "table_primary ::= parenthesized_joined_table", + /* 67 */ "alias_opt ::=", + /* 68 */ "alias_opt ::= table_alias", + /* 69 */ "alias_opt ::= AS table_alias", + /* 70 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 71 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 72 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 73 */ "join_type ::= INNER", + /* 74 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 75 */ "set_quantifier_opt ::=", + /* 76 */ "set_quantifier_opt ::= DISTINCT", + /* 77 */ "set_quantifier_opt ::= ALL", + /* 78 */ "select_list ::= NK_STAR", + /* 79 */ "select_list ::= select_sublist", + /* 80 */ "select_sublist ::= select_item", + /* 81 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 82 */ "select_item ::= expression", + /* 83 */ "select_item ::= expression column_alias", + /* 84 */ "select_item ::= expression AS column_alias", + /* 85 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 86 */ "where_clause_opt ::=", + /* 87 */ "where_clause_opt ::= WHERE search_condition", + /* 88 */ "partition_by_clause_opt ::=", + /* 89 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 90 */ "twindow_clause_opt ::=", + /* 91 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 92 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 93 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 94 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 95 */ "sliding_opt ::=", + /* 96 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 97 */ "fill_opt ::=", + /* 98 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 99 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 100 */ "fill_mode ::= NONE", + /* 101 */ "fill_mode ::= PREV", + /* 102 */ "fill_mode ::= NULL", + /* 103 */ "fill_mode ::= LINEAR", + /* 104 */ "fill_mode ::= NEXT", + /* 105 */ "group_by_clause_opt ::=", + /* 106 */ "group_by_clause_opt ::= GROUP BY expression_list", + /* 107 */ "having_clause_opt ::=", + /* 108 */ "having_clause_opt ::= HAVING search_condition", + /* 109 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 110 */ "query_expression_body ::= query_primary", + /* 111 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 112 */ "query_primary ::= query_specification", + /* 113 */ "order_by_clause_opt ::=", + /* 114 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 115 */ "slimit_clause_opt ::=", + /* 116 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 117 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 118 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 119 */ "limit_clause_opt ::=", + /* 120 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 121 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 122 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 123 */ "subquery ::= NK_LP query_expression NK_RP", + /* 124 */ "search_condition ::= boolean_value_expression", + /* 125 */ "sort_specification_list ::= sort_specification", + /* 126 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 127 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 128 */ "ordering_specification_opt ::=", + /* 129 */ "ordering_specification_opt ::= ASC", + /* 130 */ "ordering_specification_opt ::= DESC", + /* 131 */ "null_ordering_opt ::=", + /* 132 */ "null_ordering_opt ::= NULLS FIRST", + /* 133 */ "null_ordering_opt ::= NULLS LAST", +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int yyGrowStack(yyParser *p){ + int newSize; + int idx; + yyStackEntry *pNew; + + newSize = p->yystksz*2 + 100; + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } + if( pNew ){ + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); + } +#endif + p->yystksz = newSize; + } + return pNew==0; +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to NewParseAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +void NewParseInit(void *yypRawParser NewParseCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + NewParseCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +#if YYSTACKDEPTH>0 + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#endif +} + +#ifndef NewParse_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to NewParse and NewParseFree. +*/ +void *NewParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) NewParseCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + NewParseCTX_STORE + NewParseInit(yypParser NewParseCTX_PARAM); + } + return (void*)yypParser; +} +#endif /* NewParse_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ + /* Default NON-TERMINAL Destructor */ + case 72: /* cmd */ + case 73: /* query_expression */ + case 74: /* literal */ + case 75: /* duration_literal */ + case 83: /* expression */ + case 84: /* column_reference */ + case 86: /* subquery */ + case 87: /* predicate */ + case 90: /* in_predicate_value */ + case 91: /* boolean_value_expression */ + case 92: /* boolean_primary */ + case 93: /* from_clause */ + case 94: /* table_reference_list */ + case 95: /* table_reference */ + case 96: /* table_primary */ + case 97: /* joined_table */ + case 99: /* parenthesized_joined_table */ + case 101: /* search_condition */ + case 102: /* query_specification */ + case 105: /* where_clause_opt */ + case 107: /* twindow_clause_opt */ + case 109: /* having_clause_opt */ + case 111: /* select_item */ + case 112: /* sliding_opt */ + case 113: /* fill_opt */ + case 115: /* query_expression_body */ + case 117: /* slimit_clause_opt */ + case 118: /* limit_clause_opt */ + case 119: /* query_primary */ + case 121: /* sort_specification */ +{ + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy168)); +} + break; + case 76: /* literal_list */ + case 85: /* expression_list */ + case 104: /* select_list */ + case 106: /* partition_by_clause_opt */ + case 108: /* group_by_clause_opt */ + case 110: /* select_sublist */ + case 116: /* order_by_clause_opt */ + case 120: /* sort_specification_list */ +{ + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy192)); +} + break; + case 77: /* db_name */ + case 78: /* table_name */ + case 79: /* column_name */ + case 80: /* function_name */ + case 81: /* table_alias */ + case 82: /* column_alias */ + case 98: /* alias_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 88: /* compare_op */ + case 89: /* in_op */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 100: /* join_type */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 103: /* set_quantifier_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 114: /* fill_mode */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 122: /* ordering_specification_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 123: /* null_ordering_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +void NewParseFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef NewParse_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +void NewParseFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + NewParseFinalize(p); + (*freeProc)(p); +} +#endif /* NewParse_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +int NewParseStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +int NewParseCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = yy_shift_ofst[stateno]; + assert( i>=0 ); + /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); + i += iLookAhead; + if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + if( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j0 + ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + return yy_action[i]; + } + }while(1); +} + +/* +** Find the appropriate action for a parser given the non-terminal +** look-ahead token iLookAhead. +*/ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ + YYCODETYPE iLookAhead /* The look-ahead token */ +){ + int i; +#ifdef YYERRORSYMBOL + if( stateno>YY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ +/******** End %stack_overflow code ********************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument var */ + NewParseCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ + if( yyTraceFILE ){ + if( yyNewStateyytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); + } + } +} +#else +# define yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ + NewParseTOKENTYPE yyMinor /* The minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yytos++; +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>yypParser->yystackEnd ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } + } +#endif + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; + yytos->stateno = yyNewState; + yytos->major = yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState, "Shift"); +} + +/* The following table contains information about every rule that +** is used during the reduce. +*/ +static const struct { + YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + signed char nrhs; /* Negative of the number of RHS symbols in the rule */ +} yyRuleInfo[] = { + { 72, -2 }, /* (0) cmd ::= SHOW DATABASES */ + { 72, -1 }, /* (1) cmd ::= query_expression */ + { 74, -1 }, /* (2) literal ::= NK_INTEGER */ + { 74, -1 }, /* (3) literal ::= NK_FLOAT */ + { 74, -1 }, /* (4) literal ::= NK_STRING */ + { 74, -1 }, /* (5) literal ::= NK_BOOL */ + { 74, -2 }, /* (6) literal ::= TIMESTAMP NK_STRING */ + { 74, -1 }, /* (7) literal ::= duration_literal */ + { 75, -1 }, /* (8) duration_literal ::= NK_VARIABLE */ + { 76, -1 }, /* (9) literal_list ::= literal */ + { 76, -3 }, /* (10) literal_list ::= literal_list NK_COMMA literal */ + { 77, -1 }, /* (11) db_name ::= NK_ID */ + { 78, -1 }, /* (12) table_name ::= NK_ID */ + { 79, -1 }, /* (13) column_name ::= NK_ID */ + { 80, -1 }, /* (14) function_name ::= NK_ID */ + { 81, -1 }, /* (15) table_alias ::= NK_ID */ + { 82, -1 }, /* (16) column_alias ::= NK_ID */ + { 83, -1 }, /* (17) expression ::= literal */ + { 83, -1 }, /* (18) expression ::= column_reference */ + { 83, -4 }, /* (19) expression ::= function_name NK_LP expression_list NK_RP */ + { 83, -1 }, /* (20) expression ::= subquery */ + { 83, -3 }, /* (21) expression ::= NK_LP expression NK_RP */ + { 83, -2 }, /* (22) expression ::= NK_PLUS expression */ + { 83, -2 }, /* (23) expression ::= NK_MINUS expression */ + { 83, -3 }, /* (24) expression ::= expression NK_PLUS expression */ + { 83, -3 }, /* (25) expression ::= expression NK_MINUS expression */ + { 83, -3 }, /* (26) expression ::= expression NK_STAR expression */ + { 83, -3 }, /* (27) expression ::= expression NK_SLASH expression */ + { 83, -3 }, /* (28) expression ::= expression NK_REM expression */ + { 85, -1 }, /* (29) expression_list ::= expression */ + { 85, -3 }, /* (30) expression_list ::= expression_list NK_COMMA expression */ + { 84, -1 }, /* (31) column_reference ::= column_name */ + { 84, -3 }, /* (32) column_reference ::= table_name NK_DOT column_name */ + { 87, -3 }, /* (33) predicate ::= expression compare_op expression */ + { 87, -5 }, /* (34) predicate ::= expression BETWEEN expression AND expression */ + { 87, -6 }, /* (35) predicate ::= expression NOT BETWEEN expression AND expression */ + { 87, -3 }, /* (36) predicate ::= expression IS NULL */ + { 87, -4 }, /* (37) predicate ::= expression IS NOT NULL */ + { 87, -3 }, /* (38) predicate ::= expression in_op in_predicate_value */ + { 88, -1 }, /* (39) compare_op ::= NK_LT */ + { 88, -1 }, /* (40) compare_op ::= NK_GT */ + { 88, -1 }, /* (41) compare_op ::= NK_LE */ + { 88, -1 }, /* (42) compare_op ::= NK_GE */ + { 88, -1 }, /* (43) compare_op ::= NK_NE */ + { 88, -1 }, /* (44) compare_op ::= NK_EQ */ + { 88, -1 }, /* (45) compare_op ::= LIKE */ + { 88, -2 }, /* (46) compare_op ::= NOT LIKE */ + { 88, -1 }, /* (47) compare_op ::= MATCH */ + { 88, -1 }, /* (48) compare_op ::= NMATCH */ + { 89, -1 }, /* (49) in_op ::= IN */ + { 89, -2 }, /* (50) in_op ::= NOT IN */ + { 90, -3 }, /* (51) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 91, -1 }, /* (52) boolean_value_expression ::= boolean_primary */ + { 91, -2 }, /* (53) boolean_value_expression ::= NOT boolean_primary */ + { 91, -3 }, /* (54) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 91, -3 }, /* (55) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 92, -1 }, /* (56) boolean_primary ::= predicate */ + { 92, -3 }, /* (57) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 93, -2 }, /* (58) from_clause ::= FROM table_reference_list */ + { 94, -1 }, /* (59) table_reference_list ::= table_reference */ + { 94, -3 }, /* (60) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 95, -1 }, /* (61) table_reference ::= table_primary */ + { 95, -1 }, /* (62) table_reference ::= joined_table */ + { 96, -2 }, /* (63) table_primary ::= table_name alias_opt */ + { 96, -4 }, /* (64) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 96, -2 }, /* (65) table_primary ::= subquery alias_opt */ + { 96, -1 }, /* (66) table_primary ::= parenthesized_joined_table */ + { 98, 0 }, /* (67) alias_opt ::= */ + { 98, -1 }, /* (68) alias_opt ::= table_alias */ + { 98, -2 }, /* (69) alias_opt ::= AS table_alias */ + { 99, -3 }, /* (70) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 99, -3 }, /* (71) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 97, -6 }, /* (72) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 100, -1 }, /* (73) join_type ::= INNER */ + { 102, -9 }, /* (74) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 103, 0 }, /* (75) set_quantifier_opt ::= */ + { 103, -1 }, /* (76) set_quantifier_opt ::= DISTINCT */ + { 103, -1 }, /* (77) set_quantifier_opt ::= ALL */ + { 104, -1 }, /* (78) select_list ::= NK_STAR */ + { 104, -1 }, /* (79) select_list ::= select_sublist */ + { 110, -1 }, /* (80) select_sublist ::= select_item */ + { 110, -3 }, /* (81) select_sublist ::= select_sublist NK_COMMA select_item */ + { 111, -1 }, /* (82) select_item ::= expression */ + { 111, -2 }, /* (83) select_item ::= expression column_alias */ + { 111, -3 }, /* (84) select_item ::= expression AS column_alias */ + { 111, -3 }, /* (85) select_item ::= table_name NK_DOT NK_STAR */ + { 105, 0 }, /* (86) where_clause_opt ::= */ + { 105, -2 }, /* (87) where_clause_opt ::= WHERE search_condition */ + { 106, 0 }, /* (88) partition_by_clause_opt ::= */ + { 106, -3 }, /* (89) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 107, 0 }, /* (90) twindow_clause_opt ::= */ + { 107, -6 }, /* (91) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 107, -4 }, /* (92) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 107, -6 }, /* (93) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 107, -8 }, /* (94) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 112, 0 }, /* (95) sliding_opt ::= */ + { 112, -4 }, /* (96) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 113, 0 }, /* (97) fill_opt ::= */ + { 113, -4 }, /* (98) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 113, -6 }, /* (99) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 114, -1 }, /* (100) fill_mode ::= NONE */ + { 114, -1 }, /* (101) fill_mode ::= PREV */ + { 114, -1 }, /* (102) fill_mode ::= NULL */ + { 114, -1 }, /* (103) fill_mode ::= LINEAR */ + { 114, -1 }, /* (104) fill_mode ::= NEXT */ + { 108, 0 }, /* (105) group_by_clause_opt ::= */ + { 108, -3 }, /* (106) group_by_clause_opt ::= GROUP BY expression_list */ + { 109, 0 }, /* (107) having_clause_opt ::= */ + { 109, -2 }, /* (108) having_clause_opt ::= HAVING search_condition */ + { 73, -4 }, /* (109) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 115, -1 }, /* (110) query_expression_body ::= query_primary */ + { 115, -4 }, /* (111) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 119, -1 }, /* (112) query_primary ::= query_specification */ + { 116, 0 }, /* (113) order_by_clause_opt ::= */ + { 116, -3 }, /* (114) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 117, 0 }, /* (115) slimit_clause_opt ::= */ + { 117, -2 }, /* (116) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 117, -4 }, /* (117) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 117, -4 }, /* (118) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 118, 0 }, /* (119) limit_clause_opt ::= */ + { 118, -2 }, /* (120) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 118, -4 }, /* (121) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 118, -4 }, /* (122) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 86, -3 }, /* (123) subquery ::= NK_LP query_expression NK_RP */ + { 101, -1 }, /* (124) search_condition ::= boolean_value_expression */ + { 120, -1 }, /* (125) sort_specification_list ::= sort_specification */ + { 120, -3 }, /* (126) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 121, -3 }, /* (127) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 122, 0 }, /* (128) ordering_specification_opt ::= */ + { 122, -1 }, /* (129) ordering_specification_opt ::= ASC */ + { 122, -1 }, /* (130) ordering_specification_opt ::= DESC */ + { 123, 0 }, /* (131) null_ordering_opt ::= */ + { 123, -2 }, /* (132) null_ordering_opt ::= NULLS FIRST */ + { 123, -2 }, /* (133) null_ordering_opt ::= NULLS LAST */ +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static YYACTIONTYPE yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + NewParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + NewParseCTX_PDECL /* %extra_context */ +){ + int yygoto; /* The next state */ + YYACTIONTYPE yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + NewParseARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; +#ifndef NDEBUG + if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + yysize = yyRuleInfo[yyruleno].nrhs; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s].\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno]); + } + } +#endif /* NDEBUG */ + + /* Check that the stack is large enough to grow by a single entry + ** if the RHS of the rule is empty. This ensures that there is room + ** enough on the stack to push the LHS value */ + if( yyRuleInfo[yyruleno].nrhs==0 ){ +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } + yymsp = yypParser->yytos; + } +#endif + } + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ + YYMINORTYPE yylhsminor; + case 0: /* cmd ::= SHOW DATABASES */ +{ PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } + break; + case 1: /* cmd ::= query_expression */ +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy168; } + break; + case 2: /* literal ::= NK_INTEGER */ +{ PARSER_TRACE; yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 3: /* literal ::= NK_FLOAT */ +{ PARSER_TRACE; yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 4: /* literal ::= NK_STRING */ +{ PARSER_TRACE; yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 5: /* literal ::= NK_BOOL */ +{ PARSER_TRACE; yylhsminor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 6: /* literal ::= TIMESTAMP NK_STRING */ +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 7: /* literal ::= duration_literal */ + case 17: /* expression ::= literal */ yytestcase(yyruleno==17); + case 18: /* expression ::= column_reference */ yytestcase(yyruleno==18); + case 20: /* expression ::= subquery */ yytestcase(yyruleno==20); + case 52: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==52); + case 56: /* boolean_primary ::= predicate */ yytestcase(yyruleno==56); + case 59: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==59); + case 61: /* table_reference ::= table_primary */ yytestcase(yyruleno==61); + case 62: /* table_reference ::= joined_table */ yytestcase(yyruleno==62); + case 66: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==66); + case 82: /* select_item ::= expression */ yytestcase(yyruleno==82); + case 110: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==110); + case 112: /* query_primary ::= query_specification */ yytestcase(yyruleno==112); + case 124: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==124); +{ PARSER_TRACE; yylhsminor.yy168 = yymsp[0].minor.yy168; } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 8: /* duration_literal ::= NK_VARIABLE */ +{ PARSER_TRACE; yylhsminor.yy168 = createDurationValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 9: /* literal_list ::= literal */ + case 29: /* expression_list ::= expression */ yytestcase(yyruleno==29); + case 80: /* select_sublist ::= select_item */ yytestcase(yyruleno==80); + case 125: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==125); +{ PARSER_TRACE; yylhsminor.yy192 = createNodeList(pCxt, yymsp[0].minor.yy168); } + yymsp[0].minor.yy192 = yylhsminor.yy192; + break; + case 10: /* literal_list ::= literal_list NK_COMMA literal */ + case 30: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==30); + case 81: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==81); + case 126: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==126); +{ PARSER_TRACE; yylhsminor.yy192 = addNodeToList(pCxt, yymsp[-2].minor.yy192, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy192 = yylhsminor.yy192; + break; + case 11: /* db_name ::= NK_ID */ + case 12: /* table_name ::= NK_ID */ yytestcase(yyruleno==12); + case 13: /* column_name ::= NK_ID */ yytestcase(yyruleno==13); + case 14: /* function_name ::= NK_ID */ yytestcase(yyruleno==14); + case 15: /* table_alias ::= NK_ID */ yytestcase(yyruleno==15); + case 16: /* column_alias ::= NK_ID */ yytestcase(yyruleno==16); +{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy241 = yylhsminor.yy241; + break; + case 19: /* expression ::= function_name NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yylhsminor.yy168 = createFunctionNode(pCxt, &yymsp[-3].minor.yy241, yymsp[-1].minor.yy192); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 21: /* expression ::= NK_LP expression NK_RP */ + case 57: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==57); + case 70: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==70); + case 71: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==71); + case 123: /* subquery ::= NK_LP query_expression NK_RP */ yytestcase(yyruleno==123); +{ PARSER_TRACE; yymsp[-2].minor.yy168 = yymsp[-1].minor.yy168; } + break; + case 22: /* expression ::= NK_PLUS expression */ + case 58: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==58); + case 87: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==87); + case 108: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==108); +{ PARSER_TRACE; yymsp[-1].minor.yy168 = yymsp[0].minor.yy168; } + break; + case 23: /* expression ::= NK_MINUS expression */ +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createOperatorNode(pCxt, OP_TYPE_SUB, yymsp[0].minor.yy168, NULL); } + break; + case 24: /* expression ::= expression NK_PLUS expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, OP_TYPE_ADD, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 25: /* expression ::= expression NK_MINUS expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, OP_TYPE_SUB, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 26: /* expression ::= expression NK_STAR expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, OP_TYPE_MULTI, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 27: /* expression ::= expression NK_SLASH expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, OP_TYPE_DIV, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 28: /* expression ::= expression NK_REM expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, OP_TYPE_MOD, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 31: /* column_reference ::= column_name */ +{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy241); } + yymsp[0].minor.yy168 = yylhsminor.yy168; + break; + case 32: /* column_reference ::= table_name NK_DOT column_name */ +{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 33: /* predicate ::= expression compare_op expression */ + case 38: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==38); +{ PARSER_TRACE; yylhsminor.yy168 = createOperatorNode(pCxt, yymsp[-1].minor.yy228, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 34: /* predicate ::= expression BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createBetweenAnd(pCxt, yymsp[-4].minor.yy168, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-4].minor.yy168 = yylhsminor.yy168; + break; + case 35: /* predicate ::= expression NOT BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createNotBetweenAnd(pCxt, yymsp[-2].minor.yy168, yymsp[-5].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-5].minor.yy168 = yylhsminor.yy168; + break; + case 36: /* predicate ::= expression IS NULL */ +{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, yymsp[-2].minor.yy168, true); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 37: /* predicate ::= expression IS NOT NULL */ +{ PARSER_TRACE; yylhsminor.yy168 = createIsNullCondNode(pCxt, yymsp[-3].minor.yy168, false); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 39: /* compare_op ::= NK_LT */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_THAN; } + break; + case 40: /* compare_op ::= NK_GT */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_THAN; } + break; + case 41: /* compare_op ::= NK_LE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LOWER_EQUAL; } + break; + case 42: /* compare_op ::= NK_GE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_GREATER_EQUAL; } + break; + case 43: /* compare_op ::= NK_NE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NOT_EQUAL; } + break; + case 44: /* compare_op ::= NK_EQ */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_EQUAL; } + break; + case 45: /* compare_op ::= LIKE */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_LIKE; } + break; + case 46: /* compare_op ::= NOT LIKE */ +{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_LIKE; } + break; + case 47: /* compare_op ::= MATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_MATCH; } + break; + case 48: /* compare_op ::= NMATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_NMATCH; } + break; + case 49: /* in_op ::= IN */ +{ PARSER_TRACE; yymsp[0].minor.yy228 = OP_TYPE_IN; } + break; + case 50: /* in_op ::= NOT IN */ +{ PARSER_TRACE; yymsp[-1].minor.yy228 = OP_TYPE_NOT_IN; } + break; + case 51: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yymsp[-2].minor.yy168 = createNodeListNode(pCxt, yymsp[-1].minor.yy192); } + break; + case 53: /* boolean_value_expression ::= NOT boolean_primary */ +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy168, NULL); } + break; + case 54: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 55: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy168 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 60: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy168, yymsp[0].minor.yy168, NULL); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 63: /* table_primary ::= table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; + break; + case 64: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createRealTableNode(pCxt, &yymsp[-3].minor.yy241, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 65: /* table_primary ::= subquery alias_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createTempTableNode(pCxt, yymsp[-1].minor.yy168, &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; + break; + case 67: /* alias_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy241 = nil_token; } + break; + case 68: /* alias_opt ::= table_alias */ +{ PARSER_TRACE; yylhsminor.yy241 = yymsp[0].minor.yy241; } + yymsp[0].minor.yy241 = yylhsminor.yy241; + break; + case 69: /* alias_opt ::= AS table_alias */ +{ PARSER_TRACE; yymsp[-1].minor.yy241 = yymsp[0].minor.yy241; } + break; + case 72: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ PARSER_TRACE; yylhsminor.yy168 = createJoinTableNode(pCxt, yymsp[-4].minor.yy229, yymsp[-5].minor.yy168, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-5].minor.yy168 = yylhsminor.yy168; + break; + case 73: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy229 = JOIN_TYPE_INNER; } + break; + case 74: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ +{ + PARSER_TRACE; + yymsp[-8].minor.yy168 = createSelectStmt(pCxt, yymsp[-7].minor.yy209, yymsp[-6].minor.yy192, yymsp[-5].minor.yy168); + yymsp[-8].minor.yy168 = addWhereClause(pCxt, yymsp[-8].minor.yy168, yymsp[-4].minor.yy168); + yymsp[-8].minor.yy168 = addPartitionByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-3].minor.yy192); + yymsp[-8].minor.yy168 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy168, yymsp[-2].minor.yy168); + yymsp[-8].minor.yy168 = addGroupByClause(pCxt, yymsp[-8].minor.yy168, yymsp[-1].minor.yy192); + yymsp[-8].minor.yy168 = addHavingClause(pCxt, yymsp[-8].minor.yy168, yymsp[0].minor.yy168); + } + break; + case 75: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy209 = false; } + break; + case 76: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy209 = true; } + break; + case 77: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy209 = false; } + break; + case 78: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy192 = NULL; } + break; + case 79: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy192 = yymsp[0].minor.yy192; } + yymsp[0].minor.yy192 = yylhsminor.yy192; + break; + case 83: /* select_item ::= expression column_alias */ +{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, yymsp[-1].minor.yy168, &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy168 = yylhsminor.yy168; + break; + case 84: /* select_item ::= expression AS column_alias */ +{ PARSER_TRACE; yylhsminor.yy168 = setProjectionAlias(pCxt, yymsp[-2].minor.yy168, &yymsp[0].minor.yy241); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 85: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy168 = createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 86: /* where_clause_opt ::= */ + case 90: /* twindow_clause_opt ::= */ yytestcase(yyruleno==90); + case 95: /* sliding_opt ::= */ yytestcase(yyruleno==95); + case 97: /* fill_opt ::= */ yytestcase(yyruleno==97); + case 107: /* having_clause_opt ::= */ yytestcase(yyruleno==107); + case 115: /* slimit_clause_opt ::= */ yytestcase(yyruleno==115); + case 119: /* limit_clause_opt ::= */ yytestcase(yyruleno==119); +{ PARSER_TRACE; yymsp[1].minor.yy168 = NULL; } + break; + case 88: /* partition_by_clause_opt ::= */ + case 105: /* group_by_clause_opt ::= */ yytestcase(yyruleno==105); + case 113: /* order_by_clause_opt ::= */ yytestcase(yyruleno==113); +{ PARSER_TRACE; yymsp[1].minor.yy192 = NULL; } + break; + case 89: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 106: /* group_by_clause_opt ::= GROUP BY expression_list */ yytestcase(yyruleno==106); + case 114: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==114); +{ PARSER_TRACE; yymsp[-2].minor.yy192 = yymsp[0].minor.yy192; } + break; + case 91: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy168 = createSessionWindowNode(pCxt, yymsp[-3].minor.yy168, &yymsp[-1].minor.yy0); } + break; + case 92: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createStateWindowNode(pCxt, yymsp[-1].minor.yy168); } + break; + case 93: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-5].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy168, NULL, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } + break; + case 94: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-7].minor.yy168 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy168, yymsp[-3].minor.yy168, yymsp[-1].minor.yy168, yymsp[0].minor.yy168); } + break; + case 96: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy168 = yymsp[-1].minor.yy168; } + break; + case 98: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createFillNode(pCxt, yymsp[-1].minor.yy14, NULL); } + break; + case 99: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy168 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy192)); } + break; + case 100: /* fill_mode ::= NONE */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NONE; } + break; + case 101: /* fill_mode ::= PREV */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_PREV; } + break; + case 102: /* fill_mode ::= NULL */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NULL; } + break; + case 103: /* fill_mode ::= LINEAR */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_LINEAR; } + break; + case 104: /* fill_mode ::= NEXT */ +{ PARSER_TRACE; yymsp[0].minor.yy14 = FILL_MODE_NEXT; } + break; + case 109: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ +{ + PARSER_TRACE; + yylhsminor.yy168 = addOrderByClause(pCxt, yymsp[-3].minor.yy168, yymsp[-2].minor.yy192); + yylhsminor.yy168 = addSlimitClause(pCxt, yylhsminor.yy168, yymsp[-1].minor.yy168); + yylhsminor.yy168 = addLimitClause(pCxt, yylhsminor.yy168, yymsp[0].minor.yy168); + } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 111: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy168 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy168, yymsp[0].minor.yy168); } + yymsp[-3].minor.yy168 = yylhsminor.yy168; + break; + case 116: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 120: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==120); +{ PARSER_TRACE; yymsp[-1].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + break; + case 117: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 121: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==121); +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 118: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 122: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==122); +{ PARSER_TRACE; yymsp[-3].minor.yy168 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + break; + case 127: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy168 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy168, yymsp[-1].minor.yy10, yymsp[0].minor.yy177); } + yymsp[-2].minor.yy168 = yylhsminor.yy168; + break; + case 128: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy10 = ORDER_ASC; } + break; + case 129: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_ASC; } + break; + case 130: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy10 = ORDER_DESC; } + break; + case 131: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy177 = NULL_ORDER_DEFAULT; } + break; + case 132: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_FIRST; } + break; + case 133: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy177 = NULL_ORDER_LAST; } + break; + default: + break; +/********** End reduce actions ************************************************/ + }; + assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + NewParseCTX_STORE +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + NewParseTOKENTYPE yyminor /* The minor type of the error token */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ + + if(TOKEN.z) { + char msg[] = "syntax error near \"%s\""; + int32_t sqlLen = strlen(&TOKEN.z[0]); + + if (sqlLen + sizeof(msg)/sizeof(msg[0]) + 1 > pCxt->pQueryCxt->msgLen) { + char tmpstr[128] = {0}; + memcpy(tmpstr, &TOKEN.z[0], sizeof(tmpstr)/sizeof(tmpstr[0]) - 1); + sprintf(pCxt->pQueryCxt->pMsg, msg, tmpstr); + } else { + sprintf(pCxt->pQueryCxt->pMsg, msg, &TOKEN.z[0]); + } + } else { + sprintf(pCxt->pQueryCxt->pMsg, "Incomplete SQL statement"); + } + pCxt->valid = false; +/************ End %syntax_error code ******************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + NewParseCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + NewParseARG_FETCH + NewParseCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ + PARSER_COMPLETE; +/*********** End %parse_accept code *******************************************/ + NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ + NewParseCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "NewParseAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
    +**
  • A pointer to the parser (an opaque structure.) +**
  • The major token number. +**
  • The minor token number. +**
  • An option argument of a grammar-specified type. +**
+** +** Outputs: +** None. +*/ +void NewParse( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + NewParseTOKENTYPE yyminor /* The value for the token */ + NewParseARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + YYACTIONTYPE yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + NewParseCTX_FETCH + NewParseARG_STORE + + assert( yypParser->yytos!=0 ); +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + + yyact = yypParser->yytos->stateno; +#ifndef NDEBUG + if( yyTraceFILE ){ + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } + } +#endif + + do{ + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, + yyminor NewParseCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; + }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yytos->major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + }else{ + while( yypParser->yytos >= yypParser->yystack + && (yyact = yy_find_reduce_action( + yypParser->yytos->stateno, + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + ){ + yy_pop_parser_stack(yypParser); + } + if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + break; +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + } + break; +#endif + } + }while( yypParser->yytos>yypParser->yystack ); +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int NewParseFallback(int iToken){ +#ifdef YYFALLBACK + if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ + return yyFallback[iToken]; + } +#else + (void)iToken; +#endif + return 0; +} diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index c98f787f0b..3b79a2de92 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -249,7 +249,7 @@ void qDestroyQuery(SQueryNode* pQueryNode) { return; } - int32_t type = nodeType(pQueryNode); + int32_t type = queryNodeType(pQueryNode); if (type == TSDB_SQL_INSERT || type == TSDB_SQL_CREATE_TABLE) { SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode; taosArrayDestroy(pModifInfo->pDataBlocks); diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c new file mode 100644 index 0000000000..7182bcfedf --- /dev/null +++ b/source/libs/parser/src/parserImpl.c @@ -0,0 +1,581 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "parserImpl.h" + +#include "astCreateContext.h" +#include "parserInt.h" +#include "ttoken.h" + +typedef void* (*FMalloc)(size_t); +typedef void (*FFree)(void*); + +extern void* NewParseAlloc(FMalloc); +extern void NewParse(void*, int, SToken, void*); +extern void NewParseFree(void*, FFree); +extern void NewParseTrace(FILE*, char*); + +static uint32_t toNewTokenId(uint32_t tokenId) { +// #define 1 +// #define NEW_TK_AND 2 +// #define NEW_TK_UNION 3 +// #define NEW_TK_ALL 4 +// #define NEW_TK_MINUS 5 +// #define NEW_TK_EXCEPT 6 +// #define NEW_TK_INTERSECT 7 +// #define NEW_TK_NK_PLUS 8 +// #define NEW_TK_NK_MINUS 9 +// #define NEW_TK_NK_STAR 10 +// #define NEW_TK_NK_SLASH 11 +// #define NEW_TK_NK_REM 12 +// #define NEW_TK_SHOW 13 +// #define NEW_TK_DATABASES 14 +// #define NEW_TK_NK_INTEGER 15 +// #define NEW_TK_NK_FLOAT 16 +// #define NEW_TK_NK_STRING 17 +// #define NEW_TK_NK_BOOL 18 +// #define NEW_TK_TIMESTAMP 19 +// #define NEW_TK_NK_VARIABLE 20 +// #define NEW_TK_NK_COMMA 21 +// #define NEW_TK_NK_ID 22 +// #define NEW_TK_NK_LP 23 +// #define NEW_TK_NK_RP 24 +// #define NEW_TK_NK_DOT 25 +// #define NEW_TK_BETWEEN 26 +// #define NEW_TK_NOT 27 +// #define NEW_TK_IS 28 +// #define NEW_TK_NULL 29 +// #define NEW_TK_NK_LT 30 +// #define NEW_TK_NK_GT 31 +// #define NEW_TK_NK_LE 32 +// #define NEW_TK_NK_GE 33 +// #define NEW_TK_NK_NE 34 +// #define 35 +// #define NEW_TK_LIKE 36 +// #define NEW_TK_MATCH 37 +// #define NEW_TK_NMATCH 38 +// #define NEW_TK_IN 39 +// #define NEW_TK_FROM 40 +// #define NEW_TK_AS 41 +// #define NEW_TK_JOIN 42 +// #define NEW_TK_ON 43 +// #define NEW_TK_INNER 44 +// #define NEW_TK_SELECT 45 +// #define NEW_TK_DISTINCT 46 +// #define 47 +// #define NEW_TK_PARTITION 48 +// #define NEW_TK_BY 49 +// #define NEW_TK_SESSION 50 +// #define NEW_TK_STATE_WINDOW 51 +// #define NEW_TK_INTERVAL 52 +// #define NEW_TK_SLIDING 53 +// #define NEW_TK_FILL 54 +// #define NEW_TK_VALUE 55 +// #define NEW_TK_NONE 56 +// #define NEW_TK_PREV 57 +// #define NEW_TK_LINEAR 58 +// #define NEW_TK_NEXT 59 +// #define NEW_TK_GROUP 60 +// #define NEW_TK_HAVING 61 +// #define NEW_TK_ORDER 62 +// #define NEW_TK_SLIMIT 63 +// #define NEW_TK_SOFFSET 64 +// #define NEW_TK_LIMIT 65 +// #define NEW_TK_OFFSET 66 +// #define NEW_TK_NK_LR 67 +// #define NEW_TK_ASC 68 +// #define NEW_TK_DESC 69 +// #define NEW_TK_NULLS 70 +// #define NEW_TK_FIRST 71 +// #define NEW_TK_LAST 72 + + switch (tokenId) { + case TK_OR: + return NEW_TK_OR; + case TK_UNION: + return NEW_TK_UNION; + case TK_ALL: + return NEW_TK_ALL; + case TK_MINUS: + return NEW_TK_NK_MINUS; + case TK_PLUS: + return NEW_TK_NK_PLUS; + case TK_STAR: + return NEW_TK_NK_STAR; + case TK_SLASH: + return NEW_TK_NK_SLASH; + case TK_SHOW: + return NEW_TK_SHOW; + case TK_DATABASES: + return NEW_TK_DATABASES; + case TK_ID: + return NEW_TK_NK_ID; + case TK_LP: + return NEW_TK_NK_LP; + case TK_RP: + return NEW_TK_NK_RP; + case TK_COMMA: + return NEW_TK_NK_COMMA; + case TK_DOT: + return NEW_TK_NK_DOT; + case TK_EQ: + return NEW_TK_NK_EQ; + case TK_SELECT: + return NEW_TK_SELECT; + case TK_DISTINCT: + return NEW_TK_DISTINCT; + case TK_WHERE: + return NEW_TK_WHERE; + case TK_AS: + return NEW_TK_AS; + case TK_FROM: + return NEW_TK_FROM; + case TK_ORDER: + return NEW_TK_ORDER; + case TK_BY: + return NEW_TK_BY; + case TK_ASC: + return NEW_TK_ASC; + case TK_DESC: + return NEW_TK_DESC; + case TK_SPACE: + break; + default: + printf("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!tokenId = %d\n", tokenId); + } + return tokenId; +} + +static uint32_t getToken(const char* z, uint32_t* tokenId) { + uint32_t n = tGetToken(z, tokenId); + *tokenId = toNewTokenId(*tokenId); + return n; +} + +int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { + SAstCreateContext cxt; + createAstCreateContext(pParseCxt, &cxt); + void *pParser = NewParseAlloc(malloc); + int32_t i = 0; + while (1) { + SToken t0 = {0}; + // printf("===========================\n"); + if (cxt.pQueryCxt->pSql[i] == 0) { + NewParse(pParser, 0, t0, &cxt); + goto abort_parse; + } + // printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); + t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); + t0.z = (char *)(cxt.pQueryCxt->pSql + i); + // printf("token : %d %d [%s]\n", t0.type, t0.n, t0.z); + i += t0.n; + + switch (t0.type) { + case TK_SPACE: + case TK_COMMENT: { + break; + } + case TK_SEMI: { + NewParse(pParser, 0, t0, &cxt); + goto abort_parse; + } + + case TK_QUESTION: + case TK_ILLEGAL: { + snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unrecognized token: \"%s\"", t0.z); + cxt.valid = false; + goto abort_parse; + } + + case TK_HEX: + case TK_OCT: + case TK_BIN: { + snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unsupported token: \"%s\"", t0.z); + cxt.valid = false; + goto abort_parse; + } + + default: + NewParse(pParser, t0.type, t0, &cxt); + // NewParseTrace(stdout, ""); + if (!cxt.valid) { + goto abort_parse; + } + } + } + +abort_parse: + // printf("doParse completed.\n"); + NewParseFree(pParser, free); + destroyAstCreateContext(&cxt); + pQuery->pRoot = cxt.pRootNode; + return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; +} + +// typedef struct SNamespace { +// int16_t level; // todo for correlated subquery +// char dbName[TSDB_DB_NAME_LEN]; +// char tableAlias[TSDB_TABLE_NAME_LEN]; +// SHashObj* pColHash; // key is colname, value is index of STableMeta.schema +// STableMeta* pMeta; +// } SNamespace; + +typedef enum ESqlClause { + SQL_CLAUSE_FROM = 1, + SQL_CLAUSE_WHERE +} ESqlClause; + +typedef struct STranslateContext { + SParseContext* pParseCxt; + int32_t errCode; + SMsgBuf msgBuf; + SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode* + int32_t currLevel; + ESqlClause currClause; +} STranslateContext; + +static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); + +static char* getSyntaxErrFormat(int32_t errCode) { + switch (errCode) { + case TSDB_CODE_PARSER_INVALID_COLUMN: + return "Invalid column name : %s"; + case TSDB_CODE_PARSER_TABLE_NOT_EXIST: + return "Table does not exist : %s"; + case TSDB_CODE_PARSER_AMBIGUOUS_COLUMN: + return "Column ambiguously defined : %s"; + default: + return "Unknown error"; + } +} + +static int32_t generateSyntaxErrMsg(STranslateContext* pCxt, int32_t errCode, const char* additionalInfo) { + snprintf(pCxt->msgBuf.buf, pCxt->msgBuf.len, getSyntaxErrFormat(errCode), additionalInfo); + pCxt->errCode = errCode; + return errCode; +} + +static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { + size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); + if (currTotalLevel > pCxt->currLevel) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + taosArrayPush(pTables, &pTable); + } else { + do { + SArray* pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (pCxt->currLevel == currTotalLevel) { + taosArrayPush(pTables, &pTable); + } + taosArrayPush(pCxt->pNsLevel, &pTables); + ++currTotalLevel; + } while (currTotalLevel <= pCxt->currLevel); + } + return TSDB_CODE_SUCCESS; +} + +static SName* toName(int32_t acctId, const SRealTableNode* pRealTable, SName* pName) { + pName->type = TSDB_TABLE_NAME_T; + pName->acctId = acctId; + strcpy(pName->dbname, pRealTable->table.dbName); + strcpy(pName->tname, pRealTable->table.tableName); + return pName; +} + +static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { + int cmp = 0; + if ('\0' != pCol->dbName[0]) { + cmp = strcmp(pCol->dbName, pTable->dbName); + } else { + cmp = strcmp(currentDb, pTable->dbName); + } + if (0 == cmp) { + cmp = strcmp(pCol->tableAlias, pTable->tableAlias); + } + return (0 == cmp); +} + +static SNodeList* getProjectList(SNode* pNode) { + if (QUERY_NODE_SELECT_STMT == nodeType(pNode)) { + return ((SSelectStmt*)pNode)->pProjectionList; + } + return NULL; +} + +static void setColumnInfoBySchema(const STableNode* pTable, const SSchema* pColSchema, SColumnNode* pCol) { + strcpy(pCol->dbName, pTable->dbName); + strcpy(pCol->tableAlias, pTable->tableAlias); + strcpy(pCol->tableName, pTable->tableName); + strcpy(pCol->colName, pColSchema->name); + if ('\0' == pCol->node.aliasName[0]) { + strcpy(pCol->node.aliasName, pColSchema->name); + } + pCol->colId = pColSchema->colId; + pCol->colType = pColSchema->type; + pCol->node.resType.bytes = pColSchema->bytes; +} + +static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SColumnNode* pCol) { + pCol->pProjectRef = (SNode*)pExpr; + pExpr->pAssociationList = nodesListAppend(pExpr->pAssociationList, (SNode*)pCol); + strcpy(pCol->tableAlias, pTable->tableAlias); + strcpy(pCol->colName, pExpr->aliasName); + pCol->node.resType = pExpr->resType; +} + +static int32_t createColumnNodeByTable(const STableNode* pTable, SNodeList* pList) { + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + setColumnInfoBySchema(pTable, pMeta->schema + i, pCol); + nodesListAppend(pList, (SNode*)pCol); + } + } else { + SNodeList* pProjectList = getProjectList(((STempTableNode*)pTable)->pSubquery); + SNode* pNode; + FOREACH(pNode, pProjectList) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + setColumnInfoByExpr(pTable, (SExprNode*)pNode, pCol); + nodesListAppend(pList, (SNode*)pCol); + } + } +} + +static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { + bool found = false; + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { + setColumnInfoBySchema(pTable, pMeta->schema + i, pCol); + found = true; + break; + } + } + } else { + SNodeList* pProjectList = getProjectList(((STempTableNode*)pTable)->pSubquery); + SNode* pNode; + FOREACH(pNode, pProjectList) { + SExprNode* pExpr = (SExprNode*)pNode; + if (0 == strcmp(pCol->colName, pExpr->aliasName)) { + setColumnInfoByExpr(pTable, pExpr, pCol); + found = true; + break; + } + } + } + return found; +} + +static bool translateColumnWithPrefix(STranslateContext* pCxt, SColumnNode* pCol) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (belongTable(pCxt->pParseCxt->db, pCol, pTable)) { + if (findAndSetColumn(pCol, pTable)) { + break; + } + generateSyntaxErrMsg(pCxt, TSDB_CODE_PARSER_INVALID_COLUMN, pCol->colName); + return false; + } + } + return true; +} + +static bool translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNode* pCol) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + bool found = false; + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (findAndSetColumn(pCol, pTable)) { + if (found) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PARSER_AMBIGUOUS_COLUMN, pCol->colName); + return false; + } + found = true; + } + } + if (!found) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PARSER_INVALID_COLUMN, pCol->colName); + return false; + } + return true; +} + +static bool translateColumn(STranslateContext* pCxt, SColumnNode* pCol) { + if ('\0' != pCol->tableAlias[0]) { + return translateColumnWithPrefix(pCxt, pCol); + } + return translateColumnWithoutPrefix(pCxt, pCol); +} + +// check literal format +static bool translateValue(STranslateContext* pCxt, SValueNode* pVal) { + return true; +} + +static bool translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + return true; +} + +static bool translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { + return true; +} + +static bool doTranslateExpr(SNode* pNode, void* pContext) { + STranslateContext* pCxt = (STranslateContext*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + return translateColumn(pCxt, (SColumnNode*)pNode); + case QUERY_NODE_VALUE: + return translateValue(pCxt, (SValueNode*)pNode); + case QUERY_NODE_OPERATOR: + return translateOperator(pCxt, (SOperatorNode*)pNode); + case QUERY_NODE_FUNCTION: + return translateFunction(pCxt, (SFunctionNode*)pNode); + case QUERY_NODE_TEMP_TABLE: + return translateSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); + default: + break; + } + return true; +} + +static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { + nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt); + return pCxt->errCode; +} + +static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { + nodesWalkListPostOrder(pList, doTranslateExpr, pCxt); + return pCxt->errCode; +} + +static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pTable)) { + case QUERY_NODE_REAL_TABLE: { + SRealTableNode* pRealTable = (SRealTableNode*)pTable; + SName name; + code = catalogGetTableMeta(pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &(pCxt->pParseCxt->mgmtEpSet), + toName(pCxt->pParseCxt->acctId, pRealTable, &name), &(pRealTable->pMeta)); + if (TSDB_CODE_SUCCESS != code) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PARSER_TABLE_NOT_EXIST, pRealTable->table.tableName); + } + code = addNamespace(pCxt, pRealTable); + break; + } + case QUERY_NODE_TEMP_TABLE: { + STempTableNode* pTempTable = (STempTableNode*)pTable; + code = translateSubquery(pCxt, pTempTable->pSubquery); + if (TSDB_CODE_SUCCESS == code) { + code = addNamespace(pCxt, pTempTable); + } + break; + } + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTable = (SJoinTableNode*)pTable; + code = translateTable(pCxt, pJoinTable->pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = translateTable(pCxt, pJoinTable->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, pJoinTable->pOnCond); + } + break; + } + default: + break; + } + return code; +} + +static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { + pCxt->currClause = SQL_CLAUSE_FROM; + return translateTable(pCxt, pTable); +} + +static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect, bool* pIsSelectStar) { + if (NULL == pSelect->pProjectionList) { // select * ... + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + pSelect->pProjectionList = nodesMakeList(); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + createColumnNodeByTable(pTable, pSelect->pProjectionList); + } + *pIsSelectStar = true; + } else { + + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; + code = translateFrom(pCxt, pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, pSelect->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExprList(pCxt, pSelect->pGroupByList); + } + bool isSelectStar = false; + if (TSDB_CODE_SUCCESS == code) { + code = translateStar(pCxt, pSelect, &isSelectStar); + } + if (TSDB_CODE_SUCCESS == code && !isSelectStar) { + code = translateExprList(pCxt, pSelect->pProjectionList); + } + // printf("%s:%d code = %d\n", __FUNCTION__, __LINE__, code); + return code; +} + +static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pNode)) { + case QUERY_NODE_SELECT_STMT: + code = translateSelect(pCxt, (SSelectStmt*)pNode); + break; + default: + break; + } + return code; +} + +static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { + ++(pCxt->currLevel); + ESqlClause currClause = pCxt->currClause; + int32_t code = translateQuery(pCxt, pNode); + --(pCxt->currLevel); + pCxt->currClause = currClause; + return code; +} + +int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { + STranslateContext cxt = { + .pParseCxt = pParseCxt, + .errCode = TSDB_CODE_SUCCESS, + .msgBuf = { .buf = pParseCxt->pMsg, .len = pParseCxt->msgLen }, + .pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES), + .currLevel = 0, + .currClause = 0 + }; + return translateQuery(&cxt, pQuery->pRoot); +} diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 82f3fafe04..4fa27bbaeb 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1419,35 +1419,6 @@ bool isQueryWithLimit(SQueryStmtInfo* pQueryInfo) { return false; } -SVgroupsInfo* vgroupInfoClone(SVgroupsInfo *vgroupList) { - if (vgroupList == NULL) { - return NULL; - } - - size_t size = sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * vgroupList->numOfVgroups; - SVgroupsInfo* pNew = malloc(size); - if (pNew == NULL) { - return NULL; - } - - pNew->numOfVgroups = vgroupList->numOfVgroups; - - for(int32_t i = 0; i < vgroupList->numOfVgroups; ++i) { - SVgroupMsg* pNewVInfo = &pNew->vgroups[i]; - - SVgroupMsg* pvInfo = &vgroupList->vgroups[i]; - pNewVInfo->vgId = pvInfo->vgId; - pNewVInfo->numOfEps = pvInfo->numOfEps; - - for(int32_t j = 0; j < pvInfo->numOfEps; ++j) { - pNewVInfo->epAddr[j].port = pvInfo->epAddr[j].port; - tstrncpy(pNewVInfo->epAddr[j].fqdn, pvInfo->epAddr[j].fqdn, TSDB_FQDN_LEN); - } - } - - return pNew; -} - void* vgroupInfoClear(SVgroupsInfo *vgroupList) { if (vgroupList == NULL) { return NULL; @@ -1498,19 +1469,6 @@ STableMeta* tableMetaDup(const STableMeta* pTableMeta) { return p; } -SVgroupsInfo* vgroupsInfoDup(SVgroupsInfo* pVgroupsInfo) { - assert(pVgroupsInfo != NULL); - - size_t size = sizeof(SVgroupMsg) * pVgroupsInfo->numOfVgroups + sizeof(SVgroupsInfo); - SVgroupsInfo* pInfo = calloc(1, size); - pInfo->numOfVgroups = pVgroupsInfo->numOfVgroups; - for (int32_t m = 0; m < pVgroupsInfo->numOfVgroups; ++m) { - memcpy(&pInfo->vgroups[m], &pVgroupsInfo->vgroups[m], sizeof(SVgroupMsg)); - } - - return pInfo; -} - int32_t getNumOfOutput(SFieldInfo* pFieldInfo) { return pFieldInfo->numOfOutput; } diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index e722b873b6..84164c8c76 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -15,5 +15,5 @@ TARGET_INCLUDE_DIRECTORIES( TARGET_LINK_LIBRARIES( parserTest - PUBLIC os util common parser catalog transport gtest function planner qcom + PUBLIC os util common nodes parser catalog transport gtest function planner qcom ) diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index e2b8766700..00d64bd12a 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -15,6 +15,7 @@ #include "mockCatalogService.h" +#include "tep.h" #include #include #include @@ -39,7 +40,16 @@ public: virtual TableBuilder& setVgid(int16_t vgid) { schema()->vgId = vgid; - meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + + SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, }; + + vgroup.epset.eps[0] = (SEp){"dnode_1", 6030}; + vgroup.epset.eps[1] = (SEp){"dnode_2", 6030}; + vgroup.epset.eps[2] = (SEp){"dnode_3", 6030}; + vgroup.epset.inUse = 0; + vgroup.epset.numOfEps = 3; + + meta_->vgs.emplace_back(vgroup); return *this; } @@ -112,9 +122,7 @@ public: int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const { // todo vgInfo->vgId = 1; - vgInfo->numOfEps = 1; - vgInfo->epAddr[0].port = 6030; - strcpy(vgInfo->epAddr[0].fqdn, "node1"); + addEpIntoEpSet(&vgInfo->epset, "node1", 6030); return 0; } @@ -133,9 +141,16 @@ public: meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema.reset(table.release()); meta_[db][tbname]->schema->uid = id_++; - meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + + SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; + addEpIntoEpSet(&vgroup.epset, "dnode_1", 6030); + addEpIntoEpSet(&vgroup.epset, "dnode_2", 6030); + addEpIntoEpSet(&vgroup.epset, "dnode_3", 6030); + vgroup.epset.inUse = 0; + + meta_[db][tbname]->vgs.emplace_back(vgroup); // super table - meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_[db][stbname]->vgs.emplace_back(vgroup); } void showTables() const { diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp new file mode 100644 index 0000000000..973a6aff1e --- /dev/null +++ b/source/libs/parser/test/newParserTest.cpp @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include "parserImpl.h" + +using namespace std; +using namespace testing; + +class NewParserTest : public Test { +protected: + void setDatabase(const string& acctId, const string& db) { + acctId_ = acctId; + db_ = db; + } + + void bind(const char* sql) { + reset(); + cxt_.acctId = atoi(acctId_.c_str()); + cxt_.db = (char*) db_.c_str(); + strcpy(sqlBuf_, sql); + cxt_.sqlLen = strlen(sql); + sqlBuf_[cxt_.sqlLen] = '\0'; + cxt_.pSql = sqlBuf_; + + } + + bool run(int32_t parseCode = TSDB_CODE_SUCCESS, int32_t translateCode = TSDB_CODE_SUCCESS) { + int32_t code = doParse(&cxt_, &query_); + // cout << "doParse return " << code << endl; + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + return (TSDB_CODE_SUCCESS != parseCode); + } + if (TSDB_CODE_SUCCESS != parseCode) { + return false; + } + code = doTranslate(&cxt_, &query_); + // cout << "doTranslate return " << code << endl; + if (code != TSDB_CODE_SUCCESS) { + cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + return (TSDB_CODE_SUCCESS != translateCode); + } + if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { + string sql; + selectToSql(query_.pRoot, sql); + cout << "input sql : [" << cxt_.pSql << "]" << endl; + cout << "output sql : [" << sql << "]" << endl; + } + return (TSDB_CODE_SUCCESS == translateCode); + } + +private: + static const int max_err_len = 1024; + static const int max_sql_len = 1024 * 1024; + + void selectToSql(const SNode* node, string& sql) { + SSelectStmt* select = (SSelectStmt*)node; + sql.append("SELECT "); + if (select->isDistinct) { + sql.append("DISTINCT "); + } + if (nullptr == select->pProjectionList) { + sql.append("* "); + } else { + nodeListToSql(select->pProjectionList, sql); + sql.append(" "); + } + sql.append("FROM "); + tableToSql(select->pFromTable, sql); + if (nullptr != select->pWhere) { + sql.append(" WHERE "); + nodeToSql(select->pWhere, sql); + } + } + + void tableToSql(const SNode* node, string& sql) { + const STableNode* table = (const STableNode*)node; + switch (nodeType(node)) { + case QUERY_NODE_REAL_TABLE: { + SRealTableNode* realTable = (SRealTableNode*)table; + if ('\0' != realTable->table.dbName[0]) { + sql.append(realTable->table.dbName); + sql.append("."); + } + sql.append(realTable->table.tableName); + break; + } + case QUERY_NODE_TEMP_TABLE: { + STempTableNode* tempTable = (STempTableNode*)table; + sql.append("("); + selectToSql(tempTable->pSubquery, sql); + sql.append(") "); + sql.append(tempTable->table.tableAlias); + break; + } + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* joinTable = (SJoinTableNode*)table; + tableToSql(joinTable->pLeft, sql); + sql.append(" JOIN "); + tableToSql(joinTable->pRight, sql); + if (nullptr != joinTable->pOnCond) { + sql.append(" ON "); + nodeToSql(joinTable->pOnCond, sql); + } + break; + } + default: + break; + } + } + + string opTypeToSql(EOperatorType type) { + switch (type) { + case OP_TYPE_ADD: + return " + "; + case OP_TYPE_SUB: + return " - "; + case OP_TYPE_MULTI: + case OP_TYPE_DIV: + case OP_TYPE_MOD: + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + case OP_TYPE_EQUAL: + return " = "; + case OP_TYPE_NOT_EQUAL: + case OP_TYPE_IN: + case OP_TYPE_NOT_IN: + case OP_TYPE_LIKE: + case OP_TYPE_NOT_LIKE: + case OP_TYPE_MATCH: + case OP_TYPE_NMATCH: + case OP_TYPE_JSON_GET_VALUE: + case OP_TYPE_JSON_CONTAINS: + default: + break; + } + return " unknown operator "; + } + + void nodeToSql(const SNode* node, string& sql) { + if (nullptr == node) { + return; + } + + switch (nodeType(node)) { + case QUERY_NODE_COLUMN: { + SColumnNode* pCol = (SColumnNode*)node; + if ('\0' != pCol->dbName[0]) { + sql.append(pCol->dbName); + sql.append("."); + } + if ('\0' != pCol->tableAlias[0]) { + sql.append(pCol->tableAlias); + sql.append("."); + } + sql.append(pCol->colName); + break; + } + case QUERY_NODE_VALUE: + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOp = (SOperatorNode*)node; + nodeToSql(pOp->pLeft, sql); + sql.append(opTypeToSql(pOp->opType)); + nodeToSql(pOp->pRight, sql); + break; + } + default: + break; + } + } + + void nodeListToSql(const SNodeList* nodelist, string& sql, const string& seq = ",") { + SNode* node = nullptr; + bool firstNode = true; + FOREACH(node, nodelist) { + if (!firstNode) { + sql.append(", "); + } + firstNode = false; + nodeToSql(node, sql); + } + } + + void reset() { + memset(&cxt_, 0, sizeof(cxt_)); + memset(errMagBuf_, 0, max_err_len); + cxt_.pMsg = errMagBuf_; + cxt_.msgLen = max_err_len; + } + + string acctId_; + string db_; + char errMagBuf_[max_err_len]; + char sqlBuf_[max_sql_len]; + SParseContext cxt_; + SQuery query_; +}; + +// SELECT * FROM t1 +TEST_F(NewParserTest, selectStar) { + setDatabase("root", "test"); + + bind("SELECT * FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT * FROM test.t1"); + ASSERT_TRUE(run()); + + bind("SELECT ts, c1 FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT ts, t.c1 FROM (SELECT * FROM t1) t"); + ASSERT_TRUE(run()); + + bind("SELECT * FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); + ASSERT_TRUE(run()); +} + +TEST_F(NewParserTest, syntaxError) { + setDatabase("root", "test"); + + bind("SELECTT * FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *, * FROM test.t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT * FROM test.t1 t WHER"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); +} + +TEST_F(NewParserTest, semanticError) { + setDatabase("root", "test"); + + bind("SELECT * FROM t10"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + + bind("SELECT c1, c3 FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); + + bind("SELECT c2 FROM t1 tt1, t1 tt2 WHERE tt1.c1 = tt2.c1"); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FAILED)); +} diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index de62f4a2ef..a5b1fd6f6b 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -74,7 +74,7 @@ int32_t createSelectPlan(const SQueryStmtInfo* pSelect, SQueryPlanNode** pQueryP } int32_t createQueryPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { - switch (nodeType(pNode)) { + switch (queryNodeType(pNode)) { case TSDB_SQL_SELECT: { return createSelectPlan((const SQueryStmtInfo*)pNode, pQueryPlan); } @@ -395,7 +395,7 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { } static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - int32_t type = nodeType(pQueryNode); + int32_t type = queryNodeType(pQueryNode); if (type == QNODE_MODIFY) { SDataPayloadInfo* pInfo = pQueryNode->pExtInfo; diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 12c0d0780b..20e8378c26 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -216,7 +216,8 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } else if (needSeqScan(pPlanNode)) { return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan); } - return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan); + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; + return createUserTableScanNode(pPlanNode, pTable, type); } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { @@ -251,24 +252,9 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { return subplan; } -static void vgroupInfoToEpSet(const SVgroupInfo* vg, SQueryNodeAddr* execNode) { - execNode->nodeId = vg->vgId; - execNode->inUse = vg->inUse; - execNode->numOfEps = vg->numOfEps; - for (int8_t i = 0; i < vg->numOfEps; ++i) { - execNode->epAddr[i] = vg->epAddr[i]; - } - return; -} - -static void vgroupMsgToEpSet(const SVgroupMsg* vg, SQueryNodeAddr* execNode) { - execNode->nodeId = vg->vgId; - execNode->inUse = 0; // todo - execNode->numOfEps = vg->numOfEps; - for (int8_t i = 0; i < vg->numOfEps; ++i) { - execNode->epAddr[i] = vg->epAddr[i]; - } - return; +static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) { + pNodeAddr->nodeId = vg->vgId; + pNodeAddr->epset = vg->epset; } static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) { @@ -277,7 +263,8 @@ static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNod STORE_CURRENT_SUBPLAN(pCxt); SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_SCAN); subplan->msgType = TDMT_VND_QUERY; - vgroupMsgToEpSet(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode); + + vgroupInfoToNodeAddr(&(pTableInfo->pMeta->vgroupList->vgroups[i]), &subplan->execNode); subplan->pNode = createMultiTableScanNode(pPlanNode, pTableInfo); subplan->pDataSink = createDataDispatcher(pCxt, pPlanNode, subplan->pNode); RECOVERY_CURRENT_SUBPLAN(pCxt); @@ -297,11 +284,12 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) { return (TSDB_SUPER_TABLE == pTable->pMeta->pTableMeta->tableType); } -static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, SSubplan* subplan) { - vgroupMsgToEpSet(&(pTable->pMeta->vgroupList->vgroups[0]), &subplan->execNode); - - int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan; - return createUserTableScanNode(pPlanNode, pTable, type); +// TODO: the SVgroupInfo index +static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) { + SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList; + vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode); + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; + return createUserTableScanNode(pPlanNode, pTableInfo, type); } static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { @@ -309,6 +297,7 @@ static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNo if (needMultiNodeScan(pTable)) { return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } + return createSingleTableScanNode(pPlanNode, pTable, pCxt->pCurrentSubplan); } @@ -374,7 +363,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY); SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i); - vgroupInfoToEpSet(&blocks->vg, &subplan->execNode); + subplan->execNode.epset = blocks->vg.epset; subplan->pDataSink = createDataInserter(pCxt, blocks, NULL); subplan->pNode = NULL; subplan->type = QUERY_TYPE_MODIFY; @@ -399,6 +388,33 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { // todo deal subquery } +static void postCreateDag(SQueryPlanNode* pQueryNode, SQueryDag* pDag, SArray* pNodeList) { + // The exchange operator is not necessary, in case of the stream scan. + // Here we need to remove it from the DAG. + if (pQueryNode->info.type == QNODE_STREAMSCAN) { + SArray* pRootLevel = taosArrayGetP(pDag->pSubplans, 0); + SSubplan *pSubplan = taosArrayGetP(pRootLevel, 0); + + if (pSubplan->pNode->info.type == OP_Exchange) { + ASSERT(taosArrayGetSize(pRootLevel) == 1); + + taosArrayRemove(pDag->pSubplans, 0); + // And then update the number of the subplans. + pDag->numOfSubplans -= 1; + } + } else { + // Traverse the dag again to acquire the execution node. + if (pNodeList != NULL) { + SArray** pSubLevel = taosArrayGetLast(pDag->pSubplans); + size_t num = taosArrayGetSize(*pSubLevel); + for (int32_t j = 0; j < num; ++j) { + SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); + taosArrayPush(pNodeList, &pPlan->execNode); + } + } + } +} + int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId) { TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { @@ -420,16 +436,7 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_FAILED; } END_TRY - // traverse the dag again to acquire the execution node. - if (pNodeList != NULL) { - SArray** pSubLevel = taosArrayGetLast((*pDag)->pSubplans); - size_t num = taosArrayGetSize(*pSubLevel); - for (int32_t j = 0; j < num; ++j) { - SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); - taosArrayPush(pNodeList, &pPlan->execNode); - } - } - + postCreateDag(pQueryNode, *pDag, pNodeList); return TSDB_CODE_SUCCESS; } @@ -461,7 +468,7 @@ static void destroyDataSinkNode(SDataSink* pSinkNode) { return; } - if (nodeType(pSinkNode) == DSINK_Dispatch) { + if (queryNodeType(pSinkNode) == DSINK_Dispatch) { SDataDispatcher* pDdSink = (SDataDispatcher*)pSinkNode; tfree(pDdSink->sink.schema.pSchema); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 4fbec534f6..e367f2e74b 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -88,7 +88,6 @@ static const char* jkPnodeType = "Type"; static int32_t getPnodeTypeSize(cJSON* json) { switch (getNumber(json, jkPnodeType)) { case OP_StreamScan: - case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return sizeof(STableScanPhyNode); @@ -736,7 +735,7 @@ static const char* jkEpAddrFqdn = "Fqdn"; static const char* jkEpAddrPort = "Port"; static bool epAddrToJson(const void* obj, cJSON* json) { - const SEpAddr* ep = (const SEpAddr*)obj; + const SEp* ep = (const SEp*)obj; bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn); if (res) { res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port); @@ -745,7 +744,7 @@ static bool epAddrToJson(const void* obj, cJSON* json) { } static bool epAddrFromJson(const cJSON* json, void* obj) { - SEpAddr* ep = (SEpAddr*)obj; + SEp* ep = (SEp*)obj; copyString(json, jkEpAddrFqdn, ep->fqdn); ep->port = getNumber(json, jkEpAddrPort); return true; @@ -763,11 +762,11 @@ static bool queryNodeAddrToJson(const void* obj, cJSON* json) { bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId); if (res) { - res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->inUse); + res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epset.inUse); } if (res) { - res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epAddr, sizeof(SEpAddr), pAddr->numOfEps); + res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epset.eps, sizeof(SEp), pAddr->epset.numOfEps); } return res; } @@ -776,11 +775,11 @@ static bool queryNodeAddrFromJson(const cJSON* json, void* obj) { SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj; pAddr->nodeId = getNumber(json, jkNodeAddrId); - pAddr->inUse = getNumber(json, jkNodeAddrInUse); + pAddr->epset.inUse = getNumber(json, jkNodeAddrInUse); int32_t numOfEps = 0; - bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epAddr, sizeof(SEpAddr), &numOfEps); - pAddr->numOfEps = numOfEps; + bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epset.eps, sizeof(SEp), &numOfEps); + pAddr->epset.numOfEps = numOfEps; return res; } @@ -831,7 +830,6 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { const SPhyNode* phyNode = (const SPhyNode*)obj; switch (phyNode->info.type) { case OP_StreamScan: - case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return tableScanNodeToJson(obj, json); @@ -869,7 +867,6 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* phyNode = (SPhyNode*)obj; switch (phyNode->info.type) { - case OP_TableScan: case OP_StreamScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: @@ -1125,8 +1122,6 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { *str = cJSON_Print(json); cJSON_Delete(json); -// printf("====Physical plan:====\n"); -// printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index d546925c5f..e6b7eaca7f 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -40,7 +40,8 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, uint64_t requestId) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pResSchema, int32_t* numOfCols, SArray* pNodeList, + uint64_t requestId) { SQueryPlanNode* pLogicPlan; int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { @@ -49,9 +50,10 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(pLogicPlan, &str); -// printf("%s\n", str); + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + qDebug("reqId:0x%"PRIx64": %s", requestId, str); + tfree(str); } code = optimizeQueryPlan(pLogicPlan); diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index edf5fa5a81..f4bdf57572 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -124,12 +124,10 @@ private: } void copyStorageMeta(SVgroupsInfo** dst, const std::vector& src) { - *dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupMsg) * src.size()); + *dst = (SVgroupsInfo*)myCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * src.size()); (*dst)->numOfVgroups = src.size(); for (int32_t i = 0; i < src.size(); ++i) { - (*dst)->vgroups[i].vgId = src[i].vgId; - (*dst)->vgroups[i].numOfEps = src[i].numOfEps; - memcpy((*dst)->vgroups[i].epAddr, src[i].epAddr, src[i].numOfEps); + (*dst)->vgroups[i] = src[i]; } } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index cfe2ed96ad..9f678b4528 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -128,7 +128,7 @@ int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransp if (NULL == pMsg) { qError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return -1; + return terrno; } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 52e632ffbb..e7b3d08bc5 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -45,11 +45,11 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 bMsg->header.vgId = htonl(bInput->vgId); - if (bInput->dbName) { - tstrncpy(bMsg->dbFname, bInput->dbName, tListLen(bMsg->dbFname)); + if (bInput->dbFName) { + tstrncpy(bMsg->dbFName, bInput->dbFName, tListLen(bMsg->dbFName)); } - tstrncpy(bMsg->tableFname, bInput->tableFullName, tListLen(bMsg->tableFname)); + tstrncpy(bMsg->tbName, bInput->tbName, tListLen(bMsg->tbName)); *msgLen = (int32_t)sizeof(*bMsg); return TSDB_CODE_SUCCESS; @@ -113,12 +113,19 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - pOut->dbVgroup.vgVersion = pRsp->vgVersion; - pOut->dbVgroup.hashMethod = pRsp->hashMethod; - pOut->dbVgroup.dbId = pRsp->uid; - pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == pOut->dbVgroup.vgInfo) { - qError("hash init[%d] failed", pRsp->vgNum); + pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo)); + if (NULL == pOut->dbVgroup) { + qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pOut->dbVgroup->vgVersion = pRsp->vgVersion; + pOut->dbVgroup->hashMethod = pRsp->hashMethod; + pOut->dbVgroup->dbId = pRsp->uid; + pOut->dbVgroup->vgHash = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pOut->dbVgroup->vgHash) { + qError("taosHashInit %d failed", pRsp->vgNum); + tfree(pOut->dbVgroup); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -127,12 +134,12 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { pRsp->vgroupInfo[i].hashBegin = ntohl(pRsp->vgroupInfo[i].hashBegin); pRsp->vgroupInfo[i].hashEnd = ntohl(pRsp->vgroupInfo[i].hashEnd); - for (int32_t n = 0; n < pRsp->vgroupInfo[i].numOfEps; ++n) { - pRsp->vgroupInfo[i].epAddr[n].port = ntohs(pRsp->vgroupInfo[i].epAddr[n].port); + for (int32_t n = 0; n < pRsp->vgroupInfo[i].epset.numOfEps; ++n) { + pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port); } - if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { - qError("hash push failed"); + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { + qError("taosHashPut failed"); goto _return; } } @@ -142,8 +149,10 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return code; _return: + if (pOut) { - tfree(pOut->dbVgroup.vgInfo); + taosHashCleanup(pOut->dbVgroup->vgHash); + tfree(pOut->dbVgroup); } return code; @@ -248,16 +257,13 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_VALUE; } + strcpy(pOut->dbFName, pMetaMsg->dbFName); + if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { SET_META_TYPE_BOTH_TABLE(pOut->metaType); - if (pMetaMsg->dbFname[0]) { - snprintf(pOut->ctbFname, sizeof(pOut->ctbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); - snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->stbFname); - } else { - memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname)); - memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname)); - } + strcpy(pOut->ctbName, pMetaMsg->tbName); + strcpy(pOut->tbName, pMetaMsg->stbName); pOut->ctbMeta.vgId = pMetaMsg->vgId; pOut->ctbMeta.tableType = pMetaMsg->tableType; @@ -268,11 +274,7 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { } else { SET_META_TYPE_TABLE(pOut->metaType); - if (pMetaMsg->dbFname[0]) { - snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); - } else { - memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); - } + strcpy(pOut->tbName, pMetaMsg->tbName); code = queryCreateTableMetaFromMsg(pMetaMsg, (pMetaMsg->tableType == TSDB_SUPER_TABLE), &pOut->tbMeta); } @@ -291,4 +293,4 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index d07e20d5c1..de2940846c 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -137,6 +137,7 @@ typedef struct SQWorkerMgmt { SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx void *nodeObj; putReqToQueryQFp putToQueueFp; + sendReqToDnodeFp sendReqFp; } SQWorkerMgmt; #define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId @@ -181,9 +182,9 @@ typedef struct SQWorkerMgmt { #define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) #define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index c28e30333c..cebe8178d9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -479,7 +479,6 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { if (NULL == pRes) { QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); - dsEndPut(sinkHandle, useconds); if (TASK_TYPE_TEMP == ctx->taskType) { @@ -493,6 +492,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { break; } + ASSERT(pRes->info.rows > 0); + SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); if (code) { @@ -560,7 +561,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); - QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); + QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); *rspMsg = rsp; *dataLen = 0; @@ -572,7 +573,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_TASK_DLOG("no res data in sink, need response later, queryEnd:%d", queryEnd); return TSDB_CODE_SUCCESS; - } + } // Got data from sink @@ -1100,8 +1101,6 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { atomic_store_8(&ctx->queryInQueue, 0); atomic_store_8(&ctx->queryContinue, 0); - DataSinkHandle sinkHandle = ctx->sinkHandle; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd)); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { @@ -1313,12 +1312,13 @@ _return: QW_RET(code); } -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) { - if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp) { +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, + putReqToQueryQFp fp1, sendReqToDnodeFp fp2) { + if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp1 || NULL == fp2) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } - + SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); @@ -1361,7 +1361,8 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->nodeType = nodeType; mgmt->nodeId = nodeId; mgmt->nodeObj = nodeObj; - mgmt->putToQueueFp = fp; + mgmt->putToQueueFp = fp1; + mgmt->sendReqFp = fp2; *qWorkerMgmt = mgmt; diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 56c882f404..368a590918 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -282,18 +282,21 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); + msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->contentLen = ntohl(msg->contentLen); - + msg->taskId = be64toh(msg->taskId); + msg->phyLen = ntohl(msg->phyLen); + msg->sqlLen = ntohl(msg->sqlLen); + uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; - SQWMsg qwMsg = {.node = node, .msg = msg->msg, .msgLen = msg->contentLen, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connection = pMsg}; - QW_SCH_TASK_DLOG("processQuery start, node:%p", node); + char* sql = strndup(msg->msg, msg->sqlLen); + QW_SCH_TASK_DLOG("processQuery start, node:%p, sql:%s", node, sql); + tfree(sql); QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); @@ -421,6 +424,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } +int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { + qProcessFetchRsp(NULL, pMsg, NULL); + return TSDB_CODE_SUCCESS; +} + int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index b9d84b25aa..38890150ee 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -113,7 +113,8 @@ void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { qwtqueryMsg.queryId = htobe64(atomic_add_fetch_64(&qwtTestQueryId, 1)); qwtqueryMsg.sId = htobe64(1); qwtqueryMsg.taskId = htobe64(1); - qwtqueryMsg.contentLen = htonl(100); + qwtqueryMsg.phyLen = htonl(100); + qwtqueryMsg.sqlLen = 0; queryRpc->msgType = TDMT_VND_QUERY; queryRpc->pCont = &qwtqueryMsg; queryRpc->contLen = sizeof(SSubQueryMsg) + 100; @@ -1081,7 +1082,7 @@ TEST(rcTest, shortExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1162,7 +1163,7 @@ TEST(rcTest, longExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 1000000; @@ -1245,7 +1246,7 @@ TEST(rcTest, shortExeclongDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 2ace1c66f5..94c833307f 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -128,10 +128,11 @@ typedef struct SSchJob { int32_t errCode; void *res; //TODO free it or not int32_t resNumOfRows; + const char *sql; SQueryProfileSummary summary; } SSchJob; -#define SCH_TASK_READY_TO_LUNCH(task) (atomic_load_32(&(task)->childReady) >= taosArrayGetSize((task)->children)) +#define SCH_TASK_READY_TO_LUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) @@ -150,11 +151,11 @@ typedef struct SSchJob { #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 934b222f64..1b6d871274 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -345,7 +345,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { } pLevel = taosArrayGet(pJob->levels, i); - pLevel->level = i; plans = taosArrayGetP(pDag->pSubplans, i); @@ -375,7 +374,7 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SSchTask task = {0}; SSchTask *pTask = &task; - + SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); void *p = taosArrayPush(pLevel->subTasks, &task); @@ -388,8 +387,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - - SCH_TASK_DLOG("task initialized, level:%d", pLevel->level); } SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); @@ -417,13 +414,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (pTask->plan->execNode.numOfEps > 0) { + if (pTask->plan->execNode.epset.numOfEps > 0) { if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) { SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.numOfEps); + SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epset.numOfEps); return TSDB_CODE_SUCCESS; } @@ -446,7 +443,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } if (addNum <= 0) { - SCH_TASK_ELOG("no available execNode as candidate addr, nodeNum:%d", nodeNum); + SCH_TASK_ELOG("no available execNode as candidates, nodeNum:%d", nodeNum); return TSDB_CODE_QRY_INVALID_INPUT; } @@ -716,8 +713,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t code = 0; SSchTask *pErrTask = pTask; - SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); - code = schMoveTaskToSuccList(pJob, pTask, &moved); if (code && moved) { SCH_ERR_RET(code); @@ -739,7 +734,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); - return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum); @@ -779,14 +773,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); pErrTask = par; - atomic_add_fetch_32(&par->childReady, 1); + int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); SDownstreamSource source = {.taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); - if (SCH_TASK_READY_TO_LUNCH(par)) { + if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { SCH_ERR_RET(schLaunchTask(pJob, par)); } } @@ -1031,50 +1025,33 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t pMsgSendInfo->fp = fp; int64_t transporterId = 0; - code = asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo); if (code) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); SCH_ERR_JRET(code); } - qDebug("QID:%"PRIx64 ",TID:%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); - + qDebug("QID:0x%"PRIx64 ",TID:0x%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: tfree(param); tfree(pMsgSendInfo); - SCH_RET(code); } -void schConvertAddrToEpSet(SQueryNodeAddr *addr, SEpSet *epSet) { - epSet->inUse = addr->inUse; - epSet->numOfEps = addr->numOfEps; - - for (int8_t i = 0; i < epSet->numOfEps; ++i) { - strncpy(epSet->fqdn[i], addr->epAddr[i].fqdn, sizeof(addr->epAddr[i].fqdn)); - epSet->port[i] = addr->epAddr[i].port; - } -} - int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; - SEpSet epSet; - if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); - isCandidateAddr = true; } - schConvertAddrToEpSet(addr, &epSet); - + SEpSet epSet = addr->epset; + switch (msgType) { case TDMT_VND_CREATE_TABLE: case TDMT_VND_SUBMIT: { @@ -1090,7 +1067,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_QUERY: { - msgSize = sizeof(SSubQueryMsg) + pTask->msgLen; + uint32_t len = strlen(pJob->sql); + + msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); @@ -1098,15 +1077,16 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } SSubQueryMsg *pMsg = msg; - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->taskType = TASK_TYPE_TEMP; - pMsg->contentLen = htonl(pTask->msgLen); - memcpy(pMsg->msg, pTask->msg, pTask->msgLen); + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(pJob->queryId); + pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskType = TASK_TYPE_TEMP; + pMsg->phyLen = htonl(pTask->msgLen); + pMsg->sqlLen = htonl(len); + + memcpy(pMsg->msg, pJob->sql, len); + memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); break; } @@ -1206,20 +1186,19 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { code = atomic_load_32(&pJob->errCode); SCH_ERR_RET(code); - SCH_RET(TSDB_CODE_SCH_STATUS_ERROR); } SSubplan *plan = pTask->plan; - if (NULL == pTask->msg) { + if (NULL == pTask->msg) { // TODO add more detailed reason for failure code = qSubPlanToString(plan, &pTask->msg, &pTask->msgLen); if (TSDB_CODE_SUCCESS != code || NULL == pTask->msg || pTask->msgLen <= 0) { - SCH_TASK_ELOG("subplanToString error, code:%x, msg:%p, len:%d", code, pTask->msg, pTask->msgLen); + SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); + } else { + SCH_TASK_DLOG("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } - -// printf("physical plan:%s\n", pTask->msg); } SCH_ERR_JRET(schSetTaskCandidateAddrs(pJob, pTask)); @@ -1227,18 +1206,14 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { // NOTE: race condition: the task should be put into the hash table before send msg to server if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask)); - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, plan->msgType)); - return TSDB_CODE_SUCCESS; _return: - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, code)); - SCH_RET(code); } @@ -1297,10 +1272,10 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { +static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, const char* sql, bool syncSchedule) { qDebug("QID:0x%"PRIx64" job started", pDag->queryId); - if (pNodeList && taosArrayGetSize(pNodeList) <= 0) { + if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { qDebug("QID:0x%"PRIx64" input exec nodeList is empty", pDag->queryId); } @@ -1313,6 +1288,7 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; + pJob->sql = sql; if (pNodeList != NULL) { pJob->nodeList = taosArrayDup(pNodeList); @@ -1352,7 +1328,6 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru } pJob->status = JOB_TASK_STATUS_NOT_START; - SCH_ERR_JRET(schLaunchJob(pJob)); *(SSchJob **)job = pJob; @@ -1363,15 +1338,12 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru } SCH_JOB_DLOG("job exec done, job status:%d", SCH_GET_JOB_STATUS(pJob)); - return TSDB_CODE_SUCCESS; _return: *(SSchJob **)job = NULL; - schedulerFreeJob(pJob); - SCH_RET(code); } @@ -1415,34 +1387,29 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = NULL; - - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, true)); - - job = *pJob; - - pRes->code = atomic_load_32(&job->errCode); - pRes->numOfRows = job->resNumOfRows; + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + pRes->code = atomic_load_32(&(*pJob)->errCode); + pRes->numOfRows = (*pJob)->resNumOfRows; return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); return TSDB_CODE_SUCCESS; } int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { - if (NULL == pDag || pDag->numOfSubplans <= 0 || taosArrayGetSize(pDag->pSubplans) <= 0) { + if (NULL == pDag || pDag->numOfSubplans <= 0 || taosArrayGetSize(pDag->pSubplans) == 0) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1472,7 +1439,6 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { for (int32_t i = 0; i < taskNum; ++i) { SSubplan *plan = taosArrayGetP(plans, i); - tInfo.addr = plan->execNode; code = qSubPlanToString(plan, &msg, &msgLen); @@ -1488,16 +1454,16 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { } SSubQueryMsg* pMsg = calloc(1, msgSize); - /*SSubQueryMsg *pMsg = (SSubQueryMsg*) msg;*/ - memcpy(pMsg->msg, msg, msgLen); pMsg->header.vgId = tInfo.addr.nodeId; - pMsg->sId = schMgmt.sId; - pMsg->queryId = plan->id.queryId; - pMsg->taskId = schGenUUID(); + pMsg->sId = schMgmt.sId; + pMsg->queryId = plan->id.queryId; + pMsg->taskId = schGenUUID(); pMsg->taskType = TASK_TYPE_PERSISTENT; - pMsg->contentLen = msgLen; + pMsg->phyLen = msgLen; + pMsg->sqlLen = 0; + memcpy(pMsg->msg, msg, msgLen); /*memcpy(pMsg->msg, ((SSubQueryMsg*)msg)->msg, msgLen);*/ tInfo.msg = pMsg; @@ -1530,7 +1496,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgSize = src->msg->contentLen + sizeof(*src->msg); + int32_t msgSize = src->msg->phyLen + sizeof(*src->msg); STaskInfo info = {0}; info.addr = src->addr; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 58714c51cc..2c75683ecb 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -29,7 +29,6 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wliteral-suffix" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" #pragma GCC diagnostic ignored "-Wsign-compare" @@ -92,11 +91,11 @@ void schtBuildQueryDag(SQueryDag *dag) { scanPlan->id.templateId = 0x0000000000000002; scanPlan->id.subplanId = 0x0000000000000003; scanPlan->type = QUERY_TYPE_SCAN; - scanPlan->execNode.numOfEps = 1; + scanPlan->execNode.nodeId = 1; - scanPlan->execNode.inUse = 0; - scanPlan->execNode.epAddr[0].port = 6030; - strcpy(scanPlan->execNode.epAddr[0].fqdn, "ep0"); + scanPlan->execNode.epset.inUse = 0; + addEpIntoEpSet(&scanPlan->execNode.epset, "ep0", 6030); + scanPlan->pChildren = NULL; scanPlan->level = 1; scanPlan->pParents = taosArrayInit(1, POINTER_BYTES); @@ -108,7 +107,8 @@ void schtBuildQueryDag(SQueryDag *dag) { mergePlan->id.subplanId = 0x5555555555; mergePlan->type = QUERY_TYPE_MERGE; mergePlan->level = 0; - mergePlan->execNode.numOfEps = 0; + mergePlan->execNode.epset.numOfEps = 0; + mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); mergePlan->pParents = NULL; mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); @@ -144,11 +144,11 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[0].id.subplanId = 0x0000000000000004; insertPlan[0].type = QUERY_TYPE_MODIFY; insertPlan[0].level = 0; - insertPlan[0].execNode.numOfEps = 1; + insertPlan[0].execNode.nodeId = 1; - insertPlan[0].execNode.inUse = 0; - insertPlan[0].execNode.epAddr[0].port = 6030; - strcpy(insertPlan[0].execNode.epAddr[0].fqdn, "ep0"); + insertPlan[0].execNode.epset.inUse = 0; + addEpIntoEpSet(&insertPlan[0].execNode.epset, "ep0", 6030); + insertPlan[0].pChildren = NULL; insertPlan[0].pParents = NULL; insertPlan[0].pNode = NULL; @@ -160,11 +160,11 @@ void schtBuildInsertDag(SQueryDag *dag) { insertPlan[1].id.subplanId = 0x0000000000000005; insertPlan[1].type = QUERY_TYPE_MODIFY; insertPlan[1].level = 0; - insertPlan[1].execNode.numOfEps = 1; + insertPlan[1].execNode.nodeId = 1; - insertPlan[1].execNode.inUse = 1; - insertPlan[1].execNode.epAddr[0].port = 6030; - strcpy(insertPlan[1].execNode.epAddr[0].fqdn, "ep1"); + insertPlan[1].execNode.epset.inUse = 0; + addEpIntoEpSet(&insertPlan[1].execNode.epset, "ep0", 6030); + insertPlan[1].pChildren = NULL; insertPlan[1].pParents = NULL; insertPlan[1].pNode = NULL; @@ -371,14 +371,14 @@ void* schtRunJobThread(void *aa) { while (!schtTestStop) { schtBuildQueryDag(&dag); - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); - SEpAddr qnodeAddr = {0}; + SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &job); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); assert(code == 0); execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); @@ -523,9 +523,9 @@ TEST(queryTest, normalCase) { schtInitLogFile(); - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); - SEpAddr qnodeAddr = {0}; + SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); @@ -539,7 +539,7 @@ TEST(queryTest, normalCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &pJob); ASSERT_EQ(code, 0); SSchJob *job = (SSchJob *)pJob; @@ -627,9 +627,9 @@ TEST(insertTest, normalCase) { schtInitLogFile(); - SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + SArray *qnodeList = taosArrayInit(1, sizeof(SEp)); - SEpAddr qnodeAddr = {0}; + SEp qnodeAddr = {0}; strcpy(qnodeAddr.fqdn, "qnode0.ep"); qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); @@ -649,7 +649,7 @@ TEST(insertTest, normalCase) { pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, "insert into tb values(now,1)", &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 61d781210c..5c214b75a1 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -13,7 +13,7 @@ target_link_libraries( PUBLIC util PUBLIC common ) -if (${BUILD_WITH_UV}) +if (${BUILD_WITH_UV_TRANS}) target_include_directories( transport PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include" @@ -25,7 +25,7 @@ if (${BUILD_WITH_UV}) PUBLIC uv_a ) add_definitions(-DUSE_UV) -endif(${BUILD_WITH_UV}) +endif(${BUILD_WITH_UV_TRANS}) if (${BUILD_TEST}) add_subdirectory(test) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index c7a23a2482..082c89fed4 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -202,6 +202,8 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); void transConnCtxDestroy(STransConnCtx* ctx); void transFreeMsg(void* msg); + +// typedef struct SConnBuffer { char* buf; int len; @@ -209,4 +211,27 @@ typedef struct SConnBuffer { int left; } SConnBuffer; +typedef void (*AsyncCB)(uv_async_t* handle); + +typedef struct { + void* pThrd; + queue qmsg; + pthread_mutex_t mtx; // protect qmsg; +} SAsyncItem; + +typedef struct { + int index; + int nAsync; + uv_async_t* asyncs; +} SAsyncPool; + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb); +void transDestroyAsyncPool(SAsyncPool* pool); +int transSendAsync(SAsyncPool* pool, queue* mq); + +int transInitBuffer(SConnBuffer* buf); +int transClearBuffer(SConnBuffer* buf); +int transDestroyBuffer(SConnBuffer* buf); +int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); + #endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index b0927e9ecf..a286482fc1 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -814,9 +814,9 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SEpSet * pEpSet = &pContext->epSet; pConn = - rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); + rpcGetConnFromCache(pRpc->pCache, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType); if (pConn == NULL || pConn->user[0] == 0) { - pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); + pConn = rpcOpenConn(pRpc, pEpSet->eps[pEpSet->inUse].fqdn, pEpSet->eps[pEpSet->inUse].port, pContext->connType); } if (pConn) { @@ -1188,7 +1188,7 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.eps[pContext->epSet.inUse].port, pConn->connType); } else { rpcCloseConn(pConn); @@ -1202,9 +1202,9 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps, pContext->epSet.inUse); for (int i = 0; i < pContext->epSet.numOfEps; ++i) { - pContext->epSet.port[i] = htons(pContext->epSet.port[i]); - tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.fqdn[i], - pContext->epSet.port[i]); + pContext->epSet.eps[i].port = htons(pContext->epSet.eps[i].port); + tDebug("%s, redirect is received, index:%d ep:%s:%u", pConn->info, i, pContext->epSet.eps[i].fqdn, + pContext->epSet.eps[i].port); } } rpcSendReqToServer(pRpc, pContext); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index bc9a9de318..91f9a8ead2 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -30,7 +30,8 @@ void* rpcOpen(const SRpcInit* pInit) { tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); } pRpc->cfp = pInit->cfp; - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + // pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + pRpc->numOfThreads = pInit->numOfThreads; pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); @@ -55,7 +56,13 @@ void* rpcMallocCont(int contLen) { } return start + sizeof(STransMsgHead); } -void rpcFreeCont(void* cont) { return; } +void rpcFreeCont(void* cont) { + // impl + if (cont == NULL) { + return; + } + free((char*)cont - TRANS_MSG_OVERHEAD); +} void* rpcReallocCont(void* ptr, int contLen) { return NULL; } void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index f265acf8c1..24ff5e956a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -31,6 +31,7 @@ typedef struct SCliConn { char secured; uint64_t expireTime; int8_t notifyCount; // timers already notify to client + int32_t ref; } SCliConn; typedef struct SCliMsg { @@ -41,16 +42,17 @@ typedef struct SCliMsg { } SCliMsg; typedef struct SCliThrdObj { - pthread_t thread; - uv_loop_t* loop; - uv_async_t* cliAsync; // - uv_timer_t* pTimer; + pthread_t thread; + uv_loop_t* loop; + // uv_async_t* cliAsync; // + SAsyncPool* asyncPool; + uv_timer_t* timer; void* pool; // conn pool queue msg; pthread_mutex_t msgMtx; uint64_t nextTimeout; // next timeout void* pTransInst; // - + bool quit; } SCliThrdObj; typedef struct SClientObj { @@ -93,9 +95,13 @@ static void clientHandleResp(SCliConn* conn); static void clientHandleExcept(SCliConn* conn); // handle req from app static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void clientSendQuit(SCliThrdObj* thrd); -static void clientMsgDestroy(SCliMsg* pMsg); -static void destroyTransConnCtx(STransConnCtx* ctx); +static void destroyUserdata(SRpcMsg* userdata); + +static void destroyCmsg(SCliMsg* cmsg); +static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static SCliThrdObj* createThrdObj(); static void destroyThrdObj(SCliThrdObj* pThrd); @@ -103,7 +109,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd); static void* clientThread(void* arg); static void clientHandleResp(SCliConn* conn) { - STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; + SCliMsg* pMsg = conn->data; + STransConnCtx* pCtx = pMsg->ctx; SRpcInfo* pRpc = pCtx->pTransInst; STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); @@ -112,41 +119,53 @@ static void clientHandleResp(SCliConn* conn) { SRpcMsg rpcMsg; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); - rpcMsg.pCont = transContFromHead(pHead); + rpcMsg.pCont = transContFromHead((char*)pHead); rpcMsg.code = pHead->code; rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; + tDebug("conn %p handle resp", conn); (pRpc->cfp)(NULL, &rpcMsg, NULL); conn->notifyCount += 1; + // buf's mem alread translated to rpcMsg.pCont + transClearBuffer(&conn->readBuf); + + uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb); + SCliThrdObj* pThrd = conn->hostThrd; - tfree(conn->data); addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + destroyCmsg(pMsg); + conn->data = NULL; // start thread's timer of conn pool if not active - if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { - uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); + if (!uv_is_active((uv_handle_t*)pThrd->timer) && pRpc->idleTime > 0) { + uv_timer_start((uv_timer_t*)pThrd->timer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } - destroyTransConnCtx(pCtx); } static void clientHandleExcept(SCliConn* pConn) { + if (pConn->data == NULL) { + // handle conn except in conn pool + clientConnDestroy(pConn, true); + return; + } + tDebug("conn %p start to destroy", pConn); SCliMsg* pMsg = pConn->data; - STransConnCtx* pCtx = pMsg->ctx; - SRpcInfo* pRpc = pCtx->pTransInst; + destroyUserdata(&pMsg->msg); - transFreeMsg((pMsg->msg.pCont)); - pMsg->msg.pCont = NULL; + STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; - rpcMsg.code = -1; + rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; // SRpcInfo* pRpc = pMsg->ctx->pRpc; - (pRpc->cfp)(NULL, &rpcMsg, NULL); - tfree(pConn->data); + (pCtx->pTransInst->cfp)(NULL, &rpcMsg, NULL); pConn->notifyCount += 1; - destroyTransConnCtx(pCtx); + + destroyCmsg(pMsg); + pConn->data = NULL; + // transDestroyConnCtx(pCtx); clientConnDestroy(pConn, true); } @@ -213,14 +232,20 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { } queue* h = QUEUE_HEAD(&plist->conn); QUEUE_REMOVE(h); - return QUEUE_DATA(h, SCliConn, conn); + + SCliConn* conn = QUEUE_DATA(h, SCliConn, conn); + QUEUE_INIT(&conn->conn); + return conn; } static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { char key[128] = {0}; + tstrncpy(key, ip, strlen(ip)); tstrncpy(key + strlen(key), (char*)(&port), sizeof(port)); + tDebug("conn %p added to conn pool, read buf cap: %d", conn, conn->readBuf.cap); SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; + conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); conn->notifyCount = 0; @@ -237,40 +262,18 @@ static bool clientReadComplete(SConnBuffer* data) { if (msgLen > data->len) { data->left = msgLen - data->len; return false; - } else { + } else if (msgLen == data->len) { + data->left = 0; return true; } } else { return false; } } -static void clientAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - // impl later - static const int CAPACITY = 512; - +static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SCliConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; - if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(1, CAPACITY * sizeof(char)); - pBuf->len = 0; - pBuf->cap = CAPACITY; - pBuf->left = -1; - - buf->base = pBuf->buf; - buf->len = CAPACITY; - } else { - if (pBuf->len >= pBuf->cap) { - if (pBuf->left == -1) { - pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap); - } else if (pBuf->len + pBuf->left > pBuf->cap) { - pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->cap); - } - } - buf->base = pBuf->buf + pBuf->len; - buf->len = pBuf->cap - pBuf->len; - } + transAllocBuffer(pBuf, buf); } static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { // impl later @@ -279,6 +282,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { + uv_read_stop((uv_stream_t*)conn->stream); tDebug("conn %p read complete", conn); clientHandleResp(conn); } else { @@ -288,10 +292,12 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf } assert(nread <= 0); if (nread == 0) { - tError("conn %p closed", conn); + // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb + // nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under + // read(2). return; } - if (nread < 0) { + if (nread < 0 || nread == UV_EOF) { tError("conn %p read error: %s", conn, uv_err_name(nread)); clientHandleExcept(conn); } @@ -300,43 +306,46 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf } static void clientConnDestroy(SCliConn* conn, bool clear) { - tDebug("conn %p destroy", conn); - if (clear) { - uv_close((uv_handle_t*)conn->stream, NULL); + // + conn->ref--; + if (conn->ref == 0) { + tDebug("conn %p remove from conn pool", conn); + QUEUE_REMOVE(&conn->conn); + tDebug("conn %p remove from conn pool successfully", conn); + if (clear) { + uv_close((uv_handle_t*)conn->stream, clientDestroy); + } } - free(conn->stream); - free(conn->readBuf.buf); - free(conn->writeReq); - free(conn); } static void clientDestroy(uv_handle_t* handle) { SCliConn* conn = handle->data; - // QUEUE_REMOVE(&conn->conn); - clientConnDestroy(conn, false); + // transDestroyBuffer(&conn->readBuf); + + free(conn->stream); + free(conn->writeReq); + tDebug("conn %p destroy successfully", conn); + free(conn); + + // clientConnDestroy(conn, false); } static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; - - SCliMsg* pMsg = pConn->data; - transFreeMsg((pMsg->msg.pCont)); - pMsg->msg.pCont = NULL; - if (status == 0) { tDebug("conn %p data already was written out", pConn); + SCliMsg* pMsg = pConn->data; + if (pMsg == NULL) { + // handle + return; + } + destroyUserdata(&pMsg->msg); } else { tError("conn %p failed to write: %s", pConn, uv_err_name(status)); clientHandleExcept(pConn); return; } SCliThrdObj* pThrd = pConn->hostThrd; - // if (pConn->stream == NULL) { - // pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); - // uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); - // pConn->stream->data = pConn; - //} - uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); - // impl later + uv_read_start((uv_stream_t*)pConn->stream, clientAllocBufferCb, clientReadCb); } static void clientWrite(SCliConn* pConn) { @@ -368,6 +377,15 @@ static void clientConnCb(uv_connect_t* req, int status) { clientWrite(pConn); } +static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { + tDebug("thread %p start to quit", pThrd); + destroyCmsg(pMsg); + // transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL); + uv_timer_stop(pThrd->timer); + pThrd->quit = true; + // uv__async_stop(pThrd->cliAsync); + uv_stop(pThrd->loop); +} static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { uint64_t et = taosGetTimestampUs(); uint64_t el = et - pMsg->st; @@ -381,14 +399,17 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("conn %p get from conn pool", conn); conn->data = pMsg; conn->writeReq->data = conn; + transDestroyBuffer(&conn->readBuf); - conn->readBuf.len = 0; - memset(conn->readBuf.buf, 0, conn->readBuf.cap); - conn->readBuf.left = -1; + if (pThrd->quit) { + clientHandleExcept(conn); + return; + } clientWrite(conn); + } else { SCliConn* conn = calloc(1, sizeof(SCliConn)); - + conn->ref++; // read/write stream handle conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); @@ -397,6 +418,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { // write req handle conn->writeReq = malloc(sizeof(uv_write_t)); conn->writeReq->data = conn; + QUEUE_INIT(&conn->conn); conn->connReq.data = conn; @@ -410,14 +432,15 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { } } static void clientAsyncCb(uv_async_t* handle) { - SCliThrdObj* pThrd = handle->data; + SAsyncItem* item = handle->data; + SCliThrdObj* pThrd = item->pThrd; SCliMsg* pMsg = NULL; queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&pThrd->msgMtx); - QUEUE_MOVE(&pThrd->msg, &wq); - pthread_mutex_unlock(&pThrd->msgMtx); + pthread_mutex_lock(&item->mtx); + QUEUE_MOVE(&item->qmsg, &wq); + pthread_mutex_unlock(&item->mtx); int count = 0; while (!QUEUE_IS_EMPTY(&wq)) { @@ -425,7 +448,12 @@ static void clientAsyncCb(uv_async_t* handle) { QUEUE_REMOVE(h); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - clientHandleReq(pMsg, pThrd); + if (pMsg->ctx == NULL) { + clientHandleQuit(pMsg, pThrd); + } else { + clientHandleReq(pMsg, pThrd); + } + // clientHandleReq(pMsg, pThrd); count++; } if (count >= 2) { @@ -453,16 +481,29 @@ void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, int err = pthread_create(&pThrd->thread, NULL, clientThread, (void*)(pThrd)); if (err == 0) { - tDebug("sucess to create tranport-client thread %d", i); + tDebug("success to create tranport-client thread %d", i); } cli->pThreadObj[i] = pThrd; } return cli; } -static void clientMsgDestroy(SCliMsg* pMsg) { - // impl later + +static void destroyUserdata(SRpcMsg* userdata) { + if (userdata->pCont == NULL) { + return; + } + transFreeMsg(userdata->pCont); + userdata->pCont = NULL; +} +static void destroyCmsg(SCliMsg* pMsg) { + if (pMsg == NULL) { + return; + } + transDestroyConnCtx(pMsg->ctx); + destroyUserdata(&pMsg->msg); free(pMsg); } + static SCliThrdObj* createThrdObj() { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); @@ -471,39 +512,55 @@ static SCliThrdObj* createThrdObj() { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->cliAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); - pThrd->cliAsync->data = pThrd; + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb); - pThrd->pTimer = malloc(sizeof(uv_timer_t)); - uv_timer_init(pThrd->loop, pThrd->pTimer); - pThrd->pTimer->data = pThrd; + pThrd->timer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, pThrd->timer); + pThrd->timer->data = pThrd; pThrd->pool = creatConnPool(1); + + pThrd->quit = false; return pThrd; } static void destroyThrdObj(SCliThrdObj* pThrd) { if (pThrd == NULL) { return; } + uv_stop(pThrd->loop); pthread_join(pThrd->thread, NULL); pthread_mutex_destroy(&pThrd->msgMtx); - free(pThrd->cliAsync); + transDestroyAsyncPool(pThrd->asyncPool); + // free(pThrd->cliAsync); + free(pThrd->timer); free(pThrd->loop); free(pThrd); } -static void destroyTransConnCtx(STransConnCtx* ctx) { +static void transDestroyConnCtx(STransConnCtx* ctx) { if (ctx != NULL) { free(ctx->ip); } free(ctx); } // +static void clientSendQuit(SCliThrdObj* thrd) { + // cli can stop gracefully + SCliMsg* msg = calloc(1, sizeof(SCliMsg)); + msg->ctx = NULL; // + + // pthread_mutex_lock(&thrd->msgMtx); + // QUEUE_PUSH(&thrd->msg, &msg->q); + // pthread_mutex_unlock(&thrd->msgMtx); + + transSendAsync(thrd->asyncPool, &msg->q); + // uv_async_send(thrd->cliAsync); +} void taosCloseClient(void* arg) { // impl later SClientObj* cli = arg; for (int i = 0; i < cli->numOfThreads; i++) { + clientSendQuit(cli->pThreadObj[i]); destroyThrdObj(cli->pThreadObj[i]); } free(cli->pThreadObj); @@ -542,10 +599,14 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; - pthread_mutex_lock(&thrd->msgMtx); - QUEUE_PUSH(&thrd->msg, &cliMsg->q); - pthread_mutex_unlock(&thrd->msgMtx); + // pthread_mutex_lock(&thrd->msgMtx); + // QUEUE_PUSH(&thrd->msg, &cliMsg->q); + // pthread_mutex_unlock(&thrd->msgMtx); - uv_async_send(thrd->cliAsync); + // int start = taosGetTimestampUs(); + transSendAsync(thrd->asyncPool, &(cliMsg->q)); + // uv_async_send(thrd->cliAsync); + // int end = taosGetTimestampUs() - start; + // tError("client sent to rpc, time cost: %d", (int)end); } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5bece11bec..d0e504a0a1 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -198,4 +198,105 @@ void transFreeMsg(void* msg) { } free((char*)msg - sizeof(STransMsgHead)); } + +int transInitBuffer(SConnBuffer* buf) { + transClearBuffer(buf); + return 0; +} +int transClearBuffer(SConnBuffer* buf) { + memset(buf, 0, sizeof(*buf)); + return 0; +} +int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { + /* + * formate of data buffer: + * |<--------------------------data from socket------------------------------->| + * |<------STransMsgHead------->|<-------------------other data--------------->| + */ + static const int CAPACITY = 1024; + + SConnBuffer* p = connBuf; + if (p->cap == 0) { + p->buf = (char*)calloc(CAPACITY, sizeof(char)); + p->len = 0; + p->cap = CAPACITY; + p->left = -1; + + uvBuf->base = p->buf; + uvBuf->len = CAPACITY; + } else { + if (p->len >= p->cap) { + if (p->left == -1) { + p->cap *= 2; + p->buf = realloc(p->buf, p->cap); + } else if (p->len + p->left > p->cap) { + p->cap = p->len + p->left; + p->buf = realloc(p->buf, p->len + p->left); + } + } + uvBuf->base = p->buf + p->len; + uvBuf->len = p->cap - p->len; + } + return 0; +} +int transDestroyBuffer(SConnBuffer* buf) { + if (buf->cap > 0) { + tfree(buf->buf); + } + transClearBuffer(buf); +} + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) { + static int sz = 10; + + SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); + pool->index = 0; + pool->nAsync = sz; + pool->asyncs = calloc(1, sizeof(uv_async_t) * pool->nAsync); + + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + uv_async_init(loop, async, cb); + + SAsyncItem* item = calloc(1, sizeof(SAsyncItem)); + item->pThrd = arg; + QUEUE_INIT(&item->qmsg); + pthread_mutex_init(&item->mtx, NULL); + + async->data = item; + } + return pool; +} +void transDestroyAsyncPool(SAsyncPool* pool) { + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + + SAsyncItem* item = async->data; + pthread_mutex_destroy(&item->mtx); + free(item); + } + free(pool->asyncs); + free(pool); +} +int transSendAsync(SAsyncPool* pool, queue* q) { + int idx = pool->index; + idx = idx % pool->nAsync; + // no need mutex here + if (pool->index++ > pool->nAsync) { + pool->index = 0; + } + uv_async_t* async = &(pool->asyncs[idx]); + SAsyncItem* item = async->data; + + int64_t st = taosGetTimestampUs(); + pthread_mutex_lock(&item->mtx); + QUEUE_PUSH(&item->qmsg, q); + pthread_mutex_unlock(&item->mtx); + int64_t el = taosGetTimestampUs() - st; + if (el > 50) { + // tInfo("lock and unlock cost: %d", (int)el); + } + + return uv_async_send(async); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index c70b1a5b28..a005b31fe4 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -17,22 +17,23 @@ #include "transComm.h" -typedef struct SConn { +typedef struct SSrvConn { uv_tcp_t* pTcp; uv_write_t* pWriter; uv_timer_t* pTimer; - uv_async_t* pWorkerAsync; + // uv_async_t* pWorkerAsync; queue queue; int ref; int persist; // persist connection or not - SConnBuffer connBuf; // read buf, + SConnBuffer readBuf; // read buf, int inType; void* pTransInst; // rpc init void* ahandle; // void* hostThrd; + void* pSrvMsg; - SRpcMsg sendMsg; + // SRpcMsg sendMsg; // del later char secured; int spi; @@ -40,16 +41,23 @@ typedef struct SConn { char user[TSDB_UNI_LEN]; // user ID for the link char secret[TSDB_PASSWORD_LEN]; char ckey[TSDB_PASSWORD_LEN]; // ciphering key -} SConn; +} SSrvConn; + +typedef struct SSrvMsg { + SSrvConn* pConn; + SRpcMsg msg; + queue q; +} SSrvMsg; typedef struct SWorkThrdObj { - pthread_t thread; - uv_pipe_t* pipe; - int fd; - uv_loop_t* loop; - uv_async_t* workerAsync; // - queue conn; - pthread_mutex_t connMtx; + pthread_t thread; + uv_pipe_t* pipe; + int fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + // uv_async_t* workerAsync; // + queue msg; + pthread_mutex_t msgMtx; void* pTransInst; } SWorkThrdObj; @@ -63,14 +71,15 @@ typedef struct SServerObj { uv_pipe_t** pipe; uint32_t ip; uint32_t port; + uv_async_t* pAcceptAsync; // just to quit from from accept thread } SServerObj; static const char* notify = "a"; // refactor later -static int transAddAuthPart(SConn* pConn, char* msg, int msgLen); +static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen); -static int uvAuthMsg(SConn* pConn, char* msg, int msgLen); +static int uvAuthMsg(SSrvConn* pConn, char* msg, int msgLen); static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -81,13 +90,16 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status); static void uvOnAcceptCb(uv_stream_t* stream, int status); static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void uvWorkerAsyncCb(uv_async_t* handle); +static void uvAcceptAsyncCb(uv_async_t* handle); -static void uvPrepareSendData(SConn* conn, uv_buf_t* wb); +static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); +static void uvStartSendResp(SSrvMsg* msg); +static void destroySmsg(SSrvMsg* smsg); // check whether already read complete packet -static bool readComplete(SConnBuffer* buf); -static SConn* createConn(); -static void destroyConn(SConn* conn, bool clear /*clear handle or not*/); +static bool readComplete(SConnBuffer* buf); +static SSrvConn* createConn(); +static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); static void uvDestroyConn(uv_handle_t* handle); @@ -105,31 +117,9 @@ void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b * |<--------------------------data from socket------------------------------->| * |<------STransMsgHead------->|<-------------------other data--------------->| */ - static const int CAPACITY = 1024; - - SConn* conn = handle->data; - SConnBuffer* pBuf = &conn->connBuf; - if (pBuf->cap == 0) { - pBuf->buf = (char*)calloc(CAPACITY, sizeof(char)); - pBuf->len = 0; - pBuf->cap = CAPACITY; - pBuf->left = -1; - - buf->base = pBuf->buf; - buf->len = CAPACITY; - } else { - if (pBuf->len >= pBuf->cap) { - if (pBuf->left == -1) { - pBuf->cap *= 2; - pBuf->buf = realloc(pBuf->buf, pBuf->cap); - } else if (pBuf->len + pBuf->left > pBuf->cap) { - pBuf->cap = pBuf->len + pBuf->left; - pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left); - } - } - buf->base = pBuf->buf + pBuf->len; - buf->len = pBuf->cap - pBuf->len; - } + SSrvConn* conn = handle->data; + SConnBuffer* pBuf = &conn->readBuf; + transAllocBuffer(pBuf, buf); } // check data read from socket completely or not @@ -159,7 +149,7 @@ static bool readComplete(SConnBuffer* data) { // // impl later // STransMsgHead* pHead = (STransMsgHead*)pRecv->msg; // SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; -// SConn* pConn = pRecv->thandle; +// SSrvConn* pConn = pRecv->thandle; // tDump(pRecv->msg, pRecv->msgLen); // terrno = 0; // // SRpcReqContext* pContest; @@ -167,7 +157,7 @@ static bool readComplete(SConnBuffer* data) { // // do auth and check //} -static int uvAuthMsg(SConn* pConn, char* msg, int len) { +static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { STransMsgHead* pHead = (STransMsgHead*)msg; int code = 0; @@ -222,14 +212,14 @@ static int uvAuthMsg(SConn* pConn, char* msg, int len) { // refers specifically to query or insert timeout static void uvHandleActivityTimeout(uv_timer_t* handle) { - SConn* conn = handle->data; + SSrvConn* conn = handle->data; tDebug("%p timeout since no activity", conn); } -static void uvHandleReq(SConn* pConn) { +static void uvHandleReq(SSrvConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; - SConnBuffer* pBuf = &pConn->connBuf; + SConnBuffer* pBuf = &pConn->readBuf; p->msg = pBuf->buf; p->msgLen = pBuf->len; p->ip = 0; @@ -255,7 +245,6 @@ static void uvHandleReq(SConn* pConn) { pHead->code = htonl(pHead->code); int32_t dlen = 0; - SRpcMsg rpcMsg; if (transDecompressMsg(NULL, 0, NULL)) { // add compress later // pHead = rpcDecompressRpcMsg(pHead); @@ -264,6 +253,8 @@ static void uvHandleReq(SConn* pConn) { // impl later // } + + SRpcMsg rpcMsg; rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.pCont = pHead->content; rpcMsg.msgType = pHead->msgType; @@ -271,6 +262,7 @@ static void uvHandleReq(SConn* pConn) { rpcMsg.ahandle = NULL; rpcMsg.handle = pConn; + transClearBuffer(&pConn->readBuf); pConn->ref++; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); @@ -280,8 +272,8 @@ static void uvHandleReq(SConn* pConn) { void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { // opt - SConn* conn = cli->data; - SConnBuffer* pBuf = &conn->connBuf; + SSrvConn* conn = cli->data; + SConnBuffer* pBuf = &conn->readBuf; if (nread > 0) { pBuf->len += nread; tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread); @@ -294,11 +286,12 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { return; } if (nread == 0) { - tDebug("conn %p except read", conn); - // destroyConn(conn, true); return; } - if (nread != UV_EOF) { + if (nread < 0 || nread != UV_EOF) { + if (conn->ref > 1) { + conn->ref++; // ref > 1 signed that write is in progress + } tDebug("conn %p read error: %s", conn, uv_err_name(nread)); destroyConn(conn, true); } @@ -310,25 +303,23 @@ void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b void uvOnTimeoutCb(uv_timer_t* handle) { // opt - SConn* pConn = handle->data; + SSrvConn* pConn = handle->data; tDebug("conn %p time out", pConn); } void uvOnWriteCb(uv_write_t* req, int status) { - SConn* conn = req->data; + SSrvConn* conn = req->data; - SConnBuffer* buf = &conn->connBuf; - buf->len = 0; - memset(buf->buf, 0, buf->cap); - buf->left = -1; - - SRpcMsg* pMsg = &conn->sendMsg; - transFreeMsg(pMsg->pCont); + SSrvMsg* smsg = conn->pSrvMsg; + destroySmsg(smsg); + conn->pSrvMsg = NULL; + transClearBuffer(&conn->readBuf); if (status == 0) { tDebug("conn %p data already was written on stream", conn); } else { tDebug("conn %p failed to write data, %s", conn, uv_err_name(status)); + // destroyConn(conn, true); } // opt @@ -341,16 +332,16 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { } } -static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { +static void uvPrepareSendData(SSrvMsg* smsg, uv_buf_t* wb) { // impl later; - tDebug("conn %p prepare to send resp", conn); - SRpcMsg* pMsg = &conn->sendMsg; + tDebug("conn %p prepare to send resp", smsg->pConn); + SRpcMsg* pMsg = &smsg->msg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); - pHead->msgType = conn->inType + 1; + pHead->msgType = smsg->pConn->inType + 1; // add more info char* msg = (char*)pHead; int32_t len = transMsgLenFromCont(pMsg->contLen); @@ -361,30 +352,67 @@ static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { wb->base = msg; wb->len = len; } +static void uvStartSendResp(SSrvMsg* smsg) { + // impl + uv_buf_t wb; + uvPrepareSendData(smsg, &wb); + + SSrvConn* pConn = smsg->pConn; + uv_timer_stop(pConn->pTimer); + + pConn->pSrvMsg = smsg; + // conn->pWriter->data = smsg; + uv_write(pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnWriteCb); + + // SRpcMsg* rpcMsg = smsg->msg; + + return; +} +static void destroySmsg(SSrvMsg* smsg) { + if (smsg == NULL) { + return; + } + transFreeMsg(smsg->msg.pCont); + free(smsg); +} void uvWorkerAsyncCb(uv_async_t* handle) { - SWorkThrdObj* pThrd = handle->data; - SConn* conn = NULL; + SAsyncItem* item = handle->data; + SWorkThrdObj* pThrd = item->pThrd; + SSrvConn* conn = NULL; queue wq; // batch process to avoid to lock/unlock frequently - pthread_mutex_lock(&pThrd->connMtx); - QUEUE_MOVE(&pThrd->conn, &wq); - pthread_mutex_unlock(&pThrd->connMtx); + pthread_mutex_lock(&item->mtx); + QUEUE_MOVE(&item->qmsg, &wq); + pthread_mutex_unlock(&item->mtx); + // pthread_mutex_unlock(&mtx); while (!QUEUE_IS_EMPTY(&wq)) { queue* head = QUEUE_HEAD(&wq); QUEUE_REMOVE(head); - SConn* conn = QUEUE_DATA(head, SConn, queue); - if (conn == NULL) { - tError("except occurred, do nothing"); - return; - } - uv_buf_t wb; - uvPrepareSendData(conn, &wb); - uv_timer_stop(conn->pTimer); - uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); + SSrvMsg* msg = QUEUE_DATA(head, SSrvMsg, q); + if (msg == NULL) { + tError("except occurred, continue"); + continue; + } + if (msg->pConn == NULL) { + // + free(msg); + uv_stop(pThrd->loop); + } else { + uvStartSendResp(msg); + } + // uv_buf_t wb; + // uvPrepareSendData(msg, &wb); + // uv_timer_stop(conn->pTimer); + + // uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); } } +static void uvAcceptAsyncCb(uv_async_t* async) { + SServerObj* srv = async->data; + uv_stop(srv->loop); +} void uvOnAcceptCb(uv_stream_t* stream, int status) { if (status == -1) { @@ -435,7 +463,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { uv_handle_type pending = uv_pipe_pending_type(pipe); assert(pending == UV_TCP); - SConn* pConn = createConn(); + SSrvConn* pConn = createConn(); pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ @@ -444,7 +472,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - pConn->pWorkerAsync = pThrd->workerAsync; // thread safty + // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -484,13 +512,10 @@ static bool addHandleToWorkloop(void* arg) { pThrd->pipe->data = pThrd; - QUEUE_INIT(&pThrd->conn); - pthread_mutex_init(&pThrd->connMtx, NULL); - - pThrd->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); - pThrd->workerAsync->data = pThrd; + QUEUE_INIT(&pThrd->msg); + pthread_mutex_init(&pThrd->msgMtx, NULL); + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -505,14 +530,18 @@ static bool addHandleToAcceptloop(void* arg) { return false; } - struct sockaddr_in bind_addr; + // register an async here to quit server gracefully + srv->pAcceptAsync = calloc(1, sizeof(uv_async_t)); + uv_async_init(srv->loop, srv->pAcceptAsync, uvAcceptAsyncCb); + srv->pAcceptAsync->data = srv; + struct sockaddr_in bind_addr; uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); if ((err = uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0)) != 0) { tError("failed to bind: %s", uv_err_name(err)); return false; } - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { + if ((err = uv_listen((uv_stream_t*)&srv->server, 512, uvOnAcceptCb)) != 0) { tError("failed to listen: %s", uv_err_name(err)); return false; } @@ -523,34 +552,39 @@ void* workerThread(void* arg) { uv_run(pThrd->loop, UV_RUN_DEFAULT); } -static SConn* createConn() { - SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); +static SSrvConn* createConn() { + SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); + tDebug("conn %p created", pConn); ++pConn->ref; return pConn; } -static void destroyConn(SConn* conn, bool clear) { +static void destroyConn(SSrvConn* conn, bool clear) { if (conn == NULL) { return; } - if (--conn->ref == 0) { + tDebug("conn %p try to destroy", conn); + if (--conn->ref > 0) { return; } + transDestroyBuffer(&conn->readBuf); + destroySmsg(conn->pSrvMsg); + conn->pSrvMsg = NULL; + if (clear) { - uv_close((uv_handle_t*)conn->pTcp, NULL); + uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); } +} +static void uvDestroyConn(uv_handle_t* handle) { + SSrvConn* conn = handle->data; + tDebug("conn %p destroy", conn); uv_timer_stop(conn->pTimer); free(conn->pTimer); - free(conn->pTcp); - free(conn->connBuf.buf); + // free(conn->pTcp); free(conn->pWriter); free(conn); } -static void uvDestroyConn(uv_handle_t* handle) { - SConn* conn = handle->data; - destroyConn(conn, false); -} -static int transAddAuthPart(SConn* pConn, char* msg, int msgLen) { +static int transAddAuthPart(SSrvConn* pConn, char* msg, int msgLen) { STransMsgHead* pHead = (STransMsgHead*)msg; if (pConn->spi && pConn->secured == 0) { @@ -630,35 +664,63 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { return; } pthread_join(pThrd->thread, NULL); - // free(srv->pipe[i]); free(pThrd->loop); + transDestroyAsyncPool(pThrd->asyncPool); + + // free(pThrd->workerAsync); free(pThrd); } +void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + + // pthread_mutex_lock(&pThrd->msgMtx); + // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); + // pthread_mutex_unlock(&pThrd->msgMtx); + tDebug("send quit msg to work thread"); + + transSendAsync(pThrd->asyncPool, &srvMsg->q); + // uv_async_send(pThrd->workerAsync); +} + void taosCloseServer(void* arg) { // impl later SServerObj* srv = arg; for (int i = 0; i < srv->numOfThreads; i++) { + sendQuitToWorkThrd(srv->pThreadObj[i]); destroyWorkThrd(srv->pThreadObj[i]); } - free(srv->loop); - free(srv->pipe); - free(srv->pThreadObj); + + tDebug("send quit msg to accept thread"); + uv_async_send(srv->pAcceptAsync); pthread_join(srv->thread, NULL); + + free(srv->pThreadObj); + free(srv->pAcceptAsync); + free(srv->loop); + + for (int i = 0; i < srv->numOfThreads; i++) { + free(srv->pipe[i]); + } + free(srv->pipe); + free(srv); } void rpcSendResponse(const SRpcMsg* pMsg) { - SConn* pConn = pMsg->handle; + SSrvConn* pConn = pMsg->handle; SWorkThrdObj* pThrd = pConn->hostThrd; - // opt later - pConn->sendMsg = *pMsg; - pthread_mutex_lock(&pThrd->connMtx); - QUEUE_PUSH(&pThrd->conn, &pConn->queue); - pthread_mutex_unlock(&pThrd->connMtx); - tDebug("conn %p start to send resp", pConn); + SSrvMsg* srvMsg = calloc(1, sizeof(SSrvMsg)); + srvMsg->pConn = pConn; + srvMsg->msg = *pMsg; - uv_async_send(pConn->pWorkerAsync); + // pthread_mutex_lock(&pThrd->msgMtx); + // QUEUE_PUSH(&pThrd->msg, &srvMsg->q); + // pthread_mutex_unlock(&pThrd->msgMtx); + + tDebug("conn %p start to send resp", pConn); + transSendAsync(pThrd->asyncPool, &srvMsg->q); + // uv_async_send(pThrd->workerAsync); } #endif diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index c61f688060..3d9c396336 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -1,6 +1,12 @@ add_executable(transportTest "") add_executable(client "") add_executable(server "") +add_executable(transUT "") + +target_sources(transUT + PRIVATE + "transUT.cc" +) target_sources(transportTest PRIVATE @@ -28,6 +34,13 @@ target_link_libraries (transportTest gtest_main transport ) +target_link_libraries (transUT + os + util + common + gtest_main + transport +) target_include_directories(client PUBLIC @@ -48,6 +61,13 @@ target_include_directories(server "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(transUT + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + + target_link_libraries (server os util diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 84814f39fc..308b7b54bd 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -12,7 +12,9 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include +#include #include "os.h" #include "rpcLog.h" #include "taoserror.h" @@ -52,6 +54,12 @@ static void *sendRequest(void *param) { tDebug("thread:%d, start to send request", pInfo->index); + tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + int u100 = 0; + int u500 = 0; + int u1000 = 0; + int u10000 = 0; + while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { pInfo->num++; rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); @@ -59,14 +67,28 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); + int64_t end = taosGetTimestampUs() - start; + if (end <= 100) { + u100++; + } else if (end > 100 && end <= 500) { + u500++; + } else if (end > 500 && end < 1000) { + u1000++; + } else { + u10000++; + } + tDebug("recv response succefully"); + // usleep(100000000); } + tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); tDebug("thread:%d, it is over", pInfo->index); tcount++; @@ -86,12 +108,9 @@ int main(int argc, char *argv[]) { pthread_attr_t thattr; // server info - epSet.numOfEps = 1; epSet.inUse = 0; - epSet.port[0] = 7000; - epSet.port[1] = 7000; - strcpy(epSet.fqdn[0], serverIp); - strcpy(epSet.fqdn[1], "192.168.0.1"); + addEpIntoEpSet(&epSet, serverIp, 7000); + addEpIntoEpSet(&epSet, "192.168.0.1", 7000); // client info memset(&rpcInit, 0, sizeof(rpcInit)); @@ -109,9 +128,9 @@ int main(int argc, char *argv[]) { for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { - epSet.port[0] = atoi(argv[++i]); + epSet.eps[0].port = atoi(argv[++i]); } else if (strcmp(argv[i], "-i") == 0 && i < argc - 1) { - tstrncpy(epSet.fqdn[0], argv[++i], sizeof(epSet.fqdn[0])); + tstrncpy(epSet.eps[0].fqdn, argv[++i], sizeof(epSet.eps[0].fqdn)); } else if (strcmp(argv[i], "-t") == 0 && i < argc - 1) { rpcInit.numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-m") == 0 && i < argc - 1) { @@ -135,7 +154,7 @@ int main(int argc, char *argv[]) { } else { printf("\nusage: %s [options] \n", argv[0]); printf(" [-i ip]: first server IP address, default is:%s\n", serverIp); - printf(" [-p port]: server port number, default is:%d\n", epSet.port[0]); + printf(" [-p port]: server port number, default is:%d\n", epSet.eps[0].port); printf(" [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads); printf(" [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions); printf(" [-m msgSize]: message body size, default is:%d\n", msgSize); @@ -162,8 +181,8 @@ int main(int argc, char *argv[]) { tInfo("client is initialized"); tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); - // gettimeofday(&systemTime, NULL); - // startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + gettimeofday(&systemTime, NULL); + startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); @@ -185,13 +204,12 @@ int main(int argc, char *argv[]) { usleep(1); } while (tcount < appThreads); - // gettimeofday(&systemTime, NULL); - // endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - // float usedTime = (endTime - startTime) / 1000.0f; // mseconds + gettimeofday(&systemTime, NULL); + endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime) / 1000.0f; // mseconds - // tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - // tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, - // msgSize); + tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); int ch = getchar(); UNUSED(ch); diff --git a/source/libs/transport/test/rserver.c b/source/libs/transport/test/rserver.c index 12d8a01819..d1a587f4e5 100644 --- a/source/libs/transport/test/rserver.c +++ b/source/libs/transport/test/rserver.c @@ -165,6 +165,7 @@ int main(int argc, char *argv[]) { tError("failed to start RPC server"); return -1; } + // sleep(5); tInfo("RPC server is running, ctrl-c to exit"); @@ -172,7 +173,6 @@ int main(int argc, char *argv[]) { dataFd = open(dataName, O_APPEND | O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO); if (dataFd < 0) tInfo("failed to open data file, reason:%s", strerror(errno)); } - qhandle = taosOpenQueue(); qset = taosOpenQset(); taosAddIntoQset(qset, qhandle, NULL); diff --git a/source/libs/transport/test/transUT.cc b/source/libs/transport/test/transUT.cc new file mode 100644 index 0000000000..08c683590b --- /dev/null +++ b/source/libs/transport/test/transUT.cc @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include +#include +#include +#include "trpc.h" +using namespace std; + +class TransObj { + public: + TransObj() { + const char *label = "APP"; + const char *secret = "secret"; + const char *user = "user"; + const char *ckey = "ckey"; + + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.localPort = 0; + rpcInit.label = (char *)label; + rpcInit.numOfThreads = 5; + rpcInit.cfp = NULL; + rpcInit.sessions = 100; + rpcInit.idleTime = 100; + rpcInit.user = (char *)user; + rpcInit.secret = (char *)secret; + rpcInit.ckey = (char *)ckey; + rpcInit.spi = 1; + } + bool startCli() { + trans = NULL; + rpcInit.connType = TAOS_CONN_CLIENT; + trans = rpcOpen(&rpcInit); + return trans != NULL ? true : false; + } + bool startSrv() { + trans = NULL; + rpcInit.connType = TAOS_CONN_SERVER; + trans = rpcOpen(&rpcInit); + return trans != NULL ? true : false; + } + bool stop() { + rpcClose(trans); + trans = NULL; + return true; + } + + private: + void * trans; + SRpcInit rpcInit; +}; +class TransEnv : public ::testing::Test { + protected: + virtual void SetUp() { + // set up trans obj + tr = new TransObj(); + } + virtual void TearDown() { + // tear down + delete tr; + } + + TransObj *tr = NULL; +}; +TEST_F(TransEnv, test_start_stop) { + assert(tr->startCli()); + assert(tr->stop()); + + assert(tr->startSrv()); + assert(tr->stop()); +} diff --git a/source/libs/wal/CMakeLists.txt b/source/libs/wal/CMakeLists.txt index 4d2dd97c87..bcf759e04f 100644 --- a/source/libs/wal/CMakeLists.txt +++ b/source/libs/wal/CMakeLists.txt @@ -11,6 +11,7 @@ target_link_libraries( PUBLIC cjson PUBLIC os PUBLIC util + PUBLIC common ) if(${BUILD_TEST}) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 96d0c6c385..1d9201f69d 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -169,10 +169,17 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return -1; } - ASSERT(pRead->pHead->head.version == ver); + if (pRead->pHead->head.version != ver) { + wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } code = walValidBodyCksum(pRead->pHead); if (code != 0) { + wError("unexpected wal log version: checksum not passed"); + pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 2bc328b4e2..a4b34dee37 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -257,7 +257,7 @@ static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { return 0; } -int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, const void *body, int32_t bodyLen) { +int64_t walWrite(SWal *pWal, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen) { if (pWal == NULL) return -1; int code = 0; diff --git a/source/nodes/src/nodesClone.c b/source/nodes/src/nodesCloneFuncs.c similarity index 93% rename from source/nodes/src/nodesClone.c rename to source/nodes/src/nodesCloneFuncs.c index 04f6df5623..2d0a6483ae 100644 --- a/source/nodes/src/nodesClone.c +++ b/source/nodes/src/nodesCloneFuncs.c @@ -15,6 +15,6 @@ #include "nodes.h" -void cloneNode(const SNode* pNode) { +void nodesCloneNode(const SNode* pNode) { } diff --git a/source/nodes/src/nodesCodeFuncs.c b/source/nodes/src/nodesCodeFuncs.c new file mode 100644 index 0000000000..959e10fb82 --- /dev/null +++ b/source/nodes/src/nodesCodeFuncs.c @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_IS_NULL_CONDITION: + case QUERY_NODE_FUNCTION: + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + case QUERY_NODE_JOIN_TABLE: + case QUERY_NODE_GROUPING_SET: + case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_LIMIT: + case QUERY_NODE_STATE_WINDOW: + case QUERY_NODE_SESSION_WINDOW: + case QUERY_NODE_INTERVAL_WINDOW: + case QUERY_NODE_SET_OPERATOR: + case QUERY_NODE_SELECT_STMT: + case QUERY_NODE_SHOW_STMT: + default: + break; + } +} + +int32_t nodesStringToNode(const char* pStr, SNode** pNode) { + +} diff --git a/source/nodes/src/nodesEqual.c b/source/nodes/src/nodesEqualFuncs.c similarity index 85% rename from source/nodes/src/nodesEqual.c rename to source/nodes/src/nodesEqualFuncs.c index bef025fbea..41d1e5b05d 100644 --- a/source/nodes/src/nodesEqual.c +++ b/source/nodes/src/nodesEqualFuncs.c @@ -32,17 +32,17 @@ #define COMPARE_NODE_FIELD(fldname) \ do { \ - if (!nodeEqual(a->fldname, b->fldname)) \ + if (!nodesEqualNode(a->fldname, b->fldname)) \ return false; \ } while (0) -#define COMPARE_ARRAY_FIELD(fldname) \ +#define COMPARE_NODE_LIST_FIELD(fldname) \ do { \ - if (!nodeArrayEqual(a->fldname, b->fldname)) \ + if (!nodeNodeListEqual(a->fldname, b->fldname)) \ return false; \ } while (0) -static bool nodeArrayEqual(const SArray* a, const SArray* b) { +static bool nodeNodeListEqual(const SNodeList* a, const SNodeList* b) { if (a == b) { return true; } @@ -51,13 +51,13 @@ static bool nodeArrayEqual(const SArray* a, const SArray* b) { return false; } - if (taosArrayGetSize(a) != taosArrayGetSize(b)) { + if (LIST_LENGTH(a) != LIST_LENGTH(b)) { return false; } - size_t size = taosArrayGetSize(a); - for (size_t i = 0; i < size; ++i) { - if (!nodeEqual((SNode*)taosArrayGetP(a, i), (SNode*)taosArrayGetP(b, i))) { + SNode* na, *nb; + FORBOTH(na, a, nb, b) { + if (!nodesEqualNode(na, nb)) { return false; } } @@ -85,23 +85,23 @@ static bool operatorNodeEqual(const SOperatorNode* a, const SOperatorNode* b) { static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicConditionNode* b) { COMPARE_SCALAR_FIELD(condType); - COMPARE_ARRAY_FIELD(pParameterList); + COMPARE_NODE_LIST_FIELD(pParameterList); return true; } static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCondNode* b) { COMPARE_NODE_FIELD(pExpr); - COMPARE_SCALAR_FIELD(isNot); + COMPARE_SCALAR_FIELD(isNull); return true; } static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_SCALAR_FIELD(funcId); - COMPARE_ARRAY_FIELD(pParameterList); + COMPARE_NODE_LIST_FIELD(pParameterList); return true; } -bool nodeEqual(const SNode* a, const SNode* b) { +bool nodesEqualNode(const SNode* a, const SNode* b) { if (a == b) { return true; } @@ -132,6 +132,7 @@ bool nodeEqual(const SNode* a, const SNode* b) { case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_GROUPING_SET: case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_LIMIT: return false; // todo default: break; diff --git a/source/nodes/src/nodesTraverse.c b/source/nodes/src/nodesTraverse.c deleted file mode 100644 index eac8288099..0000000000 --- a/source/nodes/src/nodesTraverse.c +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "nodes.h" - -typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); - -bool nodeArrayWalker(SArray* pArray, FQueryNodeWalker walker, void* pContext) { - size_t size = taosArrayGetSize(pArray); - for (size_t i = 0; i < size; ++i) { - if (!nodeTreeWalker((SNode*)taosArrayGetP(pArray, i), walker, pContext)) { - return false; - } - } - return true; -} - -bool nodeTreeWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - if (NULL == pNode) { - return true; - } - - if (!walker(pNode, pContext)) { - return false; - } - - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: - case QUERY_NODE_VALUE: - // these node types with no subnodes - return true; - case QUERY_NODE_OPERATOR: { - SOperatorNode* pOpNode = (SOperatorNode*)pNode; - if (!nodeTreeWalker(pOpNode->pLeft, walker, pContext)) { - return false; - } - return nodeTreeWalker(pOpNode->pRight, walker, pContext); - } - case QUERY_NODE_LOGIC_CONDITION: - return nodeArrayWalker(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_IS_NULL_CONDITION: - return nodeTreeWalker(((SIsNullCondNode*)pNode)->pExpr, walker, pContext); - case QUERY_NODE_FUNCTION: - return nodeArrayWalker(((SFunctionNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_REAL_TABLE: - case QUERY_NODE_TEMP_TABLE: - return true; // todo - case QUERY_NODE_JOIN_TABLE: { - SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; - if (!nodeTreeWalker(pJoinTableNode->pLeft, walker, pContext)) { - return false; - } - if (!nodeTreeWalker(pJoinTableNode->pRight, walker, pContext)) { - return false; - } - return nodeTreeWalker(pJoinTableNode->pOnCond, walker, pContext); - } - case QUERY_NODE_GROUPING_SET: - return nodeArrayWalker(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_ORDER_BY_EXPR: - return nodeTreeWalker(((SOrderByExprNode*)pNode)->pExpr, walker, pContext); - default: - break; - } - - return false; -} - -bool stmtWalker(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - -} diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/nodes/src/nodesTraverseFuncs.c new file mode 100644 index 0000000000..444ff7cbcf --- /dev/null +++ b/source/nodes/src/nodesTraverseFuncs.c @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" + +typedef enum ETraversalOrder { + TRAVERSAL_PREORDER = 1, + TRAVERSAL_POSTORDER +} ETraversalOrder; + +static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext); + +static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { + if (NULL == pNode) { + return true; + } + + if (TRAVERSAL_PREORDER == order && !walker(pNode, pContext)) { + return false; + } + + bool res = true; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_LIMIT: + // these node types with no subnodes + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + res = walkNode(pOpNode->pLeft, order, walker, pContext); + if (res) { + res = walkNode(pOpNode->pRight, order, walker, pContext); + } + break; + } + case QUERY_NODE_LOGIC_CONDITION: + res = walkList(((SLogicConditionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_IS_NULL_CONDITION: + res = walkNode(((SIsNullCondNode*)pNode)->pExpr, order, walker, pContext); + break; + case QUERY_NODE_FUNCTION: + res = walkList(((SFunctionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + break; // todo + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; + res = walkNode(pJoinTableNode->pLeft, order, walker, pContext); + if (res) { + res = walkNode(pJoinTableNode->pRight, order, walker, pContext); + } + if (res) { + res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext); + } + break; + } + case QUERY_NODE_GROUPING_SET: + res = walkList(((SGroupingSetNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_ORDER_BY_EXPR: + res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext); + break; + default: + break; + } + + if (res && TRAVERSAL_POSTORDER == order) { + res = walker(pNode, pContext); + } + + return res; +} + +static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { + SNode* node; + FOREACH(node, pNodeList) { + if (!walkNode(node, order, walker, pContext)) { + return false; + } + } + return true; +} + +void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); +} + +void nodesWalkList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { + (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); +} + +void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); +} + +void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext) { + (void)walkList(pList, TRAVERSAL_PREORDER, walker, pContext); +} + +bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + +} diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c new file mode 100644 index 0000000000..bf4c4a83cc --- /dev/null +++ b/source/nodes/src/nodesUtilFuncs.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodes.h" +#include "nodesShowStmts.h" +#include "taoserror.h" + +static SNode* makeNode(ENodeType type, size_t size) { + SNode* p = calloc(1, size); + if (NULL == p) { + return NULL; + } + setNodeType(p, type); + return p; +} + +SNode* nodesMakeNode(ENodeType type) { + switch (type) { + case QUERY_NODE_COLUMN: + return makeNode(type, sizeof(SColumnNode)); + case QUERY_NODE_VALUE: + return makeNode(type, sizeof(SValueNode)); + case QUERY_NODE_OPERATOR: + return makeNode(type, sizeof(SOperatorNode)); + case QUERY_NODE_LOGIC_CONDITION: + return makeNode(type, sizeof(SLogicConditionNode)); + case QUERY_NODE_IS_NULL_CONDITION: + return makeNode(type, sizeof(SIsNullCondNode)); + case QUERY_NODE_FUNCTION: + return makeNode(type, sizeof(SFunctionNode)); + case QUERY_NODE_REAL_TABLE: + return makeNode(type, sizeof(SRealTableNode)); + case QUERY_NODE_TEMP_TABLE: + return makeNode(type, sizeof(STempTableNode)); + case QUERY_NODE_JOIN_TABLE: + return makeNode(type, sizeof(SJoinTableNode)); + case QUERY_NODE_GROUPING_SET: + return makeNode(type, sizeof(SGroupingSetNode)); + case QUERY_NODE_ORDER_BY_EXPR: + return makeNode(type, sizeof(SOrderByExprNode)); + case QUERY_NODE_LIMIT: + return makeNode(type, sizeof(SLimitNode)); + case QUERY_NODE_STATE_WINDOW: + return makeNode(type, sizeof(SStateWindowNode)); + case QUERY_NODE_SESSION_WINDOW: + return makeNode(type, sizeof(SSessionWindowNode)); + case QUERY_NODE_INTERVAL_WINDOW: + return makeNode(type, sizeof(SIntervalWindowNode)); + case QUERY_NODE_SET_OPERATOR: + return makeNode(type, sizeof(SSetOperator)); + case QUERY_NODE_SELECT_STMT: + return makeNode(type, sizeof(SSelectStmt)); + case QUERY_NODE_SHOW_STMT: + return makeNode(type, sizeof(SShowStmt)); + default: + break; + } + return NULL; +} + +void nodesDestroyNode(SNode* pNode) { + +} + +SNodeList* nodesMakeList() { + SNodeList* p = calloc(1, sizeof(SNodeList)); + if (NULL == p) { + return NULL; + } + return p; +} + +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) { + if (NULL == pList || NULL == pNode) { + return NULL; + } + SListCell* p = calloc(1, sizeof(SListCell)); + if (NULL == p) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pList; + } + p->pNode = pNode; + if (NULL == pList->pHead) { + pList->pHead = p; + } + if (NULL != pList->pTail) { + pList->pTail->pNext = p; + } + pList->pTail = p; + return pList; +} + +void nodesDestroyList(SNodeList* pList) { + +} + +bool nodesIsTimeorderQuery(const SNode* pQuery) { + +} + +bool nodesIsTimelineQuery(const SNode* pQuery) { + +} \ No newline at end of file diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 04efbef5b3..8999646f6a 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "os.h" #include "osString.h" diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index ed936e90f6..760a66a4fb 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -9,7 +9,6 @@ target_include_directories( target_link_libraries( util PRIVATE os - PUBLIC zlib PUBLIC lz4_static PUBLIC api ) diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index ddd283ae0f..0456d6a2ee 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "tbuffer.h" #include "exception.h" #include "os.h" diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 56919ff99e..ee5bea0ab7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -254,6 +254,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retriev TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") +// mnode-topic +TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with STable not supported yet") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") @@ -410,7 +413,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_LEVEL, "tfs invalid level") TAOS_DEFINE_ERROR(TSDB_CODE_FS_NO_VALID_DISK, "tfs no valid disk") // catalog -TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog interval error") +TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INTERNAL_ERROR, "catalog internal error") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_INVALID_INPUT, "invalid catalog input parameters") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_NOT_READY, "catalog is not ready") TAOS_DEFINE_ERROR(TSDB_CODE_CTG_MEM_ERROR, "catalog memory error") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c7b1350591..12aa77214f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,7 +19,6 @@ #include "tnote.h" #include "tutil.h" #include "ulog.h" -//#include "zlib.h" #define MAX_LOGLINE_SIZE (1000) #define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) @@ -92,7 +91,7 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; -int32_t tqDebugFlag = 131; +int32_t tqDebugFlag = 135; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; diff --git a/source/util/src/tpagedfile.c b/source/util/src/tpagedfile.c index 3373d09876..3cdba580d4 100644 --- a/source/util/src/tpagedfile.c +++ b/source/util/src/tpagedfile.c @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE + #include "tpagedfile.h" #include "thash.h" #include "stddef.h" diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 5cb149d53c..8125f550d0 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -13,35 +13,36 @@ * along with this program. If not, see . */ -#include "os.h" - -#include "taoserror.h" +#define _DEFAULT_SOURCE #include "tqueue.h" +#include "taoserror.h" #include "ulog.h" typedef struct STaosQnode STaosQnode; typedef struct STaosQnode { STaosQnode *next; + STaosQueue *queue; char item[]; } STaosQnode; typedef struct STaosQueue { int32_t itemSize; int32_t numOfItems; - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set - FProcessItem itemFp; - FProcessItems itemsFp; + int32_t threadId; + STaosQnode * head; + STaosQnode * tail; + STaosQueue * next; // for queue set + STaosQset * qset; // for queue set + void * ahandle; // for queue set + FItem itemFp; + FItems itemsFp; pthread_mutex_t mutex; } STaosQueue; typedef struct STaosQset { - STaosQueue *head; - STaosQueue *current; + STaosQueue * head; + STaosQueue * current; pthread_mutex_t mutex; int32_t numOfQueues; int32_t numOfItems; @@ -56,19 +57,23 @@ typedef struct STaosQall { } STaosQall; STaosQueue *taosOpenQueue() { - STaosQueue *queue = calloc(sizeof(STaosQueue), 1); + STaosQueue *queue = calloc(1, sizeof(STaosQueue)); if (queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pthread_mutex_init(&queue->mutex, NULL); + if (pthread_mutex_init(&queue->mutex, NULL) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - uTrace("queue:%p is opened", queue); + queue->threadId = -1; + uDebug("queue:%p is opened", queue); return queue; } -void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp) { +void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) { if (queue == NULL) return; queue->itemFp = itemFp; queue->itemsFp = itemsFp; @@ -77,7 +82,7 @@ void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsF void taosCloseQueue(STaosQueue *queue) { if (queue == NULL) return; STaosQnode *pTemp; - STaosQset *qset; + STaosQset * qset; pthread_mutex_lock(&queue->mutex); STaosQnode *pNode = queue->head; @@ -85,7 +90,9 @@ void taosCloseQueue(STaosQueue *queue) { qset = queue->qset; pthread_mutex_unlock(&queue->mutex); - if (queue->qset) taosRemoveFromQset(qset, queue); + if (queue->qset) { + taosRemoveFromQset(qset, queue); + } while (pNode) { pTemp = pNode; @@ -96,7 +103,7 @@ void taosCloseQueue(STaosQueue *queue) { pthread_mutex_destroy(&queue->mutex); free(queue); - uTrace("queue:%p is closed", queue); + uDebug("queue:%p is closed", queue); } bool taosQueueEmpty(STaosQueue *queue) { @@ -120,19 +127,23 @@ int32_t taosQueueSize(STaosQueue *queue) { } void *taosAllocateQitem(int32_t size) { - STaosQnode *pNode = (STaosQnode *)calloc(sizeof(STaosQnode) + size, 1); + STaosQnode *pNode = calloc(1, sizeof(STaosQnode) + size); + + if (pNode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - if (pNode == NULL) return NULL; uTrace("item:%p, node:%p is allocated", pNode->item, pNode); return (void *)pNode->item; } -void taosFreeQitem(void *param) { - if (param == NULL) return; +void taosFreeQitem(void *pItem) { + if (pItem == NULL) return; - char *temp = (char *)param; + char *temp = pItem; temp -= sizeof(STaosQnode); - uTrace("item:%p, node:%p is freed", param, temp); + uTrace("item:%p, node:%p is freed", pItem, temp); free(temp); } @@ -175,7 +186,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { queue->numOfItems--; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1); code = 1; - uDebug("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); @@ -183,7 +194,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { return code; } -STaosQall *taosAllocateQall() { return calloc(sizeof(STaosQall), 1); } +STaosQall *taosAllocateQall() { return calloc(1, sizeof(STaosQall)); } void taosFreeQall(STaosQall *qall) { free(qall); } @@ -238,7 +249,7 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } STaosQset *taosOpenQset() { - STaosQset *qset = (STaosQset *)calloc(sizeof(STaosQset), 1); + STaosQset *qset = calloc(sizeof(STaosQset), 1); if (qset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -247,7 +258,7 @@ STaosQset *taosOpenQset() { pthread_mutex_init(&qset->mutex, NULL); tsem_init(&qset->sem, 0, 0); - uTrace("qset:%p is opened", qset); + uDebug("qset:%p is opened", qset); return qset; } @@ -268,7 +279,7 @@ void taosCloseQset(STaosQset *qset) { pthread_mutex_destroy(&qset->mutex); tsem_destroy(&qset->sem); free(qset); - uTrace("qset:%p is closed", qset); + uDebug("qset:%p is closed", qset); } // tsem_post 'qset->sem', so that reader threads waiting for it @@ -338,12 +349,12 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { pthread_mutex_unlock(&qset->mutex); - uTrace("queue:%p is removed from qset:%p", queue, qset); + uDebug("queue:%p is removed from qset:%p", queue, qset); } int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } -int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp) { +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp) { STaosQnode *pNode = NULL; int32_t code = 0; @@ -365,6 +376,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FP *ppItem = pNode->item; if (ahandle) *ahandle = queue->ahandle; if (itemFp) *itemFp = queue->itemFp; + queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; @@ -382,7 +394,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FP return code; } -int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp) { +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp) { STaosQueue *queue; int32_t code = 0; @@ -411,7 +423,9 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand queue->tail = NULL; queue->numOfItems = 0; atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems); - for (int32_t j = 1; j < qall->numOfItems; ++j) tsem_wait(&qset->sem); + for (int32_t j = 1; j < qall->numOfItems; ++j) { + tsem_wait(&qset->sem); + } } pthread_mutex_unlock(&queue->mutex); @@ -423,6 +437,65 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand return code; } +int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId) { + STaosQnode *pNode = NULL; + int32_t code = -1; + + tsem_wait(&qset->sem); + + pthread_mutex_lock(&qset->mutex); + + for (int32_t i = 0; i < qset->numOfQueues; ++i) { + if (qset->current == NULL) qset->current = qset->head; + STaosQueue *queue = qset->current; + if (queue) qset->current = queue->next; + if (queue == NULL) break; + if (queue->head == NULL) continue; + if (queue->threadId != -1 && queue->threadId != threadId) { + code = 0; + continue; + } + + pthread_mutex_lock(&queue->mutex); + + if (queue->head) { + pNode = queue->head; + pNode->queue = queue; + queue->threadId = threadId; + *ppItem = pNode->item; + + if (ahandle) *ahandle = queue->ahandle; + if (itemFp) *itemFp = queue->itemFp; + + queue->head = pNode->next; + if (queue->head == NULL) queue->tail = NULL; + queue->numOfItems--; + atomic_sub_fetch_32(&qset->numOfItems, 1); + code = 1; + uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + } + + pthread_mutex_unlock(&queue->mutex); + if (pNode) break; + } + + pthread_mutex_unlock(&qset->mutex); + + return code; +} + +void taosResetQsetThread(STaosQset *qset, void *pItem) { + if (pItem == NULL) return; + STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); + + pthread_mutex_lock(&qset->mutex); + pNode->queue->threadId = -1; + for (int32_t i = 0; i < pNode->queue->numOfItems; ++i) { + tsem_post(&qset->sem); + } + pthread_mutex_unlock(&qset->mutex); +} + int32_t taosGetQueueItemsNumber(STaosQueue *queue) { if (!queue) return 0; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index ed74041712..ca4e5a6f30 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -14,38 +14,46 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tqueue.h" #include "tworker.h" +#include "taoserror.h" +#include "ulog.h" -typedef void* (*ThreadFp)(void *param); +typedef void *(*ThreadFp)(void *param); -int32_t tWorkerInit(SWorkerPool *pool) { +int32_t tQWorkerInit(SQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = calloc(sizeof(SWorker), pool->max); - pthread_mutex_init(&pool->mutex, NULL); - for (int i = 0; i < pool->max; ++i) { - SWorker *worker = pool->workers + i; + pool->workers = calloc(sizeof(SQWorker), pool->max); + if (pool->workers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (pthread_mutex_init(&pool->mutex, NULL)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pool->max; ++i) { + SQWorker *worker = pool->workers + i; worker->id = i; worker->pool = pool; } - uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); + uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); return 0; } -void tWorkerCleanup(SWorkerPool *pool) { - for (int i = 0; i < pool->max; ++i) { - SWorker *worker = pool->workers + i; +void tQWorkerCleanup(SQWorkerPool *pool) { + for (int32_t i = 0; i < pool->max; ++i) { + SQWorker *worker = pool->workers + i; if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { taosQsetThreadResume(pool->qset); } } - for (int i = 0; i < pool->max; ++i) { - SWorker *worker = pool->workers + i; + for (int32_t i = 0; i < pool->max; ++i) { + SQWorker *worker = pool->workers + i; if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { pthread_join(worker->thread, NULL); @@ -56,15 +64,15 @@ void tWorkerCleanup(SWorkerPool *pool) { taosCloseQset(pool->qset); pthread_mutex_destroy(&pool->mutex); - uInfo("worker:%s is closed", pool->name); + uDebug("worker:%s is closed", pool->name); } -static void *tWorkerThreadFp(SWorker *worker) { - SWorkerPool *pool = worker->pool; - FProcessItem fp = NULL; +static void *tQWorkerThreadFp(SQWorker *worker) { + SQWorkerPool *pool = worker->pool; + FItem fp = NULL; - void *msg = NULL; - void *ahandle = NULL; + void * msg = NULL; + void * ahandle = NULL; int32_t code = 0; taosBlockSIGPIPE(); @@ -77,7 +85,7 @@ static void *tWorkerThreadFp(SWorker *worker) { break; } - if (fp) { + if (fp != NULL) { (*fp)(ahandle, msg); } } @@ -85,11 +93,12 @@ static void *tWorkerThreadFp(SWorker *worker) { return NULL; } -STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) { +STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, ThreadFp threadFp) { pthread_mutex_lock(&pool->mutex); STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -99,14 +108,18 @@ STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) // spawn a thread to process queue if (pool->num < pool->max) { do { - SWorker *worker = pool->workers + pool->num; + SQWorker *worker = pool->workers + pool->num; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWorkerThreadFp, worker) != 0) { + if (pthread_create(&worker->thread, &thAttr, threadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); + taosCloseQueue(queue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + queue = NULL; + break; } pthread_attr_destroy(&thAttr); @@ -121,19 +134,73 @@ STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) return queue; } -void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue) { +STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { + return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); +} + +void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } -int32_t tMWorkerInit(SMWorkerPool *pool) { - pool->nextId = 0; - pool->workers = calloc(sizeof(SMWorker), pool->max); - if (pool->workers == NULL) return -1; +int32_t tFWorkerInit(SFWorkerPool *pool) { return tQWorkerInit((SQWorkerPool *)pool); } + +void tFWorkerCleanup(SFWorkerPool *pool) { tQWorkerCleanup(pool); } + +static void *tFWorkerThreadFp(SQWorker *worker) { + SQWorkerPool *pool = worker->pool; + + FItem fp = NULL; + void * msg = NULL; + void * ahandle = NULL; + int32_t code = 0; + + taosBlockSIGPIPE(); + setThreadName(pool->name); + uDebug("worker:%s:%d is running", pool->name, worker->id); + + while (1) { + code = taosReadQitemFromQsetByThread(pool->qset, (void **)&msg, &ahandle, &fp, worker->id); + + if (code < 0) { + uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); + break; + } else if (code == 0) { + // uTrace("worker:%s:%d qset:%p, got no message and continue", pool->name, worker->id, pool->qset); + continue; + } + + if (fp != NULL) { + (*fp)(ahandle, msg); + } + + taosResetQsetThread(pool->qset, msg); + } + + return NULL; +} + +STaosQueue *tFWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { + return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); +} + +void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueue(pool, queue); } + +int32_t tWWorkerInit(SWWorkerPool *pool) { + pool->nextId = 0; + pool->workers = calloc(sizeof(SWWorker), pool->max); + if (pool->workers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (pthread_mutex_init(&pool->mutex, NULL) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - pthread_mutex_init(&pool->mutex, NULL); for (int32_t i = 0; i < pool->max; ++i) { - SMWorker *worker = pool->workers + i; + SWWorker *worker = pool->workers + i; worker->id = i; worker->qall = NULL; worker->qset = NULL; @@ -144,16 +211,18 @@ int32_t tMWorkerInit(SMWorkerPool *pool) { return 0; } -void tMWorkerCleanup(SMWorkerPool *pool) { +void tWWorkerCleanup(SWWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { - SMWorker *worker = pool->workers + i; + SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { - if (worker->qset) taosQsetThreadResume(worker->qset); + if (worker->qset) { + taosQsetThreadResume(worker->qset); + } } } for (int32_t i = 0; i < pool->max; ++i) { - SMWorker *worker = pool->workers + i; + SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { pthread_join(worker->thread, NULL); taosFreeQall(worker->qall); @@ -167,12 +236,12 @@ void tMWorkerCleanup(SMWorkerPool *pool) { uInfo("worker:%s is closed", pool->name); } -static void *tWriteWorkerThreadFp(SMWorker *worker) { - SMWorkerPool *pool = worker->pool; - FProcessItems fp = NULL; +static void *tWWorkerThreadFp(SWWorker *worker) { + SWWorkerPool *pool = worker->pool; + FItems fp = NULL; - void *msg = NULL; - void *ahandle = NULL; + void * msg = NULL; + void * ahandle = NULL; int32_t numOfMsgs = 0; int32_t qtype = 0; @@ -187,7 +256,7 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) { break; } - if (fp) { + if (fp != NULL) { (*fp)(ahandle, worker->qall, numOfMsgs); } } @@ -195,13 +264,14 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) { return NULL; } -STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp) { +STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pthread_mutex_lock(&pool->mutex); - SMWorker *worker = pool->workers + pool->nextId; + SWWorker *worker = pool->workers + pool->nextId; STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -221,17 +291,19 @@ STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems taosCloseQset(worker->qset); taosCloseQueue(queue); pthread_mutex_unlock(&pool->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWriteWorkerThreadFp, worker) != 0) { + if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosFreeQall(worker->qall); taosCloseQset(worker->qset); taosCloseQueue(queue); + terrno = TSDB_CODE_OUT_OF_MEMORY; queue = NULL; } else { uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); @@ -250,7 +322,7 @@ STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems return queue; } -void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue) { +void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 383a00232a..ee0ade03b8 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -45,4 +45,10 @@ target_link_libraries(freelistTest os util gtest gtest_main) add_executable(encodeTest "encodeTest.cpp") target_link_libraries(encodeTest os util gtest gtest_main) - +# queueTest +add_executable(queue_test "queueTest.cpp") +target_link_libraries(queue_test os util gtest_main) +add_test( + NAME queue_test + COMMAND queue_test +) \ No newline at end of file diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp new file mode 100644 index 0000000000..310ae4350e --- /dev/null +++ b/source/util/test/queueTest.cpp @@ -0,0 +1,29 @@ +/** + * @file queue.cpp + * @author slguan (slguan@taosdata.com) + * @brief UTIL module queue tests + * @version 1.0 + * @date 2022-01-27 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "os.h" +#include "tqueue.h" + +class UtilTestQueue : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} +}; + +TEST_F(UtilTestQueue, 01_ReadQitemFromQsetByThread) { + EXPECT_EQ(0, 0); +} \ No newline at end of file diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin new file mode 160000 index 0000000000..4a4d79099b --- /dev/null +++ b/src/connector/grafanaplugin @@ -0,0 +1 @@ +Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d diff --git a/tests/parallel_test/Jenkinsfile b/tests/parallel_test/Jenkinsfile new file mode 100644 index 0000000000..fc2b3562c1 --- /dev/null +++ b/tests/parallel_test/Jenkinsfile @@ -0,0 +1,200 @@ +import hudson.model.Result +import hudson.model.*; +import jenkins.model.CauseOfInterruption +node { +} + +def skipbuild=0 +def win_stop=0 + +def abortPreviousBuilds() { + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() + + for (build in builds) { + if (!build.isBuilding()) { + continue; + } + + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } + + build.doKill() //doTerm(),doKill(),doTerm() + } +} +// abort previous build +abortPreviousBuilds() +def abort_previous(){ + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) +} +def pre_test(){ + sh'hostname' + sh ''' + sudo rmtaos || echo "taosd has not installed" + ''' + sh ''' + killall -9 taosd ||echo "no taosd running" + killall -9 gdb || echo "no gdb running" + killall -9 python3.8 || echo "no python program running" + cd ${WKC} + ''' + script { + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } + else if(env.CHANGE_TARGET == '2.0'){ + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } + else if(env.CHANGE_TARGET == '3.0'){ + sh ''' + cd ${WKC} + git checkout 3.0 + ''' + } + else{ + sh ''' + cd ${WKC} + git checkout develop + ''' + } + } + sh''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + export TZ=Asia/Harbin + date + rm -rf debug + mkdir debug + cd debug + cmake .. > /dev/null + make -j4> /dev/null + + ''' + return 1 +} + +pipeline { + agent none + options { skipDefaultCheckout() } + environment{ + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDengine' + } + stages { + stage('pre_build'){ + agent{label 'slave3_0'} + options { skipDefaultCheckout() } + when { + changeRequest() + } + steps { + script{ + abort_previous() + abortPreviousBuilds() + } + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' + sh''' + cd ${WKC}/debug + ctest + ''' + } + } + } + } + post { + success { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", + body: """ + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${env.BRANCH_NAME}
    • +
  • 构建结果： Successful
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${env.CHANGE_AUTHOR}
    • +
  • 提交信息：${env.CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • + +
    +
+
+ + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + failure { + emailext ( + subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' FAIL", + body: """ + + + + + + + + + + + + +

+ 构建信息 +
+
    +
    +
  • 构建名称>>分支：${env.BRANCH_NAME}
    • +
  • 构建结果： Failure
    • +
  • 构建编号：${BUILD_NUMBER}
    • +
  • 触发用户：${env.CHANGE_AUTHOR}
    • +
  • 提交信息：${env.CHANGE_TITLE}
    • +
  • 构建地址：${BUILD_URL}
    • +
  • 构建日志：${BUILD_URL}console
    • + +
    +
+
+ + """, + to: "${env.CHANGE_AUTHOR_EMAIL}", + from: "support@taosdata.com" + ) + } + } +} diff --git a/tests/script/tsim/dnode/basic1.sim b/tests/script/tsim/dnode/basic1.sim index e689bb3261..6061b6ece1 100644 --- a/tests/script/tsim/dnode/basic1.sim +++ b/tests/script/tsim/dnode/basic1.sim @@ -178,9 +178,9 @@ if $rows != 3 then endi sql select * from st -#if $rows != 15 then -# return -1 -#endi +if $rows != 15 then + return -1 +endi print =============== drop dnode sql drop dnode 2; diff --git a/tests/script/tsim/table/basic1.sim b/tests/script/tsim/table/basic1.sim index f47630b737..776b09b813 100644 --- a/tests/script/tsim/table/basic1.sim +++ b/tests/script/tsim/table/basic1.sim @@ -139,9 +139,9 @@ endi print =============== query data frpm st sql select * from st -#if $rows != 21 then -# return -1 -#endi +if $rows != 21 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start @@ -200,8 +200,8 @@ endi print =============== query data frpm st sql select * from st -#if $rows != 21 then -# return -1 -#endi +if $rows != 21 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index c283423fbf..4186f0dae1 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -360,6 +360,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } } else { int num_rows_affacted = taos_affected_rows(pSql); + taos_free_result(pSql); et = taosGetTimestampUs(); printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); } @@ -965,7 +966,7 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char *cmd = calloc(1, tsMaxSQLStringLen + 1); + char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; size_t line_len = 0; @@ -997,7 +998,7 @@ void source_file(TAOS *con, char *fptr) { } while ((read_len = tgetline(&line, &line_len, f)) != -1) { - if (read_len >= tsMaxSQLStringLen) continue; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -1014,7 +1015,7 @@ void source_file(TAOS *con, char *fptr) { memcpy(cmd + cmd_len, line, read_len); printf("%s%s\n", PROMPT_HEADER, cmd); shellRunCommand(con, cmd); - memset(cmd, 0, tsMaxSQLStringLen); + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; }