diff --git a/.gitignore b/.gitignore index f0067d26ba..b9b5341b06 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ build/ compile_commands.json .cache .ycm_extra_conf.py +.tasks +.vimspector.json .vscode/ .idea/ cmake-build-debug/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0383cc8aed..fd542966cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) add_subdirectory(tests) +add_subdirectory(example) # docs add_subdirectory(docs) diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt new file mode 100644 index 0000000000..71a9f9f4c5 --- /dev/null +++ b/example/CMakeLists.txt @@ -0,0 +1,16 @@ +aux_source_directory(src TMQ_DEMO_SRC) + +add_executable(tmq ${TMQ_DEMO_SRC}) +target_link_libraries( + tmq + PUBLIC taos + #PUBLIC util + #PUBLIC common + #PUBLIC os +) +target_include_directories( + tmq + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +SET_TARGET_PROPERTIES(tmq PROPERTIES OUTPUT_NAME tmq) diff --git a/example/src/tmq.c b/example/src/tmq.c new file mode 100644 index 0000000000..31bfe4197f --- /dev/null +++ b/example/src/tmq.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "taos.h" + +static int running = 1; +static void msg_process(tmq_message_t* message) { + tmqShowMsg(message); +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tu2 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + + const char* sql = "select * from st1"; + pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); + /*if (taos_errno(pRes) != 0) {*/ + /*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/ + /*return -1;*/ + /*}*/ + /*taos_free_result(pRes);*/ + taos_close(pConn); + return 0; +} + +tmq_t* build_consumer() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); + return tmq; + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + tmq_subscribe(tmq, topic_list); + return NULL; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + return topic_list; +} + +void basic_consume_loop(tmq_t *tmq, + tmq_list_t *topics) { + tmq_resp_err_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + clock_t startTime = clock(); + while (running) { + tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 0); + if (tmqmessage) { + cnt++; + /*msg_process(tmqmessage);*/ + tmq_message_destroy(tmqmessage); + } else { + break; + } + } + clock_t endTime = clock(); + printf("log cnt: %d %f s\n", cnt, (double)(endTime - startTime) / CLOCKS_PER_SEC); + + err = tmq_consumer_close(tmq); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void sync_consume_loop(tmq_t *rk, + tmq_list_t *topics) { + static const int MIN_COMMIT_COUNT = 1000; + + int msg_count = 0; + tmq_resp_err_t err; + + if ((err = tmq_subscribe(rk, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + return; + } + + while (running) { + tmq_message_t *tmqmessage = tmq_consumer_poll(rk, 500); + if (tmqmessage) { + msg_process(tmqmessage); + tmq_message_destroy(tmqmessage); + + if ((++msg_count % MIN_COMMIT_COUNT) == 0) + tmq_commit(rk, NULL, 0); + } + } + + err = tmq_consumer_close(rk); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main() { + int code; + code = init_env(); + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + /*sync_consume_loop(tmq, topic_list);*/ +} diff --git a/include/client/taos.h b/include/client/taos.h index 626264d94e..a8627a43da 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -92,17 +92,6 @@ typedef struct taosField { typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); -typedef struct tmq_t tmq_t; -typedef struct tmq_conf_t tmq_conf_t; -typedef struct tmq_list_t tmq_list_t; - -typedef struct tmq_message_t tmq_message_t; -typedef struct tmq_message_topic_t tmq_message_topic_t; -typedef struct tmq_message_tb_t tmq_message_tb_t; -typedef struct tmq_tb_iter_t tmq_tb_iter_t; -typedef struct tmq_message_col_t tmq_message_col_t; -typedef struct tmq_col_iter_t tmq_col_iter_t; - typedef struct TAOS_BIND { int buffer_type; void * buffer; @@ -204,26 +193,66 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); -DLL_EXPORT tmq_list_t* tmq_list_new(); -DLL_EXPORT int32_t tmq_list_append(tmq_list_t*, char*); +/* --------------------------TMQ INTERFACE------------------------------- */ -DLL_EXPORT tmq_conf_t* tmq_conf_new(); +enum tmq_resp_err_t { + TMQ_RESP_ERR__SUCCESS = 0, + TMQ_RESP_ERR__FAIL = 1, +}; -DLL_EXPORT int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value); +typedef enum tmq_resp_err_t tmq_resp_err_t; -DLL_EXPORT TAOS_RES *taos_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen); +typedef struct tmq_t tmq_t; +typedef struct tmq_topic_vgroup_t tmq_topic_vgroup_t; +typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t; -DLL_EXPORT tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen); +typedef struct tmq_conf_t tmq_conf_t; +typedef struct tmq_list_t tmq_list_t; +typedef struct tmq_message_t tmq_message_t; -DLL_EXPORT TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list); +typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); -DLL_EXPORT tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time); +DLL_EXPORT tmq_list_t *tmq_list_new(); +DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); -DLL_EXPORT int32_t tmq_topic_num(tmq_message_t* msg); -DLL_EXPORT char* tmq_get_topic(tmq_message_topic_t* msg); -DLL_EXPORT int32_t tmq_get_vgId(tmq_message_topic_t* msg); -DLL_EXPORT tmq_message_tb_t* tmq_get_next_tb(tmq_message_topic_t* msg, tmq_tb_iter_t* iter); -DLL_EXPORT tmq_message_col_t* tmq_get_next_col(tmq_message_tb_t* msg, tmq_col_iter_t* iter); +DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); +DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT void tmq_message_destroy(tmq_message_t* tmq_message); +DLL_EXPORT const char* tmq_err2str(tmq_resp_err_t); + +/* ------------------------TMQ CONSUMER INTERFACE------------------------ */ +DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); +DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); +#endif +DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); +DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t* tmq); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups); +DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups); +#endif +DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); +#if 0 +DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); +#endif +/* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ + +enum tmq_conf_res_t { + TMQ_CONF_UNKNOWN = -2, + TMQ_CONF_INVALID = -1, + TMQ_CONF_OK = 0, +}; + +typedef enum tmq_conf_res_t tmq_conf_res_t; + +DLL_EXPORT tmq_conf_t *tmq_conf_new(); +DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); +DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); +DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); + +//temporary used function for demo only +void tmqShowMsg(tmq_message_t* tmq_message); #ifdef __cplusplus } diff --git a/include/common/common.h b/include/common/common.h index 056d2789fe..31f905d47f 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -38,6 +38,10 @@ // int16_t bytes; //} SSchema; +#define TMQ_REQ_TYPE_COMMIT_ONLY 0 +#define TMQ_REQ_TYPE_CONSUME_ONLY 1 +#define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 + typedef struct { uint32_t numOfTables; SArray *pGroupList; @@ -115,7 +119,7 @@ static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { buf = taosDecodeFixedI32(buf, &sz); pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); for (int32_t i = 0; i < sz; i++) { - SColumnInfoData data; + SColumnInfoData data = {0}; buf = taosDecodeFixedI16(buf, &data.info.colId); buf = taosDecodeFixedI16(buf, &data.info.type); buf = taosDecodeFixedI16(buf, &data.info.bytes); @@ -130,6 +134,12 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp int32_t tlen = 0; int32_t sz = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += taosEncodeFixedI64(buf, pRsp->committedOffset); + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); + tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); + if (pRsp->numOfTopics == 0) return tlen; tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas); if (pRsp->pBlockData) { sz = taosArrayGetSize(pRsp->pBlockData); @@ -145,19 +155,58 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) { int32_t sz; buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + buf = taosDecodeFixedI64(buf, &pRsp->committedOffset); + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); + buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); + if (pRsp->numOfTopics == 0) return buf; pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); if (pRsp->schemas == NULL) return NULL; buf = tDecodeSSchemaWrapper(buf, pRsp->schemas); buf = taosDecodeFixedI32(buf, &sz); pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { - SSDataBlock block; + SSDataBlock block = {0}; tDecodeDataBlock(buf, &block); taosArrayPush(pRsp->pBlockData, &block); } return buf; } +static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return; + } + + //int32_t numOfOutput = pBlock->info.numOfCols; + int32_t sz = taosArrayGetSize(pBlock->pDataBlock); + for(int32_t i = 0; i < sz; ++i) { + SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + tfree(pColInfoData->pData); + } + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); + //tfree(pBlock); +} + + +static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { + if (pRsp->schemas) { + if (pRsp->schemas->nCols) { + tfree(pRsp->schemas->pSchema); + } + free(pRsp->schemas); + } + taosArrayDestroyEx(pRsp->pBlockData, (void(*)(void*))tDeleteSSDataBlock); + pRsp->pBlockData = NULL; + //for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + //SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); + //tDeleteSSDataBlock(pDataBlock); + //} +} + //====================================================================================================================== // the following structure shared by parser and executor typedef struct SColumn { diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1b96cbdc78..4bc8c9fb86 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -64,7 +64,6 @@ extern int8_t tsKeepOriginalColumnName; extern int8_t tsDeadLockKillQuery; // client -extern int32_t tsMaxSQLStringLen; extern int32_t tsMaxWildCardsLen; extern int32_t tsMaxRegexStringLen; extern int8_t tsTscEnableRecordSql; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index eace9bd90d..c641fbb1a3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -147,6 +147,11 @@ typedef enum _mgmt_table { #define TSDB_COL_IS_NORMAL_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_NORMAL) #define TSDB_COL_IS_UD_COL(f) ((f & (~(TSDB_COL_NULL))) == TSDB_COL_UDC) #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) + +#define TD_SUPER_TABLE TSDB_SUPER_TABLE +#define TD_CHILD_TABLE TSDB_CHILD_TABLE +#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE + typedef struct { int32_t vgId; char* dbFName; @@ -876,13 +881,21 @@ typedef struct { char desc[TSDB_STEP_DESC_LEN]; } SStartupReq; +/** + * The layout of the query message payload is as following: + * +--------------------+---------------------------------+ + * |Sql statement | Physical plan | + * |(denoted by sqlLen) |(In JSON, denoted by contentLen) | + * +--------------------+---------------------------------+ + */ typedef struct SSubQueryMsg { SMsgHead header; uint64_t sId; uint64_t queryId; uint64_t taskId; int8_t taskType; - uint32_t contentLen; + uint32_t sqlLen; // the query sql, + uint32_t phyLen; char msg[]; } SSubQueryMsg; @@ -1146,10 +1159,7 @@ typedef struct SVCreateTbReq { char* name; uint32_t ttl; uint32_t keep; -#define TD_SUPER_TABLE TSDB_SUPER_TABLE -#define TD_CHILD_TABLE TSDB_CHILD_TABLE -#define TD_NORMAL_TABLE TSDB_NORMAL_TABLE - uint8_t type; + uint8_t type; union { struct { tb_uid_t suid; @@ -1194,15 +1204,21 @@ typedef struct { } SVAlterTbRsp; typedef struct { - SMsgHead head; - char name[TSDB_TABLE_FNAME_LEN]; - int64_t suid; + uint64_t ver; + char* name; + uint8_t type; + tb_uid_t suid; } SVDropTbReq; typedef struct { - SMsgHead head; + uint64_t ver; } SVDropTbRsp; +int32_t tSerializeSVDropTbReq(void** buf, SVDropTbReq* pReq); +void* tDeserializeSVDropTbReq(void* buf, SVDropTbReq* pReq); +int32_t tSerializeSVDropTbRsp(void** buf, SVDropTbRsp* pRsp); +void* tDeserializeSVDropTbRsp(void* buf, SVDropTbRsp* pRsp); + typedef struct { SMsgHead head; int64_t uid; @@ -1562,6 +1578,7 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { } typedef struct SMqSetCVgReq { + int64_t leftForVer; int32_t vgId; int64_t oldConsumerId; int64_t newConsumerId; @@ -1578,7 +1595,8 @@ static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* tlen += taosEncodeFixedU64(buf, pMsg->sId); tlen += taosEncodeFixedU64(buf, pMsg->queryId); tlen += taosEncodeFixedU64(buf, pMsg->taskId); - tlen += taosEncodeFixedU32(buf, pMsg->contentLen); + tlen += taosEncodeFixedU32(buf, pMsg->sqlLen); + tlen += taosEncodeFixedU32(buf, pMsg->phyLen); //tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); return tlen; } @@ -1587,13 +1605,15 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { buf = taosDecodeFixedU64(buf, &pMsg->sId); buf = taosDecodeFixedU64(buf, &pMsg->queryId); buf = taosDecodeFixedU64(buf, &pMsg->taskId); - buf = taosDecodeFixedU32(buf, &pMsg->contentLen); + buf = taosDecodeFixedU32(buf, &pMsg->sqlLen); + buf = taosDecodeFixedU32(buf, &pMsg->phyLen); //buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); return buf; } static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pReq->leftForVer); tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); @@ -1602,12 +1622,13 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += taosEncodeString(buf, (char*)pReq->qmsg); + tlen += taosEncodeString(buf, pReq->qmsg); //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { + buf = taosDecodeFixedI64(buf, &pReq->leftForVer); buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); @@ -1616,7 +1637,7 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = taosDecodeString(buf, (char**)&pReq->qmsg); + buf = taosDecodeString(buf, &pReq->qmsg); //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } @@ -1692,6 +1713,10 @@ typedef struct SMqTopicBlk { typedef struct SMqConsumeRsp { int64_t consumerId; SSchemaWrapper* schemas; + int64_t committedOffset; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; int32_t numOfTopics; SArray* pBlockData; //SArray } SMqConsumeRsp; @@ -1725,10 +1750,15 @@ typedef struct SMqSubTopicEp { typedef struct SMqCMGetSubEpRsp { int64_t consumerId; + int64_t epoch; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray } SMqCMGetSubEpRsp; +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { + taosArrayDestroy(pSubTopicEp->vgs); +} + static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); @@ -1742,6 +1772,10 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { return buf; } +static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) { + taosArrayDestroyEx(pRsp->topics, (void (*)(void*)) tDeleteSMqSubTopicEp); +} + static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { int32_t tlen = 0; tlen += taosEncodeString(buf, pTopicEp->topic); @@ -1773,6 +1807,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += taosEncodeFixedI64(buf, pRsp->epoch); tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -1785,6 +1820,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + buf = taosDecodeFixedI64(buf, &pRsp->epoch); buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index cbc8c0e87c..b2a64bfac4 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -207,36 +207,78 @@ #define TK_INTO 189 #define TK_VALUES 190 -#define NEW_TK_UNION 1 -#define NEW_TK_ALL 2 -#define NEW_TK_MINUS 3 -#define NEW_TK_EXCEPT 4 -#define NEW_TK_INTERSECT 5 -#define NEW_TK_NK_PLUS 6 -#define NEW_TK_NK_MINUS 7 -#define NEW_TK_NK_STAR 8 -#define NEW_TK_NK_SLASH 9 -#define NEW_TK_SHOW 10 -#define NEW_TK_DATABASES 11 -#define NEW_TK_NK_ID 12 -#define NEW_TK_NK_LP 13 -#define NEW_TK_NK_RP 14 -#define NEW_TK_NK_COMMA 15 -#define NEW_TK_NK_LITERAL 16 -#define NEW_TK_NK_DOT 17 -#define NEW_TK_SELECT 18 -#define NEW_TK_DISTINCT 19 -#define NEW_TK_AS 20 -#define NEW_TK_FROM 21 -#define NEW_TK_WITH 22 -#define NEW_TK_RECURSIVE 23 -#define NEW_TK_ORDER 24 -#define NEW_TK_BY 25 -#define NEW_TK_ASC 26 -#define NEW_TK_DESC 27 -#define NEW_TK_NULLS 28 -#define NEW_TK_FIRST 29 -#define NEW_TK_LAST 30 +#define NEW_TK_OR 1 +#define NEW_TK_AND 2 +#define NEW_TK_UNION 3 +#define NEW_TK_ALL 4 +#define NEW_TK_MINUS 5 +#define NEW_TK_EXCEPT 6 +#define NEW_TK_INTERSECT 7 +#define NEW_TK_NK_PLUS 8 +#define NEW_TK_NK_MINUS 9 +#define NEW_TK_NK_STAR 10 +#define NEW_TK_NK_SLASH 11 +#define NEW_TK_NK_REM 12 +#define NEW_TK_SHOW 13 +#define NEW_TK_DATABASES 14 +#define NEW_TK_NK_INTEGER 15 +#define NEW_TK_NK_FLOAT 16 +#define NEW_TK_NK_STRING 17 +#define NEW_TK_NK_BOOL 18 +#define NEW_TK_TIMESTAMP 19 +#define NEW_TK_NK_VARIABLE 20 +#define NEW_TK_NK_COMMA 21 +#define NEW_TK_NK_ID 22 +#define NEW_TK_NK_LP 23 +#define NEW_TK_NK_RP 24 +#define NEW_TK_NK_DOT 25 +#define NEW_TK_BETWEEN 26 +#define NEW_TK_NOT 27 +#define NEW_TK_IS 28 +#define NEW_TK_NULL 29 +#define NEW_TK_NK_LT 30 +#define NEW_TK_NK_GT 31 +#define NEW_TK_NK_LE 32 +#define NEW_TK_NK_GE 33 +#define NEW_TK_NK_NE 34 +#define NEW_TK_NK_EQ 35 +#define NEW_TK_LIKE 36 +#define NEW_TK_MATCH 37 +#define NEW_TK_NMATCH 38 +#define NEW_TK_IN 39 +#define NEW_TK_FROM 40 +#define NEW_TK_AS 41 +#define NEW_TK_JOIN 42 +#define NEW_TK_ON 43 +#define NEW_TK_INNER 44 +#define NEW_TK_SELECT 45 +#define NEW_TK_DISTINCT 46 +#define NEW_TK_WHERE 47 +#define NEW_TK_PARTITION 48 +#define NEW_TK_BY 49 +#define NEW_TK_SESSION 50 +#define NEW_TK_STATE_WINDOW 51 +#define NEW_TK_INTERVAL 52 +#define NEW_TK_SLIDING 53 +#define NEW_TK_FILL 54 +#define NEW_TK_VALUE 55 +#define NEW_TK_NONE 56 +#define NEW_TK_PREV 57 +#define NEW_TK_LINEAR 58 +#define NEW_TK_NEXT 59 +#define NEW_TK_GROUP 60 +#define NEW_TK_HAVING 61 +#define NEW_TK_ORDER 62 +#define NEW_TK_SLIMIT 63 +#define NEW_TK_SOFFSET 64 +#define NEW_TK_LIMIT 65 +#define NEW_TK_OFFSET 66 +#define NEW_TK_NK_LR 67 +#define NEW_TK_ASC 68 +#define NEW_TK_DESC 69 +#define NEW_TK_NULLS 70 +#define NEW_TK_FIRST 71 +#define NEW_TK_LAST 72 #define TK_SPACE 300 #define TK_COMMENT 301 @@ -247,6 +289,8 @@ #define TK_FILE 306 #define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query +#define TK_NIL 65535 + #endif diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 26733696d2..d9e1957e5d 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -27,6 +27,10 @@ typedef void* DataSinkHandle; struct SRpcMsg; struct SSubplan; +typedef struct SReadHandle { + void* reader; + void* meta; +} SReadHandle; /** * Create the exec task for streaming mode * @param pMsg @@ -35,7 +39,13 @@ struct SSubplan; */ qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); -int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); +/** + * + * @param tinfo + * @param input + * @return + */ +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input); /** * Create the exec task object according to task json @@ -46,7 +56,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); * @param qId * @return */ -int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, diff --git a/include/libs/function/function.h b/include/libs/function/function.h index b15d5ad33a..aef5f7fec4 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -107,14 +107,14 @@ typedef struct SPoint1 { union{double val; char* ptr;}; } SPoint1; -struct SQLFunctionCtx; +struct SqlFunctionCtx; struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SExtTagsInfo { int16_t tagsLen; // keep the tags data for top/bottom query result int16_t numOfTagCols; - struct SQLFunctionCtx **pTagCtxList; + struct SqlFunctionCtx **pTagCtxList; } SExtTagsInfo; typedef struct SResultDataInfo { @@ -126,18 +126,18 @@ typedef struct SResultDataInfo { #define GET_RES_INFO(ctx) ((ctx)->resultInfo) typedef struct SFunctionFpSet { - bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(struct SQLFunctionCtx *pCtx); + bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment + void (*addInput)(struct SqlFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(struct SQLFunctionCtx *pCtx); - void (*combine)(struct SQLFunctionCtx *pCtx); + void (*finalize)(struct SqlFunctionCtx *pCtx); + void (*combine)(struct SqlFunctionCtx *pCtx); } SFunctionFpSet; extern SFunctionFpSet fpSet[1]; // sql function runtime context -typedef struct SQLFunctionCtx { +typedef struct SqlFunctionCtx { int32_t size; // number of rows void * pInput; // input data buffer uint32_t order; // asc|desc @@ -167,7 +167,7 @@ typedef struct SQLFunctionCtx { int32_t columnIndex; SFunctionFpSet* fpSet; -} SQLFunctionCtx; +} SqlFunctionCtx; enum { TEXPR_NODE_DUMMY = 0x0, @@ -216,14 +216,14 @@ typedef struct SAggFunctionInfo { int8_t sFunctionId; // Transfer function for super table query uint16_t status; - bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SQLFunctionCtx *pCtx); + bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment + void (*addInput)(SqlFunctionCtx *pCtx); // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(SQLFunctionCtx *pCtx); - void (*combine)(SQLFunctionCtx *pCtx); + void (*finalize)(SqlFunctionCtx *pCtx); + void (*combine)(SqlFunctionCtx *pCtx); - int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId); + int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); } SAggFunctionInfo; struct SScalarFuncParam; @@ -279,9 +279,9 @@ void extractFunctionDesc(SArray* pFunctionIdList, SMultiFunctionsDesc* pDesc); tExprNode* exprdup(tExprNode* pTree); -void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num); +void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); -int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num); +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 3c365335be..aba647f5cd 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -22,7 +22,7 @@ extern "C" { #include "nodes.h" -struct SQLFunctionCtx; +struct SqlFunctionCtx; struct SResultRowEntryInfo; struct STimeWindow; @@ -32,9 +32,9 @@ typedef struct SFuncExecEnv { typedef void* FuncMgtHandle; typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv); -typedef bool (*FExecInit)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); -typedef void (*FExecProcess)(struct SQLFunctionCtx *pCtx); -typedef void (*FExecFinalize)(struct SQLFunctionCtx *pCtx); +typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); +typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx); +typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx); typedef struct SFuncExecFuncs { FExecGetEnv getEnv; diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 42d3307ac8..31f5457c90 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -24,7 +24,6 @@ #endif OP_ENUM_MACRO(StreamScan) -OP_ENUM_MACRO(TableScan) OP_ENUM_MACRO(DataBlocksOptScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) @@ -48,3 +47,5 @@ OP_ENUM_MACRO(AllTimeWindow) OP_ENUM_MACRO(AllMultiTableTimeInterval) OP_ENUM_MACRO(Order) OP_ENUM_MACRO(Exchange) + +//OP_ENUM_MACRO(TableScan) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 5a0da46490..e856adaf31 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -72,7 +72,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -80,7 +80,7 @@ int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, str * @param pNodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob); +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob); /** * Fetch query result from the remote query executor diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index e297059bc8..39638b3757 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -22,6 +22,19 @@ extern "C" { #include "tdef.h" +#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) +#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) + +#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) + +#define FOREACH(node, list) \ + for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) + +#define FORBOTH(node1, list1, node2, list2) \ + for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHead : NULL), *cell2 = (NULL != (list2) ? (list2)->pHead : NULL); \ + (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ + cell1 = cell1->pNext, cell2 = cell2->pNext) + typedef enum ENodeType { QUERY_NODE_COLUMN = 1, QUERY_NODE_VALUE, @@ -38,6 +51,8 @@ typedef enum ENodeType { QUERY_NODE_STATE_WINDOW, QUERY_NODE_SESSION_WINDOW, QUERY_NODE_INTERVAL_WINDOW, + QUERY_NODE_NODE_LIST, + QUERY_NODE_FILL, QUERY_NODE_SET_OPERATOR, QUERY_NODE_SELECT_STMT, @@ -52,9 +67,6 @@ typedef struct SNode { ENodeType type; } SNode; -#define nodeType(nodeptr) (((const SNode*)(nodeptr))->type) -#define setNodeType(nodeptr, type) (((SNode*)(nodeptr))->type = (type)) - typedef struct SListCell { SNode* pNode; struct SListCell* pNext; @@ -62,19 +74,10 @@ typedef struct SListCell { typedef struct SNodeList { int16_t length; - SListCell* pHeader; + SListCell* pHead; + SListCell* pTail; } SNodeList; -#define LIST_LENGTH(l) (NULL != (l) ? (l)->length : 0) - -#define FOREACH(node, list) \ - for (SListCell* cell = (NULL != (list) ? (list)->pHeader : NULL); (NULL != cell ? (node = cell->pNode, true) : (node = NULL, false)); cell = cell->pNext) - -#define FORBOTH(node1, list1, node2, list2) \ - for (SListCell* cell1 = (NULL != (list1) ? (list1)->pHeader : NULL), *cell2 = (NULL != (list2) ? (list2)->pHeader : NULL); \ - (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ - cell1 = cell1->pNext, cell2 = cell2->pNext) - typedef struct SDataType { uint8_t type; uint8_t precision; @@ -156,14 +159,19 @@ typedef struct SLogicConditionNode { typedef struct SIsNullCondNode { ENodeType type; // QUERY_NODE_IS_NULL_CONDITION SNode* pExpr; - bool isNot; + bool isNull; } SIsNullCondNode; +typedef struct SNodeListNode { + ENodeType type; // QUERY_NODE_NODE_LIST + SNodeList* pNodeList; +} SNodeListNode; + typedef struct SFunctionNode { SExprNode type; // QUERY_NODE_FUNCTION char functionName[TSDB_FUNC_NAME_LEN]; int32_t funcId; - SNodeList* pParameterList; // SNode + SNodeList* pParameterList; } SFunctionNode; typedef struct STableNode { @@ -241,24 +249,41 @@ typedef struct SSessionWindowNode { typedef struct SIntervalWindowNode { ENodeType type; // QUERY_NODE_INTERVAL_WINDOW - int64_t interval; - int64_t sliding; - int64_t offset; + SNode* pInterval; // SValueNode + SNode* pOffset; // SValueNode + SNode* pSliding; // SValueNode + SNode* pFill; } SIntervalWindowNode; +typedef enum EFillMode { + FILL_MODE_NONE = 1, + FILL_MODE_VALUE, + FILL_MODE_PREV, + FILL_MODE_NULL, + FILL_MODE_LINEAR, + FILL_MODE_NEXT +} EFillMode; + +typedef struct SFillNode { + ENodeType type; // QUERY_NODE_FILL + EFillMode mode; + SNode* pValues; // SNodeListNode +} SFillNode; + typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; bool isStar; SNodeList* pProjectionList; // SNode SNode* pFromTable; - SNode* pWhereCond; + SNode* pWhere; SNodeList* pPartitionByList; // SNode - SNode* pWindowClause; + SNode* pWindow; SNodeList* pGroupByList; // SGroupingSetNode + SNode* pHaving; SNodeList* pOrderByList; // SOrderByExprNode - SLimitNode limit; - SLimitNode slimit; + SNode* pLimit; + SNode* pSlimit; } SSelectStmt; typedef enum ESetOperatorType { @@ -272,10 +297,17 @@ typedef struct SSetOperator { SNode* pRight; } SSetOperator; +SNode* nodesMakeNode(ENodeType type); +void nodesDestroyNode(SNode* pNode); + +SNodeList* nodesMakeList(); +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode); +void nodesDestroyList(SNodeList* pList); + typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); -bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext); +bool nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); @@ -289,10 +321,6 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode); bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery); -SNode* nodesMakeNode(ENodeType type); -void nodesDestroyNode(SNode* pNode); -void nodesDestroyNodeList(SNodeList* pList); - #ifdef __cplusplus } #endif diff --git a/include/os/osEndian.h b/include/os/osEndian.h index e573ba0a75..1f52ece535 100644 --- a/include/os/osEndian.h +++ b/include/os/osEndian.h @@ -30,4 +30,4 @@ static const int32_t endian_test_var = 1; } #endif -#endif /*_TD_OS_ENDIAN_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENDIAN_H_*/ diff --git a/include/os/osMath.h b/include/os/osMath.h index 948fbbf665..3fe46d557e 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -36,25 +36,25 @@ extern "C" { #else - #define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ + #define TSWAP(a, b, c) \ + do { \ + __typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ } while (0) - #define TMAX(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a > __b) ? __a : __b; \ + #define TMAX(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a > __b) ? __a : __b; \ }) - #define TMIN(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a < __b) ? __a : __b; \ + #define TMIN(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a < __b) ? __a : __b; \ }) #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8f49fce558..50af44a94c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -254,6 +254,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) #define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) +#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E7) // dnode #define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 77249e3f0b..d7574466cd 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -30,6 +30,16 @@ extern "C" { #include "tmsgtype.h" #include "trpc.h" #include "query.h" +#include "parser.h" + +#define CHECK_CODE_GOTO(expr, label) \ + do { \ + int32_t code = expr; \ + if (TSDB_CODE_SUCCESS != code) { \ + terrno = code; \ + goto label; \ + } \ + } while (0) #define HEARTBEAT_INTERVAL 1500 // ms @@ -219,6 +229,11 @@ void *doFetchRow(SRequestObj* pRequest); void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); + +int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj** pRequest); + +int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery); + // --- heartbeat // global, called by mgmt int hbMgrInit(); @@ -227,7 +242,7 @@ int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key); -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); +void appHbMgrCleanup(void); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 354df00fa7..b09c3ee9d8 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -384,6 +384,7 @@ static void hbStopThread() { } SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { + return NULL; hbMgrInit(); SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { @@ -425,28 +426,23 @@ SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { return pAppHbMgr; } -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { - if (NULL == pAppHbMgr) { - return; - } - +void appHbMgrCleanup(void) { pthread_mutex_lock(&clientHbMgr.lock); int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); - if (pAppHbMgr == pTarget) { - taosHashCleanup(pTarget->activeInfo); - pTarget->activeInfo = NULL; - taosHashCleanup(pTarget->connInfo); - pTarget->connInfo = NULL; - } + taosHashCleanup(pTarget->activeInfo); + pTarget->activeInfo = NULL; + taosHashCleanup(pTarget->connInfo); + pTarget->connInfo = NULL; } pthread_mutex_unlock(&clientHbMgr.lock); } int hbMgrInit() { + return 0; // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; @@ -464,6 +460,7 @@ int hbMgrInit() { } void hbMgrCleanUp() { + return; hbStopThread(); // destroy all appHbMgr @@ -471,6 +468,7 @@ void hbMgrCleanUp() { if (old == 0) return; pthread_mutex_lock(&clientHbMgr.lock); + appHbMgrCleanup(); taosArrayDestroy(clientHbMgr.appHbMgrs); pthread_mutex_unlock(&clientHbMgr.lock); @@ -501,6 +499,7 @@ int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { + return 0; SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; SHbConnInfo info = {0}; @@ -530,9 +529,6 @@ void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey) { return; } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); - if (atomic_load_32(&pAppHbMgr->connKeyCnt) <= 0) { - appHbMgrCleanup(pAppHbMgr); - } } int hbAddConnInfo(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d7f0afa785..c60b89f78e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,5 +1,4 @@ -#include "../../libs/scheduler/inc/schedulerInt.h" #include "clientInt.h" #include "clientLog.h" #include "parser.h" @@ -13,15 +12,6 @@ #include "tpagedfile.h" #include "tref.h" -#define CHECK_CODE_GOTO(expr, label) \ - do { \ - int32_t code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - terrno = code; \ - goto label; \ - } \ - } while (0) - static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); @@ -119,7 +109,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - p->pAppHbMgr = appHbMgrInit(p, key); + /*p->pAppHbMgr = appHbMgrInit(p, key);*/ taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -239,9 +229,10 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) { + void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = schedulerExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); + int32_t code = schedulerExecJob(pTransporter, NULL, pDag, &pRequest->body.pQueryJob, pRequest->sqlstr, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { @@ -255,524 +246,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) return pRequest->code; } - return schedulerAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, pNodeList, pDag, &pRequest->body.pQueryJob); + return schedulerAsyncExecJob(pTransporter, pNodeList, pDag, pRequest->sqlstr, &pRequest->body.pQueryJob); } - -typedef struct SMqClientVg { - // statistics - int64_t pollCnt; - // offset - int64_t committedOffset; - int64_t currentOffset; - //connection info - int32_t vgId; - SEpSet epSet; -} SMqClientVg; - -typedef struct SMqClientTopic { - // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; - int32_t nextVgIdx; - SArray* vgs; //SArray -} SMqClientTopic; - -typedef struct tmq_resp_err_t { - int32_t code; -} tmq_resp_err_t; - -typedef struct tmq_topic_vgroup_t { - char* topic; - int32_t vgId; - int64_t commitOffset; -} tmq_topic_vgroup_t; - -typedef struct tmq_topic_vgroup_list_t { - int32_t cnt; - int32_t size; - tmq_topic_vgroup_t* elems; -} tmq_topic_vgroup_list_t; - -typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); - -struct tmq_conf_t { - char clientId[256]; - char groupId[256]; - char* ip; - uint16_t port; - tmq_commit_cb* commit_cb; -}; - -tmq_conf_t* tmq_conf_new() { - tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); - return conf; -} - -int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (strcmp(key, "group.id") == 0) { - strcpy(conf->groupId, value); - } - if (strcmp(key, "client.id") == 0) { - strcpy(conf->clientId, value); - } - return 0; -} - -struct tmq_t { - char groupId[256]; - char clientId[256]; - int64_t consumerId; - int64_t status; - tsem_t rspSem; - STscObj* pTscObj; - tmq_commit_cb* commit_cb; - int32_t nextTopicIdx; - SArray* clientTopics; //SArray -}; - -tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - tmq_t* pTmq = calloc(sizeof(tmq_t), 1); - if (pTmq == NULL) { - return NULL; - } - pTmq->pTscObj = (STscObj*)conn; - pTmq->status = 0; - strcpy(pTmq->clientId, conf->clientId); - strcpy(pTmq->groupId, conf->groupId); - pTmq->commit_cb = conf->commit_cb; - tsem_init(&pTmq->rspSem, 0, 0); - pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); - pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); - return pTmq; -} - -struct tmq_list_t { - int32_t cnt; - int32_t tot; - char* elems[]; -}; -tmq_list_t* tmq_list_new() { - tmq_list_t *ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); - if (ptr == NULL) { - return ptr; - } - ptr->cnt = 0; - ptr->tot = 8; - return ptr; -} - -int32_t tmq_list_append(tmq_list_t* ptr, char* src) { - if (ptr->cnt >= ptr->tot-1) return -1; - ptr->elems[ptr->cnt] = strdup(src); - ptr->cnt++; - return 0; -} - - -int32_t tmq_null_cb(void* param, const SDataBuf* pMsg, int32_t code) { - if (code == 0) { - // - } - // - return 0; -} - -TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { - SRequestObj *pRequest = NULL; - int32_t sz = topic_list->cnt; - //destroy ex - taosArrayDestroy(tmq->clientTopics); - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); - - SCMSubscribeReq req; - req.topicNum = sz; - req.consumerId = tmq->consumerId; - req.consumerGroup = strdup(tmq->groupId); - req.topicNames = taosArrayInit(sz, sizeof(void*)); - - for (int i = 0; i < sz; i++) { - char* topicName = topic_list->elems[i]; - - SName name = {0}; - char* dbName = getDbOfConnection(tmq->pTscObj); - tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); - tNameFromString(&name, topicName, T_NAME_TABLE); - - char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); - if (topicFname == NULL) { - - } - tNameExtractFullName(&name, topicFname); - tscDebug("subscribe topic: %s", topicFname); - SMqClientTopic topic = { - .nextVgIdx = 0, - .sql = NULL, - .sqlLen = 0, - .topicId = 0, - .topicName = topicFname, - .vgs = NULL - }; - topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); - taosArrayPush(tmq->clientTopics, &topic); - /*SMqClientTopic topic = {*/ - /*.*/ - /*};*/ - taosArrayPush(req.topicNames, &topicFname); - } - - int tlen = tSerializeSCMSubscribeReq(NULL, &req); - void* buf = malloc(tlen); - if(buf == NULL) { - goto _return; - } - - void* abuf = buf; - tSerializeSCMSubscribeReq(&abuf, &req); - /*printf("formatted: %s\n", dagStr);*/ - - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); - if (pRequest == NULL) { - tscError("failed to malloc sqlObj"); - } - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - /*sendInfo->fp*/ - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - -_return: - /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ - /*}*/ - - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; -} - -void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { - conf->commit_cb = cb; -} - -SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { - tmq_t* pTmq = (void*)param; - SArray* pArray = taosArrayInit(0, sizeof(SKv)); - if (pArray == NULL) { - return NULL; - } - SKv kv = {0}; - kv.key = HEARTBEAT_KEY_MQ_TMP; - - SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg)); - if (pMqHb == NULL) { - return pArray; - } - pMqHb->consumerId = connKey.connId; - SArray* clientTopics = pTmq->clientTopics; - int sz = taosArrayGetSize(clientTopics); - for (int i = 0; i < sz; i++) { - SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); - /*if (pCTopic->vgId == -1) {*/ - /*pMqHb->status = 1;*/ - /*break;*/ - /*}*/ - } - kv.value = pMqHb; - kv.valueLen = sizeof(SMqHbMsg); - taosArrayPush(pArray, &kv); - - return pArray; -} - -tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { - tmq_t* pTmq = malloc(sizeof(tmq_t)); - if (pTmq == NULL) { - return NULL; - } - strcpy(pTmq->groupId, conf->groupId); - strcpy(pTmq->clientId, conf->clientId); - pTmq->pTscObj = (STscObj*)conn; - pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; - - return pTmq; -} - -TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { - STscObj *pTscObj = (STscObj*)taos; - SRequestObj *pRequest = NULL; - SQueryNode *pQueryNode = NULL; - char *pStr = NULL; - - terrno = TSDB_CODE_SUCCESS; - if (taos == NULL || topicName == NULL || sql == NULL) { - tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { - tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - goto _return; - } - - tscDebug("start to create topic, %s", topicName); - - CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); - - SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo* ) pQueryNode; - pQueryStmtInfo->info.continueQuery = true; - - // todo check for invalid sql statement and return with error code - - SSchema *schema = NULL; - int32_t numOfCols = 0; - CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), _return); - - pStr = qDagToString(pRequest->body.pDag); - if(pStr == NULL) { - goto _return; - } - - printf("%s\n", pStr); - - // The topic should be related to a database that the queried table is belonged to. - SName name = {0}; - char dbName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&((SQueryStmtInfo*) pQueryNode)->pTableMetaInfo[0]->name, dbName); - - tNameFromString(&name, dbName, T_NAME_ACCT|T_NAME_DB); - tNameFromString(&name, topicName, T_NAME_TABLE); - - char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; - tNameExtractFullName(&name, topicFname); - - SCMCreateTopicReq req = { - .name = (char*) topicFname, - .igExists = 1, - .physicalPlan = (char*) pStr, - .sql = (char*) sql, - .logicalPlan = "no logic plan", - }; - - int tlen = tSerializeSCMCreateTopicReq(NULL, &req); - void* buf = malloc(tlen); - if(buf == NULL) { - goto _return; - } - - void* abuf = buf; - tSerializeSCMCreateTopicReq(&abuf, &req); - /*printf("formatted: %s\n", dagStr);*/ - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - pRequest->type = TDMT_MND_CREATE_TOPIC; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - -_return: - qDestroyQuery(pQueryNode); - /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ - /*}*/ - - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; -} - -/*typedef SMqConsumeRsp tmq_message_t;*/ - -struct tmq_message_t { - SMqConsumeRsp rsp; -}; - -int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { - SMqConsumeRsp rsp; - tDecodeSMqConsumeRsp(pMsg->pData, &rsp); - int32_t colNum = rsp.schemas->nCols; - for (int32_t i = 0; i < colNum; i++) { - printf("| %s |", rsp.schemas->pSchema[i].name); - } - printf("\n"); - int32_t sz = taosArrayGetSize(rsp.pBlockData); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(rsp.pBlockData, i); - int32_t rows = pDataBlock->info.rows; - for (int32_t j = 0; j < colNum; j++) { - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, j); - for (int32_t k = 0; k < rows; k++) { - void* var = POINTER_SHIFT(pColInfoData->pData, k * pColInfoData->info.bytes); - if (j == 0) printf(" %ld ", *(int64_t*)var); - if (j == 1) printf(" %d ", *(int32_t*)var); - } - } - /*pDataBlock->*/ - } - return 0; -} - -int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { - tmq_t* tmq = (tmq_t*)param; - if (code != 0) { - tsem_post(&tmq->rspSem); - return 0; - } - tscDebug("tmq ask ep cb called"); - bool set = false; - SMqCMGetSubEpRsp rsp; - tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); - int32_t sz = taosArrayGetSize(rsp.topics); - // TODO: lock - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); - for (int32_t i = 0; i < sz; i++) { - SMqClientTopic topic = {0}; - SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); - topic.topicName = strdup(pTopicEp->topic); - int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); - topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); - for (int32_t j = 0; j < vgSz; j++) { - SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); - SMqClientVg clientVg = { - .pollCnt = 0, - .committedOffset = -1, - .currentOffset = -1, - .vgId = pVgEp->vgId, - .epSet = pVgEp->epSet - }; - taosArrayPush(topic.vgs, &clientVg); - set = true; - } - taosArrayPush(tmq->clientTopics, &topic); - } - if(set) tmq->status = 1; - // unlock - tsem_post(&tmq->rspSem); - return 0; -} - -tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { - - if (taosArrayGetSize(tmq->clientTopics) == 0 || tmq->status == 0) { - int32_t tlen = sizeof(SMqCMGetSubEpReq); - SMqCMGetSubEpReq* buf = malloc(tlen); - if (buf == NULL) { - tscError("failed to malloc get subscribe ep buf"); - } - buf->consumerId = htobe64(tmq->consumerId); - strcpy(buf->cgroup, tmq->groupId); - - SRequestObj *pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); - if (pRequest == NULL) { - tscError("failed to malloc subscribe ep request"); - } - - pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - sendInfo->param = tmq; - sendInfo->fp = tmq_ask_ep_cb; - - SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - tsem_wait(&tmq->rspSem); - } - - if (taosArrayGetSize(tmq->clientTopics) == 0) { - tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); - return NULL; - } - - SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); - pReq->reqType = 1; - pReq->blockingTime = blocking_time; - pReq->consumerId = tmq->consumerId; - tmq_message_t* tmq_message = NULL; - strcpy(pReq->cgroup, tmq->groupId); - - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); - tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); - strcpy(pReq->topic, pTopic->topicName); - int32_t nextVgIdx = pTopic->nextVgIdx; - pTopic->nextVgIdx = (nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, nextVgIdx); - pReq->offset = pVg->currentOffset; - - pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); - - SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); - pRequest->body.requestMsg = (SDataBuf){ .pData = pReq, .len = sizeof(SMqConsumeReq) }; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - sendInfo->requestObjRefId = 0; - /*sendInfo->param = &tmq_message;*/ - sendInfo->fp = tmq_poll_cb_inner; - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - - return tmq_message; - - /*tsem_wait(&pRequest->body.rspSem);*/ - - /*if (body != NULL) {*/ - /*destroySendMsgInfo(body);*/ - /*}*/ - - /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ - /*pRequest->code = terrno;*/ - /*}*/ - - /*return pRequest;*/ -} - -tmq_resp_err_t* tmq_commit(tmq_t* tmq, tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { - SMqConsumeReq req = {0}; - return NULL; -} - -void tmq_message_destroy(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return; -} - - TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { STscObj *pTscObj = (STscObj *)taos; - if (sqlLen > (size_t) tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); + if (sqlLen > (size_t) TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; } @@ -1059,8 +539,12 @@ void* doFetchRow(SRequestObj* pRequest) { tsem_wait(&pRequest->body.rspSem); pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; - } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE && pResultInfo->pData != NULL) { - return NULL; + } else if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { + epSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + if (pResultInfo->completed) { + return NULL; + } } SMsgSendInfo* body = buildMsgInfoImpl(pRequest); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index bdf3115981..b29813be03 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -78,7 +78,7 @@ void taos_close(TAOS* taos) { STscObj *pTscObj = (STscObj *)taos; tscDebug("0x%"PRIx64" try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); - taosRemoveRef(clientConnRefPool, pTscObj->id); + /*taosRemoveRef(clientConnRefPool, pTscObj->id);*/ } int taos_errno(TAOS_RES *tres) { @@ -343,4 +343,4 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { // TODO -} \ No newline at end of file +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 35d8a2a2ba..0652618e6c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -76,7 +76,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->connType = HEARTBEAT_TYPE_QUERY; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY); + /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY);*/ // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, @@ -201,6 +201,7 @@ int32_t processRetrieveMnodeRsp(void* param, const SDataBuf* pMsg, int32_t code) pResInfo->pRspMsg = pMsg->pData; pResInfo->numOfRows = pRetrieve->numOfRows; pResInfo->pData = pRetrieve->data; + pResInfo->completed = pRetrieve->completed; pResInfo->current = 0; setResultDataPtr(pResInfo, pResInfo->fields, pResInfo->numOfCols, pResInfo->numOfRows); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c new file mode 100644 index 0000000000..e57901ed2e --- /dev/null +++ b/source/client/src/tmq.c @@ -0,0 +1,800 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE + +#include "clientInt.h" +#include "clientLog.h" +#include "parser.h" +#include "planner.h" +#include "scheduler.h" +#include "tdef.h" +#include "tep.h" +#include "tglobal.h" +#include "tmsgtype.h" +#include "tnote.h" +#include "tpagedfile.h" +#include "tref.h" + +struct tmq_list_t { + int32_t cnt; + int32_t tot; + char* elems[]; +}; +struct tmq_topic_vgroup_t { + char* topic; + int32_t vgId; + int64_t offset; +}; + +struct tmq_topic_vgroup_list_t { + int32_t cnt; + int32_t size; + tmq_topic_vgroup_t* elems; +}; + +struct tmq_conf_t { + char clientId[256]; + char groupId[256]; + /*char* ip;*/ + /*uint16_t port;*/ + tmq_commit_cb* commit_cb; +}; + +struct tmq_t { + char groupId[256]; + char clientId[256]; + SRWLatch lock; + int64_t consumerId; + int64_t epoch; + int64_t status; + tsem_t rspSem; + STscObj* pTscObj; + tmq_commit_cb* commit_cb; + int32_t nextTopicIdx; + SArray* clientTopics; // SArray + // stat + int64_t pollCnt; +}; + +struct tmq_message_t { + SMqConsumeRsp rsp; +}; + +typedef struct SMqClientVg { + // statistics + int64_t pollCnt; + // offset + int64_t committedOffset; + int64_t currentOffset; + // connection info + int32_t vgId; + SEpSet epSet; +} SMqClientVg; + +typedef struct SMqClientTopic { + // subscribe info + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + int32_t nextVgIdx; + SArray* vgs; // SArray +} SMqClientTopic; + +typedef struct SMqSubscribeCbParam { + tmq_t* tmq; + tsem_t rspSem; + tmq_resp_err_t rspErr; +} SMqSubscribeCbParam; + +typedef struct SMqAskEpCbParam { + tmq_t* tmq; + int32_t wait; +} SMqAskEpCbParam; + +typedef struct SMqConsumeCbParam { + tmq_t* tmq; + SMqClientVg* pVg; + tmq_message_t** retMsg; + tsem_t rspSem; +} SMqConsumeCbParam; + +typedef struct SMqCommitCbParam { + tmq_t* tmq; + SMqClientVg* pVg; + int32_t async; + tsem_t rspSem; +} SMqCommitCbParam; + +tmq_conf_t* tmq_conf_new() { + tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + return conf; +} + +void tmq_conf_destroy(tmq_conf_t* conf) { + if (conf) free(conf); +} + +tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { + if (strcmp(key, "group.id") == 0) { + strcpy(conf->groupId, value); + } + if (strcmp(key, "client.id") == 0) { + strcpy(conf->clientId, value); + } + return TMQ_CONF_OK; +} + +tmq_list_t* tmq_list_new() { + tmq_list_t* ptr = malloc(sizeof(tmq_list_t) + 8 * sizeof(char*)); + if (ptr == NULL) { + return ptr; + } + ptr->cnt = 0; + ptr->tot = 8; + return ptr; +} + +int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { + if (ptr->cnt >= ptr->tot - 1) return -1; + ptr->elems[ptr->cnt] = strdup(src); + ptr->cnt++; + return 0; +} + +int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; + pParam->rspErr = code; + tsem_post(&pParam->rspSem); + return 0; +} + +int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqCommitCbParam* pParam = (SMqCommitCbParam*) param; + tmq_resp_err_t rspErr = code == 0 ? TMQ_RESP_ERR__SUCCESS : TMQ_RESP_ERR__FAIL; + if (pParam->tmq->commit_cb) { + pParam->tmq->commit_cb(pParam->tmq, rspErr, NULL, NULL); + } + if (!pParam->async) tsem_post(&pParam->rspSem); + return 0; +} + +tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + tmq_t* pTmq = calloc(sizeof(tmq_t), 1); + if (pTmq == NULL) { + return NULL; + } + pTmq->pTscObj = (STscObj*)conn; + pTmq->status = 0; + pTmq->pollCnt = 0; + pTmq->epoch = 0; + taosInitRWLatch(&pTmq->lock); + strcpy(pTmq->clientId, conf->clientId); + strcpy(pTmq->groupId, conf->groupId); + pTmq->commit_cb = conf->commit_cb; + tsem_init(&pTmq->rspSem, 0, 0); + pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); + pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + return pTmq; +} + +tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { + SRequestObj* pRequest = NULL; + int32_t sz = topic_list->cnt; + // destroy ex + taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + + SCMSubscribeReq req; + req.topicNum = sz; + req.consumerId = tmq->consumerId; + req.consumerGroup = strdup(tmq->groupId); + req.topicNames = taosArrayInit(sz, sizeof(void*)); + + for (int i = 0; i < sz; i++) { + char* topicName = topic_list->elems[i]; + + SName name = {0}; + char* dbName = getDbOfConnection(tmq->pTscObj); + tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + if (topicFname == NULL) { + } + tNameExtractFullName(&name, topicFname); + tscDebug("subscribe topic: %s", topicFname); + SMqClientTopic topic = { + .nextVgIdx = 0, + .sql = NULL, + .sqlLen = 0, + .topicId = 0, + .topicName = topicFname, + .vgs = NULL + }; + topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); + taosArrayPush(tmq->clientTopics, &topic); + /*SMqClientTopic topic = {*/ + /*.*/ + /*};*/ + taosArrayPush(req.topicNames, &topicFname); + free(dbName); + } + + int tlen = tSerializeSCMSubscribeReq(NULL, &req); + void* buf = malloc(tlen); + if (buf == NULL) { + goto _return; + } + + void* abuf = buf; + tSerializeSCMSubscribeReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); + if (pRequest == NULL) { + tscError("failed to malloc sqlObj"); + } + + SMqSubscribeCbParam param = {.rspErr = TMQ_RESP_ERR__SUCCESS, .tmq = tmq}; + tsem_init(¶m.rspSem, 0, 0); + + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->param = ¶m; + sendInfo->fp = tmqSubscribeCb; + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(¶m.rspSem); + tsem_destroy(¶m.rspSem); + +_return: + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + return param.rspErr; +} + +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } + +SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { + tmq_t* pTmq = (void*)param; + SArray* pArray = taosArrayInit(0, sizeof(SKv)); + if (pArray == NULL) { + return NULL; + } + SKv kv = {0}; + kv.key = HEARTBEAT_KEY_MQ_TMP; + + SMqHbMsg* pMqHb = malloc(sizeof(SMqHbMsg)); + if (pMqHb == NULL) { + return pArray; + } + pMqHb->consumerId = connKey.connId; + SArray* clientTopics = pTmq->clientTopics; + int sz = taosArrayGetSize(clientTopics); + for (int i = 0; i < sz; i++) { + SMqClientTopic* pCTopic = taosArrayGet(clientTopics, i); + /*if (pCTopic->vgId == -1) {*/ + /*pMqHb->status = 1;*/ + /*break;*/ + /*}*/ + } + kv.value = pMqHb; + kv.valueLen = sizeof(SMqHbMsg); + taosArrayPush(pArray, &kv); + + return pArray; +} + +TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { + STscObj* pTscObj = (STscObj*)taos; + SRequestObj* pRequest = NULL; + SQueryNode* pQueryNode = NULL; + char* pStr = NULL; + + terrno = TSDB_CODE_SUCCESS; + if (taos == NULL || topicName == NULL || sql == NULL) { + tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { + tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); + terrno = TSDB_CODE_TSC_INVALID_INPUT; + goto _return; + } + + if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { + tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); + terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + goto _return; + } + + tscDebug("start to create topic, %s", topicName); + + CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); + CHECK_CODE_GOTO(parseSql(pRequest, &pQueryNode), _return); + + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*)pQueryNode; + pQueryStmtInfo->info.continueQuery = true; + + // todo check for invalid sql statement and return with error code + + SSchema* schema = NULL; + int32_t numOfCols = 0; + CHECK_CODE_GOTO(qCreateQueryDag(pQueryNode, &pRequest->body.pDag, &schema, &numOfCols, NULL, pRequest->requestId), + _return); + + pStr = qDagToString(pRequest->body.pDag); + if (pStr == NULL) { + goto _return; + } + + /*printf("%s\n", pStr);*/ + + // The topic should be related to a database that the queried table is belonged to. + SName name = {0}; + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&((SQueryStmtInfo*)pQueryNode)->pTableMetaInfo[0]->name, dbName); + + tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; + tNameExtractFullName(&name, topicFname); + + SCMCreateTopicReq req = { + .name = (char*)topicFname, + .igExists = 1, + .physicalPlan = (char*)pStr, + .sql = (char*)sql, + .logicalPlan = (char*)"no logic plan", + }; + + int tlen = tSerializeSCMCreateTopicReq(NULL, &req); + void* buf = malloc(tlen); + if (buf == NULL) { + goto _return; + } + + void* abuf = buf; + tSerializeSCMCreateTopicReq(&abuf, &req); + /*printf("formatted: %s\n", dagStr);*/ + + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + pRequest->type = TDMT_MND_CREATE_TOPIC; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(&pRequest->body.rspSem); + +_return: + qDestroyQuery(pQueryNode); + /*if (sendInfo != NULL) {*/ + /*destroySendMsgInfo(sendInfo);*/ + /*}*/ + + if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { + pRequest->code = terrno; + } + + return pRequest; +} + +static char* formatTimestamp(char* buf, int64_t val, int precision) { + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm* ptm = localtime(&tt); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} + +void tmqShowMsg(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return; + + static bool noPrintSchema; + char pBuf[128]; + SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + int32_t colNum = pRsp->schemas->nCols; + if (!noPrintSchema) { + printf("|"); + for (int32_t i = 0; i < colNum; i++) { + if (i == 0) + printf(" %25s |", pRsp->schemas->pSchema[i].name); + else + printf(" %15s |", pRsp->schemas->pSchema[i].name); + } + printf("\n"); + printf("===============================================\n"); + noPrintSchema = true; + } + int32_t sz = taosArrayGetSize(pRsp->pBlockData); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pRsp->pBlockData, i); + int32_t rows = pDataBlock->info.rows; + for (int32_t j = 0; j < rows; j++) { + printf("|"); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + printf(" %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t*)var); + break; + } + } + printf("\n"); + } + } +} + +int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqConsumeCbParam* pParam = (SMqConsumeCbParam*)param; + SMqClientVg* pVg = pParam->pVg; + if (code != 0) { + /*printf("msg discard\n");*/ + tsem_post(&pParam->rspSem); + return 0; + } + + SMqConsumeRsp* pRsp = calloc(1, sizeof(SMqConsumeRsp)); + if (pRsp == NULL) { + tsem_post(&pParam->rspSem); + return -1; + } + tDecodeSMqConsumeRsp(pMsg->pData, pRsp); + /*printf("rsp %ld %ld %d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ + if (pRsp->numOfTopics == 0) { + /*printf("no data\n");*/ + free(pRsp); + tsem_post(&pParam->rspSem); + return 0; + } + *pParam->retMsg = (tmq_message_t*)pRsp; + pVg->currentOffset = pRsp->rspOffset; + /*printf("rsp offset: %ld\n", rsp.rspOffset);*/ + /*printf("-----msg begin----\n");*/ + tsem_post(&pParam->rspSem); + /*printf("\n-----msg end------\n");*/ + return 0; +} + +int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { + SMqAskEpCbParam* pParam = (SMqAskEpCbParam*)param; + tmq_t* tmq = pParam->tmq; + if (code != 0) { + printf("get topic endpoint error, not ready, wait:%d\n", pParam->wait); + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } + return 0; + } + tscDebug("tmq ask ep cb called"); + bool set = false; + SMqCMGetSubEpRsp rsp; + tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); + int32_t sz = taosArrayGetSize(rsp.topics); + // TODO: lock + /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ + /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ + if (rsp.epoch != tmq->epoch) { + // TODO + if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); + topic.topicName = strdup(pTopicEp->topic); + int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); + topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); + for (int32_t j = 0; j < vgSz; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + SMqClientVg clientVg = { + .pollCnt = 0, .committedOffset = -1, .currentOffset = -1, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet}; + taosArrayPush(topic.vgs, &clientVg); + set = true; + } + taosArrayPush(tmq->clientTopics, &topic); + } + tmq->epoch = rsp.epoch; + } + if (set) { + atomic_store_64(&tmq->status, 1); + } + // unlock + /*tsem_post(&tmq->rspSem);*/ + if (pParam->wait) { + tsem_post(&tmq->rspSem); + } + tDeleteSMqCMGetSubEpRsp(&rsp); + return 0; +} + +int32_t tmqAsyncAskEp(tmq_t* tmq, bool wait) { + int32_t tlen = sizeof(SMqCMGetSubEpReq); + SMqCMGetSubEpReq* buf = malloc(tlen); + if (buf == NULL) { + tscError("failed to malloc get subscribe ep buf"); + goto END; + } + buf->consumerId = htobe64(tmq->consumerId); + strcpy(buf->cgroup, tmq->groupId); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); + if (pRequest == NULL) { + tscError("failed to malloc subscribe ep request"); + goto END; + } + + pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; + + SMqAskEpCbParam *pParam = malloc(sizeof(SMqAskEpCbParam)); + if (pParam == NULL) { + tscError("failed to malloc subscribe param"); + goto END; + } + pParam->tmq = tmq; + pParam->wait = wait; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqAskEpCb; + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + +END: + if (wait) tsem_wait(&tmq->rspSem); + return 0; +} + +SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blocking_time, int32_t type, SMqClientTopic* pTopic, + SMqClientVg* pVg) { + SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq)); + if (pReq == NULL) { + return NULL; + } + pReq->reqType = type; + strcpy(pReq->topic, pTopic->topicName); + pReq->blockingTime = blocking_time; + pReq->consumerId = tmq->consumerId; + strcpy(pReq->cgroup, tmq->groupId); + + if (type == TMQ_REQ_TYPE_COMMIT_ONLY) { + pReq->offset = pVg->currentOffset; + } else { + pReq->offset = pVg->currentOffset + 1; + } + + pReq->head.vgId = htonl(pVg->vgId); + pReq->head.contLen = htonl(sizeof(SMqConsumeReq)); + return pReq; +} + +tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { + tmq_message_t* tmq_message = NULL; + + int64_t status = atomic_load_64(&tmq->status); + tmqAsyncAskEp(tmq, status == 0); + + if (blocking_time < 0) blocking_time = 1; + if (blocking_time > 1000) blocking_time = 1000; + /*blocking_time = 1;*/ + + if (taosArrayGetSize(tmq->clientTopics) == 0) { + tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); + usleep(blocking_time * 1000); + return NULL; + } + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); + if (taosArrayGetSize(pTopic->vgs) == 0) { + usleep(blocking_time * 1000); + return NULL; + } + + tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); + pTopic->nextVgIdx = (pTopic->nextVgIdx + 1 % taosArrayGetSize(pTopic->vgs)); + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); + if (pReq == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + + SMqConsumeCbParam* param = malloc(sizeof(SMqConsumeCbParam)); + if (param == NULL) { + usleep(blocking_time * 1000); + return NULL; + } + param->tmq = tmq; + param->retMsg = &tmq_message; + param->pVg = pVg; + tsem_init(¶m->rspSem, 0, 0); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = param; + sendInfo->fp = tmqPollCb; + + /*printf("req offset: %ld\n", pReq->offset);*/ + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + tmq->pollCnt++; + + tsem_wait(¶m->rspSem); + tsem_destroy(¶m->rspSem); + free(param); + + if (tmq_message == NULL) { + usleep(blocking_time * 1000); + } + + return tmq_message; + + /*tsem_wait(&pRequest->body.rspSem);*/ + + /*if (body != NULL) {*/ + /*destroySendMsgInfo(body);*/ + /*}*/ + + /*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/ + /*pRequest->code = terrno;*/ + /*}*/ + + /*return pRequest;*/ +} + +tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) { + + if (tmq_topic_vgroup_list != NULL) { + //TODO + } + + //TODO: change semaphore to gate + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, TMQ_REQ_TYPE_COMMIT_ONLY, pTopic, pVg); + + SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); + pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)}; + SMqCommitCbParam *pParam = malloc(sizeof(SMqCommitCbParam)); + if (pParam == NULL) { + continue; + } + pParam->tmq = tmq; + pParam->pVg = pVg; + pParam->async = async; + if (!async) tsem_init(&pParam->rspSem, 0, 0); + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqCommitCb; + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); + + if (!async) tsem_wait(&pParam->rspSem); + } + } + + return 0; +} + +void tmq_message_destroy(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return; + SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + tDeleteSMqConsumeRsp(pRsp); + free(tmq_message); +} + +tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { + return TMQ_RESP_ERR__SUCCESS; +} + +const char* tmq_err2str(tmq_resp_err_t err) { + if (err == TMQ_RESP_ERR__SUCCESS) { + return "success"; + } + return "fail"; +} +#if 0 +tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { + tmq_t* pTmq = malloc(sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + strcpy(pTmq->groupId, conf->groupId); + strcpy(pTmq->clientId, conf->clientId); + pTmq->pTscObj = (STscObj*)conn; + pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; + return pTmq; +} + + +static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { + assert(pMsgBody != NULL); + tfree(pMsgBody->msgInfo.pData); + tfree(pMsgBody); +} +#endif diff --git a/source/client/test/CMakeLists.txt b/source/client/test/CMakeLists.txt index 6886206363..3614e4364b 100644 --- a/source/client/test/CMakeLists.txt +++ b/source/client/test/CMakeLists.txt @@ -8,7 +8,7 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) ADD_EXECUTABLE(clientTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( clientTest - PUBLIC os util common transport gtest taos qcom + PUBLIC os util common transport parser catalog scheduler function gtest taos qcom ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 3d57a5de1d..553dacafdd 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -53,6 +53,7 @@ TEST(testCase, driverInit_Test) { // taos_init(); } +#if 0 TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); if (pConn == NULL) { @@ -561,9 +562,9 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } +#endif -#if 0 -TEST(testCase, create_topic_Test) { +TEST(testCase, create_topic_ctb_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -582,13 +583,37 @@ TEST(testCase, create_topic_Test) { taos_free_result(pRes); char* sql = "select * from tu"; - pRes = taos_create_topic(pConn, "test_topic_1", sql, strlen(sql)); + pRes = tmq_create_topic(pConn, "test_ctb_topic_1", sql, strlen(sql)); taos_free_result(pRes); taos_close(pConn); } +TEST(testCase, create_topic_stb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -TEST(testCase, tmq_subscribe_Test) { + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + //taos_free_result(pRes); + + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + ASSERT_TRUE(pFields == nullptr); + + int32_t numOfFields = taos_num_fields(pRes); + ASSERT_EQ(numOfFields, 0); + + taos_free_result(pRes); + + char* sql = "select * from st1"; + pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); + taos_free_result(pRes); + taos_close(pConn); +} + +#if 0 +TEST(testCase, tmq_subscribe_ctb_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); assert(pConn != NULL); @@ -600,26 +625,60 @@ TEST(testCase, tmq_subscribe_Test) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg1"); - tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "test_topic_1"); + tmq_list_append(topic_list, "test_ctb_topic_1"); tmq_subscribe(tmq, topic_list); while (1) { - tmq_message_t* msg = tmq_consume_poll(tmq, 0); - printf("get msg\n"); + tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); + tmq_message_destroy(msg); + //printf("get msg\n"); //if (msg == NULL) break; } } #endif +TEST(testCase, tmq_subscribe_stb_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); + + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "test_stb_topic_1"); + tmq_subscribe(tmq, topic_list); + + int cnt = 1; + while (1) { + tmq_message_t* msg = tmq_consumer_poll(tmq, 1000); + if (msg == NULL) continue; + tmqShowMsg(msg); + if (cnt++ % 10 == 0){ + tmq_commit(tmq, NULL, 0); + } + //tmq_commit(tmq, NULL, 0); + tmq_message_destroy(msg); + //printf("get msg\n"); + } +} + TEST(testCase, tmq_consume_Test) { } TEST(testCase, tmq_commit_TEST) { } +#if 0 TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); @@ -732,5 +791,6 @@ TEST(testCase, agg_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif #pragma GCC diagnostic pop diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index a353c018b7..afba1190ba 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -78,7 +78,6 @@ int32_t tsCompressColData = -1; int32_t tsCompatibleModel = 1; // client -int32_t tsMaxSQLStringLen = TSDB_MAX_ALLOWED_SQL_LEN; int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; @@ -594,16 +593,6 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosAddConfigOption(cfg); - cfg.option = "maxSQLLength"; - cfg.ptr = &tsMaxSQLStringLen; - cfg.valType = TAOS_CFG_VTYPE_INT32; - cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT | TSDB_CFG_CTYPE_B_SHOW; - cfg.minValue = TSDB_MAX_SQL_LEN; - cfg.maxValue = TSDB_MAX_ALLOWED_SQL_LEN; - cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_BYTE; - taosAddConfigOption(cfg); - cfg.option = "maxWildCardsLength"; cfg.ptr = &tsMaxWildCardsLen; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 34b55fd812..3fabd2ce0d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -320,3 +320,18 @@ void *tSVCreateTbBatchReqDeserialize(void *buf, SVCreateTbBatchReq *pReq) { return buf; } + +int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pReq->ver); + tlen += taosEncodeString(buf, pReq->name); + tlen += taosEncodeFixedU8(buf, pReq->type); + return tlen; +} + +void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { + buf = taosDecodeFixedU64(buf, &pReq->ver); + buf = taosDecodeString(buf, &pReq->name); + buf = taosDecodeFixedU8(buf, &pReq->type); + return buf; +} diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 73f755bf1a..1cab413c3f 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -630,14 +630,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -#define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ - } while (0) - - void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { case TSDB_DATA_TYPE_INT: diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 17092ed737..11b32fbf0f 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -199,19 +199,16 @@ TEST_F(DndTestVnode, 03_Create_Stb) { req.stbCfg.nTagCols = 3; req.stbCfg.pTagSchema = &schemas[2]; - int32_t bsize = tSerializeSVCreateTbReq(NULL, &req); - void* buf = rpcMallocCont(sizeof(SMsgHead) + bsize); - SMsgHead* pMsgHead = (SMsgHead*)buf; + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(2); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); - void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - int32_t contLen = sizeof(SMsgHead) + bsize; - - SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, buf, contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_VND_CREATE_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); if (i == 0) { ASSERT_EQ(pRsp->code, 0); @@ -235,20 +232,28 @@ TEST_F(DndTestVnode, 04_ALTER_Stb) { } TEST_F(DndTestVnode, 05_DROP_Stb) { -#if 0 { for (int i = 0; i < 3; ++i) { - SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, pReq, contLen); + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char*)"stb1"; + req.suid = 9599; + req.type = TD_SUPER_TABLE; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead* pHead = (SMsgHead*)rpcMallocCont(contLen); + + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(2); + + void* pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + SRpcMsg* pRsp = test.SendReq(TDMT_VND_DROP_STB, (void*)pHead, contLen); ASSERT_NE(pRsp, nullptr); - if (i == 0) { - ASSERT_EQ(pRsp->code, 0); - test.Restart(); - } else { - ASSERT_EQ(pRsp->code, TSDB_CODE_TDB_INVALID_TABLE_ID); - } + ASSERT_EQ(pRsp->code, 0); } } -#endif } TEST_F(DndTestVnode, 06_DROP_Vnode) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 00d1b61b74..75f90df658 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -393,6 +393,12 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu return buf; } +static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { + if (pConsumerEp) { + tfree(pConsumerEp->qmsg); + } +} + // unit for rebalance typedef struct SMqSubscribeObj { char key[TSDB_SUBSCRIBE_KEY_LEN]; @@ -520,7 +526,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->assigned, &cEp); } @@ -533,7 +539,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->lostConsumer, &cEp); } @@ -547,7 +553,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->idleConsumer, &cEp); } @@ -563,7 +569,7 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return NULL; } for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp cEp; + SMqConsumerEp cEp = {0}; buf = tDecodeSMqConsumerEp(buf, &cEp); taosArrayPush(pSub->unassignedVg, &cEp); } @@ -571,6 +577,33 @@ static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) return buf; } +static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { + if (pSub->availConsumer) { + taosArrayDestroy(pSub->availConsumer); + pSub->availConsumer = NULL; + } + if (pSub->assigned) { + //taosArrayDestroyEx(pSub->assigned, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->assigned); + pSub->assigned = NULL; + } + if (pSub->unassignedVg) { + //taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->unassignedVg); + pSub->unassignedVg = NULL; + } + if (pSub->idleConsumer) { + //taosArrayDestroyEx(pSub->idleConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->idleConsumer); + pSub->idleConsumer = NULL; + } + if (pSub->lostConsumer) { + //taosArrayDestroyEx(pSub->lostConsumer, (void (*)(void*))tDeleteSMqConsumerEp); + taosArrayDestroy(pSub->lostConsumer); + pSub->lostConsumer = NULL; + } +} + typedef struct SMqCGroup { char name[TSDB_CONSUMER_GROUP_LEN]; int32_t status; // 0 - uninitialized, 1 - wait rebalance, 2- normal @@ -604,7 +637,7 @@ typedef struct SMqConsumerTopic { } SMqConsumerTopic; static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqTopicObj* pTopic, - SMqSubscribeObj* pSub) { + SMqSubscribeObj* pSub, int64_t* oldConsumerId) { SMqConsumerTopic* pCTopic = malloc(sizeof(SMqConsumerTopic)); if (pCTopic == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -617,6 +650,7 @@ static FORCE_INLINE SMqConsumerTopic* tNewConsumerTopic(int64_t consumerId, SMqT int32_t unassignedVgSz = taosArrayGetSize(pSub->unassignedVg); if (unassignedVgSz > 0) { SMqConsumerEp* pCEp = taosArrayPop(pSub->unassignedVg); + *oldConsumerId = pCEp->consumerId; pCEp->consumerId = consumerId; taosArrayPush(pCTopic->pVgInfo, &pCEp->vgId); taosArrayPush(pSub->assigned, pCEp); @@ -660,12 +694,17 @@ typedef struct SMqConsumerObj { SRWLatch lock; char cgroup[TSDB_CONSUMER_GROUP_LEN]; SArray* topics; // SArray - // SHashObj *topicHash; //SHashObj + int64_t epoch; + // stat + int64_t pollCnt; } SMqConsumerObj; static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) { int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeFixedI64(buf, pConsumer->connId); + tlen += taosEncodeFixedI64(buf, pConsumer->epoch); + tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt); tlen += taosEncodeString(buf, pConsumer->cgroup); int32_t sz = taosArrayGetSize(pConsumer->topics); tlen += taosEncodeFixedI32(buf, sz); @@ -678,6 +717,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerO static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) { buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeFixedI64(buf, &pConsumer->connId); + buf = taosDecodeFixedI64(buf, &pConsumer->epoch); + buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt); buf = taosDecodeStringTo(buf, pConsumer->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index e642b578fa..fd960eb63e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -54,13 +55,14 @@ void mndCleanupConsumer(SMnode *pMnode) {} SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; @@ -75,6 +77,7 @@ SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_SUCCESS; CM_ENCODE_OVER: + tfree(buf); if (terrno != 0) { mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -87,6 +90,7 @@ CM_ENCODE_OVER: SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; @@ -105,7 +109,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); - void *buf = malloc(len); + buf = malloc(len); if (buf == NULL) goto CM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); @@ -117,6 +121,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; CM_DECODE_OVER: + tfree(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); tfree(pRow); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index fcefa4838a..f855252d1d 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -181,23 +181,26 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) { mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); - atomic_exchange_32(&pOld->updateTime, pNew->updateTime); - atomic_exchange_32(&pOld->version, pNew->version); taosWLockLatch(&pOld->lock); - pOld->numOfColumns = pNew->numOfColumns; - pOld->numOfTags = pNew->numOfTags; int32_t totalCols = pNew->numOfTags + pNew->numOfColumns; int32_t totalSize = totalCols * sizeof(SSchema); - if (pOld->numOfTags + pOld->numOfColumns < totalCols) { void *pSchema = malloc(totalSize); if (pSchema != NULL) { free(pOld->pSchema); pOld->pSchema = pSchema; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr()); + taosWUnLockLatch(&pOld->lock); } } + pOld->updateTime = pNew->updateTime; + pOld->version = pNew->version; + pOld->numOfColumns = pNew->numOfColumns; + pOld->numOfTags = pNew->numOfTags; memcpy(pOld->pSchema, pNew->pSchema, totalSize); taosWUnLockLatch(&pOld->lock); return 0; @@ -228,15 +231,11 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, const char *stbName) { } static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { - SVCreateTbReq req; - void *buf; - int32_t bsize; - SMsgHead *pMsgHead; - - req.ver = 0; SName name = {0}; tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + SVCreateTbReq req = {0}; + req.ver = 0; req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; @@ -247,40 +246,48 @@ static void *mndBuildCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb req.stbCfg.nTagCols = pStb->numOfTags; req.stbCfg.pTagSchema = pStb->pSchema + pStb->numOfColumns; - bsize = tSerializeSVCreateTbReq(NULL, &req); - buf = malloc(sizeof(SMsgHead) + bsize); - if (buf == NULL) { + int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pMsgHead = (SMsgHead *)buf; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - pMsgHead->contLen = htonl(sizeof(SMsgHead) + bsize); - pMsgHead->vgId = htonl(pVgroup->vgId); - - void *pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); tSerializeSVCreateTbReq(&pBuf, &req); - *pContLen = sizeof(SMsgHead) + bsize; - return buf; + *pContLen = contLen; + return pHead; } -static SVDropTbReq *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) { - int32_t contLen = sizeof(SVDropTbReq); +static void *mndBuildDropStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { + SName name = {0}; + tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - SVDropTbReq *pDrop = calloc(1, contLen); - if (pDrop == NULL) { + SVDropTbReq req = {0}; + req.ver = 0; + req.name = (char *)tNameGetTableName(&name); + req.type = TD_SUPER_TABLE; + req.suid = pStb->uid; + + int32_t contLen = tSerializeSVDropTbReq(NULL, &req) + sizeof(SMsgHead); + SMsgHead *pHead = malloc(contLen); + if (pHead == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pDrop->head.contLen = htonl(contLen); - pDrop->head.vgId = htonl(pVgroup->vgId); - memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN); - pDrop->suid = htobe64(pStb->uid); + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); - return pDrop; + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSVDropTbReq(&pBuf, &req); + + *pContLen = contLen; + return pHead; } static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { @@ -358,7 +365,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; void *pIter = NULL; - int32_t contLen; + int32_t contLen; while (1) { pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); @@ -400,7 +407,8 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj if (pIter == NULL) break; if (pVgroup->dbUid != pDb->uid) continue; - SVDropTbReq *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb); + int32_t contLen = 0; + void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb, &contLen); if (pReq == NULL) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); @@ -411,7 +419,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = pReq; - action.contLen = sizeof(SVDropTbReq); + action.contLen = contLen; action.msgType = TDMT_VND_DROP_STB; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); @@ -498,9 +506,9 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { } // topic should have different name with stb - SStbObj *pTopic = mndAcquireStb(pMnode, pCreate->name); - if (pTopic != NULL) { - sdbRelease(pMnode->pSdb, pTopic); + SStbObj *pTopicStb = mndAcquireStb(pMnode, pCreate->name); + if (pTopicStb != NULL) { + mndReleaseStb(pMnode, pTopicStb); terrno = TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC; mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -517,7 +525,6 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { mndReleaseDb(pMnode, pDb); if (code != 0) { - terrno = code; mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -603,15 +610,6 @@ static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pS return 0; } -static int32_t mndSetDropStbUndoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { - SSdbRaw *pUndoRaw = mndStbActionEncode(pStb); - if (pUndoRaw == NULL) return -1; - if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; - - return 0; -} - static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { SSdbRaw *pCommitRaw = mndStbActionEncode(pStb); if (pCommitRaw == NULL) return -1; @@ -621,22 +619,54 @@ static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj * return 0; } -static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } +static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; -static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; } + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pDb->uid) continue; -static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) { + int32_t contLen = 0; + void *pReq = mndBuildDropStbReq(pMnode, pVgroup, pStb, &contLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_DROP_STB; + action.acceptableCode = TSDB_CODE_VND_TB_NOT_EXIST; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + free(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb, SStbObj *pStb) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); - if (pTrans == NULL)goto DROP_STB_OVER; + if (pTrans == NULL) goto DROP_STB_OVER; mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name); if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; - if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) goto DROP_STB_OVER; + if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto DROP_STB_OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_STB_OVER; code = 0; @@ -664,7 +694,16 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { } } - int32_t code = mndDropStb(pMnode, pReq, pStb); + SDbObj *pDb = mndAcquireDbByStb(pMnode, pDrop->name); + if (pDb == NULL) { + mndReleaseStb(pMnode, pStb); + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } + + int32_t code = mndDropStb(pMnode, pReq, pDb, pStb); + mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); if (code != 0) { @@ -860,7 +899,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 if (pStb->dbUid != pDb->uid) { if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) { - mError("Inconsistent table data, name:%s, db:%s, dbUid:%"PRIu64, pStb->name, pDb->name, pDb->uid); + mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); } sdbRelease(pSdb, pStb); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index f93cad0a4d..9352c0cd84 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndSubscribe.h" #include "mndConsumer.h" @@ -30,6 +31,8 @@ #define MND_SUBSCRIBE_VER_NUMBER 1 #define MND_SUBSCRIBE_RESERVE_SIZE 64 +#define MND_SUBSCRIBE_REBALANCE_MS 5000 + static char *mndMakeSubscribeKey(char *cgroup, char *topicName); static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); @@ -46,7 +49,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub); + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub, + int64_t oldConsumerId); int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -67,8 +71,9 @@ int32_t mndInitSubscribe(SMnode *pMnode) { static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; - SMqCMGetSubEpRsp rsp; + SMqCMGetSubEpRsp rsp = {0}; int64_t consumerId = be64toh(pReq->consumerId); + int64_t currentTs = taosGetTimestampMs(); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); if (pConsumer == NULL) { @@ -79,6 +84,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(rsp.cgroup, pReq->cgroup); rsp.consumerId = consumerId; + rsp.epoch = pConsumer->epoch; SArray *pTopics = pConsumer->topics; int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); @@ -88,21 +94,39 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { strcpy(topicEp.topic, pConsumerTopic->name); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, pConsumerTopic->name); - int32_t assignedSz = taosArrayGetSize(pSub->assigned); + ASSERT(pSub); + bool found = 0; + bool changed = 0; + for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { + if (*(int64_t *)taosArrayGet(pSub->availConsumer, j) == consumerId) { + found = 1; + break; + } + } + if (found == 0) { + taosArrayPush(pSub->availConsumer, &consumerId); + } + + int32_t assignedSz = taosArrayGetSize(pSub->assigned); topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp)); for (int32_t j = 0; j < assignedSz; j++) { SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, j); if (pCEp->consumerId == consumerId) { - SMqSubVgEp vgEp = { - .epSet = pCEp->epSet, - .vgId = pCEp->vgId - }; + pCEp->lastConsumerHbTs = currentTs; + SMqSubVgEp vgEp = {.epSet = pCEp->epSet, .vgId = pCEp->vgId}; taosArrayPush(topicEp.vgs, &vgEp); + changed = 1; } } if (taosArrayGetSize(topicEp.vgs) != 0) { taosArrayPush(rsp.topics, &topicEp); } + if (changed || found) { + SSdbRaw *pRaw = mndSubActionEncode(pSub); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + sdbWrite(pMnode->pSdb, pRaw); + } + mndReleaseSubscribe(pMnode, pSub); } int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp); void *buf = rpcMallocCont(tlen); @@ -112,7 +136,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } void *abuf = buf; tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); - //TODO: free rsp + tDeleteSMqCMGetSubEpRsp(&rsp); pMsg->pCont = buf; pMsg->contLen = tlen; return 0; @@ -124,9 +148,9 @@ static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) { i++; } key[i] = 0; - *topic = strdup(key); + *cgroup = strdup(key); key[i] = ':'; - *cgroup = strdup(&key[i + 1]); + *topic = strdup(&key[i + 1]); return 0; } @@ -135,9 +159,40 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SSdb *pSdb = pMnode->pSdb; SMqSubscribeObj *pSub = NULL; void *pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); - int sz; + int64_t currentTs = taosGetTimestampMs(); + int32_t sz; while (pIter != NULL) { - if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0) { + for (int i = 0; i < taosArrayGetSize(pSub->assigned); i++) { + SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i); + int64_t consumerId = pCEp->consumerId; + if (pCEp->lastConsumerHbTs != -1 && currentTs - pCEp->lastConsumerHbTs > MND_SUBSCRIBE_REBALANCE_MS) { + // put consumer into lostConsumer + taosArrayPush(pSub->lostConsumer, pCEp); + // put vg into unassigned + taosArrayPush(pSub->unassignedVg, pCEp); + // remove from assigned + // TODO: swap with last one, reduce size and reset i + taosArrayRemove(pSub->assigned, i); + // remove from available consumer + for (int j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { + if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) { + taosArrayRemove(pSub->availConsumer, j); + break; + } + // TODO: acquire consumer, set status to unavail + } +#if 0 + SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, consumerId); + pConsumer->epoch++; + printf("current epoch %ld size %ld", pConsumer->epoch, pConsumer->topics->size); + SSdbRaw* pRaw = mndConsumerActionEncode(pConsumer); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + sdbWriteNotFree(pMnode->pSdb, pRaw); + mndReleaseConsumer(pMnode, pConsumer); +#endif + } + } + if ((sz = taosArrayGetSize(pSub->unassignedVg)) > 0 && taosArrayGetSize(pSub->availConsumer) > 0) { char *topic = NULL; char *cgroup = NULL; mndSplitSubscribeKey(pSub->key, &topic, &cgroup); @@ -146,58 +201,70 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { // create trans STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pSub->availConsumer, pSub->nextConsumerIdx); SMqConsumerEp *pCEp = taosArrayPop(pSub->unassignedVg); + int64_t oldConsumerId = pCEp->consumerId; pCEp->consumerId = consumerId; taosArrayPush(pSub->assigned, pCEp); pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); + pConsumer->epoch++; + /*SSdbRaw* pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ + /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ + /*sdbWriteNotFree(pMnode->pSdb, pConsumerRaw);*/ + mndReleaseConsumer(pMnode, pConsumer); + // build msg - SMqSetCVgReq *pReq = malloc(sizeof(SMqSetCVgReq)); - if (pReq == NULL) { + SMqSetCVgReq req = {0}; + strcpy(req.cgroup, cgroup); + strcpy(req.topicName, topic); + req.sql = pTopic->sql; + req.logicalPlan = pTopic->logicalPlan; + req.physicalPlan = pTopic->physicalPlan; + req.qmsg = pCEp->qmsg; + req.oldConsumerId = oldConsumerId; + req.newConsumerId = consumerId; + req.vgId = pCEp->vgId; + int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + void *buf = malloc(sizeof(SMsgHead) + tlen); + if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - strcpy(pReq->cgroup, cgroup); - strcpy(pReq->topicName, topic); - pReq->sql = strdup(pTopic->sql); - pReq->logicalPlan = strdup(pTopic->logicalPlan); - pReq->physicalPlan = strdup(pTopic->physicalPlan); - pReq->qmsg = strdup(pCEp->qmsg); - int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq); - void *reqStr = malloc(tlen); - if (reqStr == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - void *abuf = reqStr; - tEncodeSMqSetCVgReq(&abuf, pReq); + SMsgHead *pMsgHead = (SMsgHead *)buf; + + pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); + pMsgHead->vgId = htonl(pCEp->vgId); + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + tEncodeSMqSetCVgReq(&abuf, &req); // persist msg + // TODO: no need for txn STransAction action = {0}; action.epSet = pCEp->epSet; - action.pCont = reqStr; - action.contLen = tlen; + action.pCont = buf; + action.contLen = sizeof(SMsgHead) + tlen; action.msgType = TDMT_VND_MQ_SET_CONN; mndTransAppendRedoAction(pTrans, &action); // persist raw SSdbRaw *pRaw = mndSubActionEncode(pSub); + sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); - free(pReq); tfree(topic); tfree(cgroup); } if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); } - /*mndReleaseTopic(pMnode, pTopic);*/ + mndReleaseTopic(pMnode, pTopic); mndTransDrop(pTrans); } - pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, NULL, (void **)&pSub); + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); } return 0; } @@ -205,33 +272,49 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { // convert phyplan to dag SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); - SArray *pArray; + SArray *pArray = NULL; SArray *inner = taosArrayGet(pDag->pSubplans, 0); SSubplan *plan = taosArrayGetP(inner, 0); + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; - plan->execNode.nodeId = 2; - SEpSet* pEpSet = &plan->execNode.epset; + void *pIter = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (pVgroup->dbUid != pTopic->dbUid) continue; - pEpSet->inUse = 0; - addEpIntoEpSet(pEpSet, "localhost", 6030); - if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { - return -1; - } - int32_t sz = taosArrayGetSize(pArray); - // convert dag to msg - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp CEp; + plan->execNode.nodeId = pVgroup->vgId; + plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup); + + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + mError("unsupport topic: %s, sql: %s", pTopic->name, pTopic->sql); + return -1; + } + if (pArray && taosArrayGetSize(pArray) != 1) { + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + mError("unsupport topic: %s, sql: %s, plan level: %ld", pTopic->name, pTopic->sql, taosArrayGetSize(pArray)); + return -1; + } + + SMqConsumerEp CEp = {0}; CEp.status = 0; CEp.consumerId = -1; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; - STaskInfo *pTaskInfo = taosArrayGet(pArray, i); + STaskInfo *pTaskInfo = taosArrayGet(pArray, 0); CEp.epSet = pTaskInfo->addr.epset; - - /*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], - * CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/ CEp.vgId = pTaskInfo->addr.nodeId; + + ASSERT(CEp.vgId == pVgroup->vgId); CEp.qmsg = strdup(pTaskInfo->msg->msg); taosArrayPush(unassignedVg, &CEp); + // TODO: free taskInfo + taosArrayDestroy(pArray); + + /*SEpSet *pEpSet = &plan->execNode.epset;*/ + /*pEpSet->inUse = 0;*/ + /*addEpIntoEpSet(pEpSet, "localhost", 6030);*/ } /*qDestroyQueryDag(pDag);*/ @@ -239,14 +322,15 @@ static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unas } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp) { + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp, + int64_t oldConsumerId) { int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); for (int32_t i = 0; i < sz; i++) { int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i); SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); SMqSetCVgReq req = { .vgId = vgId, - .oldConsumerId = -1, + .oldConsumerId = oldConsumerId, .newConsumerId = pConsumer->consumerId, }; strcpy(req.cgroup, pConsumer->cgroup); @@ -289,13 +373,14 @@ void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; @@ -310,6 +395,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_SUCCESS; SUB_ENCODE_OVER: + tfree(buf); if (terrno != 0) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -322,6 +408,7 @@ SUB_ENCODE_OVER: static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; @@ -341,7 +428,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - void *buf = malloc(tlen + 1); + buf = malloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -353,6 +440,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_SUCCESS; SUB_DECODE_OVER: + tfree(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); // TODO free subscribeobj @@ -370,6 +458,7 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *pSub) { static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *pSub) { mTrace("subscribe:%s, perform delete action", pSub->key); + tDeleteSMqSubscribeObj(pSub); return 0; } @@ -378,10 +467,6 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc return 0; } -static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) { - return 0; -} - static char *mndMakeSubscribeKey(char *cgroup, char *topicName) { char *key = malloc(TSDB_SHOW_SUBQUERY_LEN); if (key == NULL) { @@ -435,10 +520,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pConsumer->epoch = 1; pConsumer->consumerId = consumerId; strcpy(pConsumer->cgroup, consumerGroup); taosInitRWLatch(&pConsumer->lock); } else { + pConsumer->epoch++; oldSub = pConsumer->topics; } pConsumer->topics = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic)); @@ -542,6 +629,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, consumerGroup, newTopicName); + bool create = false; if (pSub == NULL) { mDebug("create new subscription, group: %s, topic %s", consumerGroup, newTopicName); pSub = tNewSubscribeObj(); @@ -550,14 +638,24 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return -1; } char *key = mndMakeSubscribeKey(consumerGroup, newTopicName); + if (key == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } strcpy(pSub->key, key); + free(key); // set unassigned vg - mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); + if (mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg) < 0) { + // TODO: free memory + return -1; + } // TODO: disable alter + create = true; } taosArrayPush(pSub->availConsumer, &consumerId); - SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub); + int64_t oldConsumerId; + SMqConsumerTopic *pConsumerTopic = tNewConsumerTopic(consumerId, pTopic, pSub, &oldConsumerId); taosArrayPush(pConsumer->topics, pConsumerTopic); if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { @@ -565,7 +663,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); SMqConsumerEp *pCEp = taosArrayGetLast(pSub->assigned); if (pCEp->vgId == vgId) { - if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp) < 0) { + if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp, oldConsumerId) < 0) { // TODO return -1; } @@ -576,6 +674,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { SSdbRaw *pRaw = mndSubActionEncode(pSub); sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); + if (!create) mndReleaseSubscribe(pMnode, pSub); #if 0 SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); if (pGroup == NULL) { diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index aacaa0d728..777492697a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -103,6 +103,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { if (walEndSnapshot(pWal) < 0) { goto WAL_RESTORE_OVER; } + } code = 0; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 9b821fe615..ea3cc7c2c6 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -240,15 +240,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq topicObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); topicObj.dbUid = pDb->uid; topicObj.version = 1; - topicObj.sql = strdup(pCreate->sql); - topicObj.physicalPlan = strdup(pCreate->physicalPlan); - topicObj.logicalPlan = strdup(pCreate->logicalPlan); + topicObj.sql = pCreate->sql; + topicObj.physicalPlan = pCreate->physicalPlan; + topicObj.logicalPlan = pCreate->logicalPlan; topicObj.sqlLen = strlen(pCreate->sql); SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); if (pTopicRaw == NULL) return -1; if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; - // TODO: replace with trans to support recovery return sdbWrite(pMnode->pSdb, pTopicRaw); } diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index faaf769e1a..2aceaeb016 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -35,71 +35,8 @@ extern "C" { #endif -typedef struct STqMsgHead { - int32_t protoVer; - int32_t msgType; - int64_t cgId; - int64_t clientId; -} STqMsgHead; - -typedef struct STqOneAck { - int64_t topicId; - int64_t consumeOffset; -} STqOneAck; - -typedef struct STqAcks { - int32_t ackNum; - // should be sorted - STqOneAck acks[]; -} STqAcks; - -typedef struct STqSetCurReq { - STqMsgHead head; - int64_t topicId; - int64_t offset; -} STqSetCurReq; - -typedef struct STqConsumeReq { - STqMsgHead head; - int64_t blockingTime; // milisec - STqAcks acks; -} STqConsumeReq; - -typedef struct STqMsgContent { - int64_t topicId; - int64_t msgLen; - char msg[]; -} STqMsgContent; - -typedef struct STqConsumeRsp { - STqMsgHead head; - int64_t bodySize; - STqMsgContent msgs[]; -} STqConsumeRsp; - -typedef struct STqSubscribeReq { - STqMsgHead head; - int32_t topicNum; - int64_t topic[]; -} STqSubscribeReq; - -typedef struct STqHeartbeatReq { -} STqHeartbeatReq; - -typedef struct STqHeartbeatRsp { -} STqHeartbeatRsp; - #define TQ_BUFFER_SIZE 8 -typedef struct STqExec { - void* runtimeEnv; - SSDataBlock* (*exec)(void* runtimeEnv); - void* (*assign)(void* runtimeEnv, void* inputData); - void (*clear)(void* runtimeEnv); - char* (*serialize)(struct STqExec*); - struct STqExec* (*deserialize)(char*); -} STqExec; - typedef struct STqRspHandle { void* handle; void* ahandle; @@ -107,47 +44,6 @@ typedef struct STqRspHandle { typedef enum { TQ_ITEM_READY, TQ_ITEM_PROCESS, TQ_ITEM_EMPTY } STqItemStatus; -typedef struct STqTopic STqTopic; - -typedef struct STqBufferItem { - int64_t offset; - // executors are identical but not concurrent - // so there must be a copy in each item - STqExec* executor; - int32_t status; - int64_t size; - void* content; - STqTopic* pTopic; -} STqMsgItem; - -struct STqTopic { - // char* topic; //c style, end with '\0' - // int64_t cgId; - // void* ahandle; - // int32_t head; - // int32_t tail; - int64_t nextConsumeOffset; - int64_t floatingCursor; - int64_t topicId; - void* logReader; - STqMsgItem buffer[TQ_BUFFER_SIZE]; -}; - -typedef struct STqListHandle { - STqTopic topic; - struct STqListHandle* next; -} STqList; - -typedef struct STqGroup { - int64_t clientId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - STqList* head; - SList* topicList; // SList - STqRspHandle rspHandle; -} STqGroup; - typedef struct STqTaskItem { int8_t status; int64_t offset; @@ -182,11 +78,6 @@ typedef struct STqConsumerHandle { SArray* topics; // SArray } STqConsumerHandle; -typedef struct STqQueryMsg { - STqMsgItem* item; - struct STqQueryMsg* next; -} STqQueryMsg; - typedef struct STqMemRef { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; @@ -305,20 +196,6 @@ void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqSetCursor(STQ*, STqSetCurReq* pMsg); - -#if 0 -int tqConsume(STQ*, SRpcMsg* pReq, SRpcMsg** pRsp); -int tqSetCursor(STQ*, STqSetCurReq* pMsg); -int tqBufferSetOffset(STqTopic*, int64_t offset); -STqTopic* tqFindTopic(STqGroup*, int64_t topicId); -STqGroup* tqGetGroup(STQ*, int64_t clientId); -STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqRegisterContext(STqGroup*, void* ahandle); -int tqSendLaunchQuery(STqMsgItem*, int64_t offset); -#endif - int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 5f5acc1b05..a2f935683e 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -91,7 +91,7 @@ int tsdbCommit(STsdb *pTsdb); int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); -typedef void* tsdbReadHandleT; +typedef void* tsdbReaderT; /** * Get the data block iterator, starting from position according to the query condition @@ -103,7 +103,7 @@ typedef void* tsdbReadHandleT; * @param qinfo query info handle from query processor * @return */ -tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); +tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -115,13 +115,13 @@ tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup * @param tableInfo table list. * @return */ -//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +//tsdbReaderT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, // SMemRef *pRef); -tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); -bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); +bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle); /** * @@ -138,7 +138,7 @@ bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); * @param reqId * @return */ -int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId); /** @@ -148,7 +148,7 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch * @return row size */ -int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); +int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle); /** * move to next block if exists @@ -156,7 +156,7 @@ int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); * @param pTsdbReadHandle * @return */ -bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); +bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); /** * Get current data block information @@ -165,7 +165,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); +void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); /** * @@ -177,7 +177,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SDataStatis **pBlockStatis); /** * @@ -189,7 +189,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataS * @param pColumnIdList required data columns id list * @return */ -SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList); +SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); /** * destroy the created table group list, which is generated by tag query @@ -205,7 +205,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); /** * @@ -220,7 +220,7 @@ int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGro * clean up the query handle * @param queryHandle */ -void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle); +void tsdbCleanupReadHandle(tsdbReaderT queryHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index c8b47bf4a6..7549772613 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -40,7 +40,6 @@ typedef struct STqCfg { int32_t reserved; } STqCfg; - typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; @@ -69,17 +68,18 @@ typedef struct { } SVnodeOpt; typedef struct STqReadHandle { - int64_t ver; - uint64_t tbUid; - SSubmitMsg* pMsg; - SSubmitBlk* pBlock; - SSubmitMsgIter msgIter; - SSubmitBlkIter blkIter; - SMeta* pVnodeMeta; - SArray* pColIdList; //SArray - int32_t sver; - SSchemaWrapper* pSchemaWrapper; - STSchema* pSchema; + int64_t ver; + uint64_t tbUid; + SHashObj *tbIdHash; + const SSubmitMsg *pMsg; + SSubmitBlk *pBlock; + SSubmitMsgIter msgIter; + SSubmitBlkIter blkIter; + SMeta *pVnodeMeta; + SArray *pColIdList; // SArray + int32_t sver; + SSchemaWrapper *pSchemaWrapper; + STSchema *pSchema; } STqReadHandle; /* ------------------------ SVnode ------------------------ */ @@ -201,22 +201,34 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); /* ------------------------- TQ QUERY -------------------------- */ -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta); +STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); -static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { +static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList) { pReadHandle->pColIdList = pColIdList; } -static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, uint64_t tbUid) { - pHandle->tbUid = tbUid; +// static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, const SArray* pTableIdList) { +// pHandle->tbUid = pTableIdList; +//} + +static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + if (pHandle->tbIdHash == NULL) { + return -1; + } + for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t *pKey = (int64_t *)taosArrayGet(tbUidList, i); + taosHashPut(pHandle->tbIdHash, pKey, sizeof(int64_t), NULL, 0); + // pHandle->tbUid = tbUid; + } + return 0; } -void tqReadHandleSetMsg(STqReadHandle* pHandle, SSubmitMsg* pMsg, int64_t ver); -bool tqNextDataBlock(STqReadHandle* pHandle); -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo); +void tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitMsg *pMsg, int64_t ver); +bool tqNextDataBlock(STqReadHandle *pHandle); +int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); // return SArray -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle); - +SArray *tqRetrieveDataBlock(STqReadHandle *pHandle); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index a9ba825a29..cefe9eff72 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -16,9 +16,9 @@ #ifndef _TD_TQ_INT_H_ #define _TD_TQ_INT_H_ -#include "tq.h" #include "meta.h" #include "tlog.h" +#include "tq.h" #include "trpc.h" #ifdef __cplusplus extern "C" { @@ -26,29 +26,48 @@ extern "C" { extern int32_t tqDebugFlag; -#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }} -#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }} -#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }} -#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }} -#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} -#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} +#define tqFatal(...) \ + { \ + if (tqDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \ + } \ + } +#define tqError(...) \ + { \ + if (tqDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ + } \ + } +#define tqWarn(...) \ + { \ + if (tqDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ + } \ + } +#define tqInfo(...) \ + { \ + if (tqDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TQ ", 255, __VA_ARGS__); \ + } \ + } +#define tqDebug(...) \ + { \ + if (tqDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } +#define tqTrace(...) \ + { \ + if (tqDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } -// create persistent storage for meta info such as consuming offset -// return value > 0: cgId -// return value <= 0: error code -// int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); -// create ring buffer in memory and load consuming offset -// int tqOpenTCGroup(STQ*, const char* topic, int cgId); -// destroy ring buffer and persist consuming offset -// int tqCloseTCGroup(STQ*, const char* topic, int cgId); -// delete persistent storage for meta info -// int tqDropTCGroup(STQ*, const char* topic, int cgId); +int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); +const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); -//int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -//const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); -int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h index ef71d8bf14..3bf9bb7138 100644 --- a/source/dnode/vnode/src/inc/tqMetaStore.h +++ b/source/dnode/vnode/src/inc/tqMetaStore.h @@ -40,6 +40,7 @@ int32_t tqHandleCopyPut(STqMetaStore*, int64_t key, void* value, size_t vsize); // delete committed kv pair // notice that a delete action still needs to be committed int32_t tqHandleDel(STqMetaStore*, int64_t key); +int32_t tqHandlePurge(STqMetaStore*, int64_t key); int32_t tqHandleCommit(STqMetaStore*, int64_t key); int32_t tqHandleAbort(STqMetaStore*, int64_t key); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3195691a13..a625980505 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -12,30 +12,12 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" -// static -// read next version data -// -// send to fetch queue -// -// handle management message -// - -int tqGroupSSize(const STqGroup* pGroup); -int tqTopicSSize(); -int tqItemSSize(); - -void* tqSerializeListHandle(STqList* listHandle, void* ptr); -void* tqSerializeTopic(STqTopic* pTopic, void* ptr); -void* tqSerializeItem(STqMsgItem* pItem, void* ptr); - -const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic); -const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); - int tqInit() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); if (old == 1) return 0; @@ -88,177 +70,6 @@ void tqClose(STQ* pTq) { // TODO } -static int tqProtoCheck(STqMsgHead* pMsg) { - // TODO - return pMsg->protoVer == 0; -} - -static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) { - // clean old item and move forward - int32_t consumeOffset = pAck->consumeOffset; - int idx = consumeOffset % TQ_BUFFER_SIZE; - ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor); - tfree(pTopic->buffer[idx].content); - if (1 /* TODO: need to launch new query */) { - STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); - if (pNewQuery == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - // TODO: lock executor - // TODO: read from wal and assign to src - /*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/ - /*pNewQuery->exec->src = 0;*/ - /*pNewQuery->exec->dest = &pTopic->buffer[idx];*/ - /*pNewQuery->next = *ppQuery;*/ - /**ppQuery = pNewQuery;*/ - } - return 0; -} - -static int tqAck(STqGroup* pGroup, STqAcks* pAcks) { - int32_t ackNum = pAcks->ackNum; - STqOneAck* acks = pAcks->acks; - // double ptr for acks and list - int i = 0; - STqList* node = pGroup->head; - int ackCnt = 0; - STqQueryMsg* pQuery = NULL; - while (i < ackNum && node->next) { - if (acks[i].topicId == node->next->topic.topicId) { - ackCnt++; - tqAckOneTopic(&node->next->topic, &acks[i], &pQuery); - } else if (acks[i].topicId < node->next->topic.topicId) { - i++; - } else { - node = node->next; - } - } - if (pQuery) { - // post message - } - return ackCnt; -} - -static int tqCommitGroup(STqGroup* pGroup) { - // persist modification into disk - return 0; -} - -int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) { - // create in disk - STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup)); - if (pGroup == NULL) { - // TODO - return -1; - } - *ppGroup = pGroup; - memset(pGroup, 0, sizeof(STqGroup)); - - pGroup->topicList = tdListNew(sizeof(STqTopic)); - if (pGroup->topicList == NULL) { - free(pGroup); - return -1; - } - *ppGroup = pGroup; - - return 0; -} - -STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId); - if (pGroup == NULL) { - int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup); - if (code < 0) { - // TODO - return NULL; - } - tqHandleMovePut(pTq->tqMeta, cId, pGroup); - } - ASSERT(pGroup); - - return pGroup; -} - -int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - // TODO - return 0; -} - -int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - // delete from disk - return 0; -} - -static int tqFetch(STqGroup* pGroup, STqConsumeRsp** pRsp) { - STqList* pHead = pGroup->head; - STqList* pNode = pHead; - int totSize = 0; - int numOfMsgs = 0; - // TODO: make it a macro - int sizeLimit = 4 * 1024; - - void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + sizeLimit); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - *pRsp = ptr; - STqMsgContent* buffer = (*pRsp)->msgs; - - // iterate the list to get msgs of all topics - // until all topic iterated or msgs over sizeLimit - while (pNode->next) { - pNode = pNode->next; - STqTopic* pTopic = &pNode->topic; - int idx = pTopic->nextConsumeOffset % TQ_BUFFER_SIZE; - if (pTopic->buffer[idx].content != NULL && pTopic->buffer[idx].offset == pTopic->nextConsumeOffset) { - totSize += pTopic->buffer[idx].size; - if (totSize > sizeLimit) { - void* ptr = realloc(*pRsp, sizeof(STqConsumeRsp) + totSize); - if (ptr == NULL) { - totSize -= pTopic->buffer[idx].size; - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // return msgs already copied - break; - } - *pRsp = ptr; - break; - } - *((int64_t*)buffer) = pTopic->topicId; - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = pTopic->buffer[idx].size; - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); - numOfMsgs++; - if (totSize > sizeLimit) { - break; - } - } - } - (*pRsp)->bodySize = totSize; - return numOfMsgs; -} - -STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); } - -int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) { - if (tqQueryExecuting(bufItem->status)) { - return 0; - } - bufItem->status = 1; - // load data from wal or buffer pool - // put into exec - // send exec into non blocking queue - // when query finished, put into buffer pool - return 0; -} - -/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ -/*return 0;*/ -/*}*/ - int tqPushMsg(STQ* pTq, void* p, int64_t version) { // add reference // judge and launch new query @@ -270,218 +81,6 @@ int tqCommit(STQ* pTq) { return 0; } -int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) { - int code; - memset(pTopic->buffer, 0, sizeof(pTopic->buffer)); - // launch query - for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) { - int pos = i % TQ_BUFFER_SIZE; - code = tqSendLaunchQuery(&pTopic->buffer[pos], offset); - if (code < 0) { - // TODO: error handling - } - } - // set offset - pTopic->nextConsumeOffset = offset; - pTopic->floatingCursor = offset; - return 0; -} - -STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) { - // TODO - return NULL; -} - -int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) { - int code; - int64_t clientId = pMsg->head.clientId; - int64_t topicId = pMsg->topicId; - int64_t offset = pMsg->offset; - STqGroup* gHandle = tqGetGroup(pTq, clientId); - if (gHandle == NULL) { - // client not connect - return -1; - } - STqTopic* topicHandle = tqFindTopic(gHandle, topicId); - if (topicHandle == NULL) { - return -1; - } - if (pMsg->offset == topicHandle->nextConsumeOffset) { - return 0; - } - // TODO: check log last version - - code = tqBufferSetOffset(topicHandle, offset); - if (code < 0) { - // set error code - return -1; - } - - return 0; -} - -// temporary -int tqProcessCMsg(STQ* pTq, STqConsumeReq* pMsg, STqRspHandle* pRsp) { - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - terrno = TSDB_CODE_TQ_GROUP_NOT_SET; - return -1; - } - pGroup->rspHandle.handle = pRsp->handle; - pGroup->rspHandle.ahandle = pRsp->ahandle; - - return 0; -} - -int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) { - STqConsumeReq* pMsg = pReq->pCont; - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - terrno = TSDB_CODE_TQ_GROUP_NOT_SET; - return -1; - } - - SList* topicList = pGroup->topicList; - - int totSize = 0; - int numOfMsgs = 0; - int sizeLimit = 4096; - - STqConsumeRsp* pCsmRsp = (*pRsp)->pCont; - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); - if (ptr == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - (*pRsp)->pCont = ptr; - - SListIter iter; - tdListInitIter(topicList, &iter, TD_LIST_FORWARD); - - STqMsgContent* buffer = NULL; - SArray* pArray = taosArrayInit(0, sizeof(void*)); - - SListNode* pn; - while ((pn = tdListNext(&iter)) != NULL) { - STqTopic* pTopic = *(STqTopic**)pn->data; - int idx = pTopic->floatingCursor % TQ_BUFFER_SIZE; - STqMsgItem* pItem = &pTopic->buffer[idx]; - if (pItem->content != NULL && pItem->offset == pTopic->floatingCursor) { - if (pItem->status == TQ_ITEM_READY) { - // if has data - totSize += pTopic->buffer[idx].size; - if (totSize > sizeLimit) { - void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + totSize); - if (ptr == NULL) { - totSize -= pTopic->buffer[idx].size; - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - // return msgs already copied - break; - } - (*pRsp)->pCont = ptr; - break; - } - *((int64_t*)buffer) = htobe64(pTopic->topicId); - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = htobe64(pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, pTopic->buffer[idx].content, pTopic->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, pTopic->buffer[idx].size); - numOfMsgs++; - if (totSize > sizeLimit) { - break; - } - } else if (pItem->status == TQ_ITEM_PROCESS) { - // if not have data but in process - - } else if (pItem->status == TQ_ITEM_EMPTY) { - // if not have data and not in process - int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_EMPTY, TQ_ITEM_PROCESS); - if (old != TQ_ITEM_EMPTY) { - continue; - } - pItem->offset = pTopic->floatingCursor; - taosArrayPush(pArray, &pItem); - } else { - ASSERT(0); - } - } - } - - if (numOfMsgs > 0) { - // set code and other msg - rpcSendResponse(*pRsp); - } else { - // most recent data has been fetched - - // enable timer for blocking wait - // once new data written when waiting, launch query and rsp - } - - // fetched a num of msgs, rpc response - for (int i = 0; i < pArray->size; i++) { - STqMsgItem* pItem = taosArrayGet(pArray, i); - - // read from wal - void* raw = NULL; - /*int code = pTq->tqLogReader->logRead(, &raw, pItem->offset);*/ - /*int code = pTq->tqLogHandle->logRead(pItem->pTopic->logReader, &raw, pItem->offset);*/ - /*if (code < 0) {*/ - // TODO: error - /*}*/ - // get msgType - // if submitblk - pItem->executor->assign(pItem->executor->runtimeEnv, raw); - SSDataBlock* content = pItem->executor->exec(pItem->executor->runtimeEnv); - pItem->content = content; - // if other type, send just put into buffer - /*pItem->content = raw;*/ - - int32_t old = atomic_val_compare_exchange_32(&pItem->status, TQ_ITEM_PROCESS, TQ_ITEM_READY); - ASSERT(old == TQ_ITEM_PROCESS); - } - taosArrayDestroy(pArray); - - return 0; -} - -#if 0 -int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { - if (!tqProtoCheck((STqMsgHead*)pMsg)) { - // proto version invalid - return -1; - } - int64_t clientId = pMsg->head.clientId; - STqGroup* pGroup = tqGetGroup(pTq, clientId); - if (pGroup == NULL) { - // client not connect - return -1; - } - if (pMsg->acks.ackNum != 0) { - if (tqAck(pGroup, &pMsg->acks) != 0) { - // ack not success - return -1; - } - } - - STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - - if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) { - // fetch error - return -1; - } - - // judge and launch new query - /*if (tqSendLaunchQuery(gHandle)) {*/ - // launch query error - /*return -1;*/ - /*}*/ - return 0; -} -#endif - int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) { int32_t num = taosArrayGetSize(pConsumer->topics); int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + @@ -535,153 +134,24 @@ const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHan return NULL; } -#if 0 -int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { - // calculate size - int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); - if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sz); - if (tmpPtr == NULL) { - free(*ppHead); - // TODO: memory err - return -1; - } - *ppHead = tmpPtr; - (*ppHead)->ssize = sz; - } - void* ptr = (*ppHead)->content; - // do serialization - *(int64_t*)ptr = pGroup->clientId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pGroup->cgId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = pGroup->topicNum; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if (pGroup->topicNum > 0) { - tqSerializeListHandle(pGroup->head, ptr); - } - return 0; -} - -void* tqSerializeListHandle(STqList* listHandle, void* ptr) { - STqList* node = listHandle; - ASSERT(node != NULL); - while (node) { - ptr = tqSerializeTopic(&node->topic, ptr); - node = node->next; - } - return ptr; -} - -void* tqSerializeTopic(STqTopic* pTopic, void* ptr) { - *(int64_t*)ptr = pTopic->nextConsumeOffset; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pTopic->topicId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - /**(int32_t*)ptr = pTopic->head;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - /**(int32_t*)ptr = pTopic->tail;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqSerializeItem(&pTopic->buffer[i], ptr); - } - return ptr; -} - -void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) { - // TODO: do we need serialize this? - // mainly for executor - return ptr; -} - -const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) { - STqGroup* gHandle = *ppGroup; - const void* ptr = pHead->content; - gHandle->clientId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - gHandle->cgId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - gHandle->ahandle = NULL; - gHandle->topicNum = *(int32_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - gHandle->head = NULL; - STqList* node = gHandle->head; - for (int i = 0; i < gHandle->topicNum; i++) { - if (gHandle->head == NULL) { - if ((node = malloc(sizeof(STqList))) == NULL) { - // TODO: error - return NULL; - } - node->next = NULL; - ptr = tqDeserializeTopic(ptr, &node->topic); - gHandle->head = node; - } else { - node->next = malloc(sizeof(STqList)); - if (node->next == NULL) { - // TODO: error - return NULL; - } - node->next->next = NULL; - ptr = tqDeserializeTopic(ptr, &node->next->topic); - node = node->next; - } - } - return ptr; -} - -const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) { - const void* ptr = pBytes; - topic->nextConsumeOffset = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - topic->topicId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - /*topic->head = *(int32_t*)ptr;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - /*topic->tail = *(int32_t*)ptr;*/ - /*ptr = POINTER_SHIFT(ptr, sizeof(int32_t));*/ - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqDeserializeItem(ptr, &topic->buffer[i]); - } - return ptr; -} - -const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; } - -// TODO: make this a macro -int tqGroupSSize(const STqGroup* gHandle) { - return sizeof(int64_t) * 2 // cId + cgId - + sizeof(int32_t) // topicNum - + gHandle->topicNum * tqTopicSSize(); -} - -// TODO: make this a macro -int tqTopicSSize() { - return sizeof(int64_t) * 2 // nextConsumeOffset + topicId - + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqItemSSize(); -} - -int tqItemSSize() { - // TODO: do this need serialization? - // mainly for executor - return 0; -} -#endif int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; - SRpcMsg rpcMsg; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; - int64_t reqOffset = pReq->offset; - int64_t fetchOffset = reqOffset; + int64_t fetchOffset = pReq->offset; int64_t blockingTime = pReq->blockingTime; - int rspLen = 0; - SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 1, .pBlockData = NULL}; + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); - ASSERT(pConsumer); + if (pConsumer == NULL) { + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = -1; + rpcSendResponse(pMsg); + return 0; + } int sz = taosArrayGetSize(pConsumer->topics); for (int i = 0; i < sz; i++) { @@ -691,7 +161,24 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { continue; } - if (fetchOffset == -1) { + if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { + pTopic->committedOffset = pReq->offset; + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = 0; + rpcSendResponse(pMsg); + return 0; + } + + if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { + pTopic->committedOffset = pReq->offset-1; + } + + rsp.committedOffset = pTopic->committedOffset; + rsp.reqOffset = pReq->offset; + rsp.skipLogNum = 0; + + if (fetchOffset <= pTopic->committedOffset) { fetchOffset = pTopic->committedOffset + 1; } int8_t pos; @@ -715,6 +202,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { if (pHead->head.msgType == TDMT_VND_SUBMIT) { break; } + rsp.skipLogNum++; if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { atomic_store_8(&pTopic->buffer.output[pos].status, 0); skip = 1; @@ -745,6 +233,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } //TODO copy rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; + rsp.rspOffset = fetchOffset; atomic_store_8(&pTopic->buffer.output[pos].status, 0); @@ -752,6 +241,8 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { taosArrayDestroy(pRes); fetchOffset++; continue; + } else { + rsp.numOfTopics++; } rsp.pBlockData = pRes; @@ -774,6 +265,15 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } void* abuf = buf; tEncodeSMqConsumeRsp(&abuf, &rsp); + if (rsp.pBlockData) { + taosArrayDestroyEx(rsp.pBlockData, (void(*)(void*))tDeleteSSDataBlock); + rsp.pBlockData = NULL; + /*for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) {*/ + /*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/ + /*tDeleteSSDataBlock(pBlock);*/ + /*}*/ + /*taosArrayDestroy(rsp.pBlockData);*/ + } pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; @@ -782,11 +282,22 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { - SMqSetCVgReq req; + SMqSetCVgReq req = {0}; tDecodeSMqSetCVgReq(msg, &req); - STqConsumerHandle* pConsumer = calloc(sizeof(STqConsumerHandle), 1); + + STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); if (pConsumer == NULL) { - return -1; + pConsumer = calloc(sizeof(STqConsumerHandle), 1); + if (pConsumer == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + } else { + tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); + tqHandleCommit(pTq->tqMeta, req.newConsumerId); + tqHandlePurge(pTq->tqMeta, req.oldConsumerId); + terrno = TSDB_CODE_SUCCESS; + return 0; } strcpy(pConsumer->cgroup, req.cgroup); pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); @@ -799,9 +310,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { return -1; } strcpy(pTopic->topicName, req.topicName); - pTopic->sql = strdup(req.sql); - pTopic->logicalPlan = strdup(req.logicalPlan); - pTopic->physicalPlan = strdup(req.physicalPlan); + pTopic->sql = req.sql; + pTopic->logicalPlan = req.logicalPlan; + pTopic->physicalPlan = req.physicalPlan; pTopic->committedOffset = -1; pTopic->currentOffset = -1; @@ -813,8 +324,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); + SReadHandle handle = { .reader = pReadHandle, .meta = pTq->pMeta }; pTopic->buffer.output[i].pReadHandle = pReadHandle; - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, pReadHandle); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); } taosArrayPush(pConsumer->topics, pTopic); tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); @@ -855,7 +367,10 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { if (pHandle->pBlock == NULL) return false; pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); - if (pHandle->tbUid == pHandle->pBlock->uid) { + /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ + ASSERT(pHandle->tbIdHash); + void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); + if (ret != NULL) { pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); @@ -901,6 +416,11 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { int32_t numOfCols = pHandle->pSchema->numOfCols; int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + //TODO: stable case + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); if (pArray == NULL) { return NULL; @@ -913,7 +433,6 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { j++; } SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; - ASSERT(pColSchema->colId == colId); SColumnInfoData colInfo = {0}; int sz = numOfRows * pColSchema->bytes; colInfo.info.bytes = pColSchema->bytes; @@ -931,6 +450,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { SMemRow row; int32_t kvIdx = 0; + int32_t curRow = 0; tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { // get all wanted col of that block @@ -940,8 +460,9 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { // TODO ASSERT(pCol->colId == pColData->info.colId); void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - memcpy(pColData->pData, val, pCol->bytes); + memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), val, pCol->bytes); } + curRow++; } return pArray; } diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 57e20010e3..d220966ba6 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -584,12 +584,30 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { int64_t bucketKey = key & TQ_BUCKET_MASK; STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { - if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { - if (pNode->handle.valueInTxn) { - pMeta->pDeleter(pNode->handle.valueInTxn); - } + if (pNode->handle.key == key) { + if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { + if (pNode->handle.valueInTxn) { + pMeta->pDeleter(pNode->handle.valueInTxn); + } - pNode->handle.valueInTxn = TQ_DELETE_TOKEN; + pNode->handle.valueInTxn = TQ_DELETE_TOKEN; + tqLinkUnpersist(pMeta, pNode); + return 0; + } + } else { + pNode = pNode->next; + } + } + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; +} + +int32_t tqHandlePurge(STqMetaStore* pMeta, int64_t key) { + int64_t bucketKey = key & TQ_BUCKET_MASK; + STqMetaList* pNode = pMeta->bucket[bucketKey]; + while (pNode) { + if (pNode->handle.key == key) { + pNode->handle.valueInUse = TQ_DELETE_TOKEN; tqLinkUnpersist(pMeta, pNode); return 0; } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1bca7aea10..1351b9627b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -159,7 +159,7 @@ static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGro static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); //static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); -static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle); +static void changeQueryHandleForInterpQuery(tsdbReaderT pHandle); static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); @@ -167,7 +167,7 @@ static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); //static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef); //static void* doFreeColumnInfoData(SArray* pColumnInfoData); //static void* destroyTableCheckInfo(SArray* pTableCheckInfo); -static bool tsdbGetExternalRow(tsdbReadHandleT pHandle); +static bool tsdbGetExternalRow(tsdbReaderT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; @@ -208,7 +208,7 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load return pLocalIdList; } -//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { +//int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT* pHandle) { // STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // // int64_t rows = 0; @@ -424,28 +424,28 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); - return (tsdbReadHandleT)pReadHandle; + return (tsdbReaderT)pReadHandle; _end: - tsdbCleanupQueryHandle(pReadHandle); + tsdbCleanupReadHandle(pReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } -tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } if (emptyQueryTimewindow(pTsdbReadHandle)) { - return (tsdbReadHandleT*) pTsdbReadHandle; + return (tsdbReaderT*) pTsdbReadHandle; } // todo apply the lastkey of table check to avoid to load header file pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList); if (pTsdbReadHandle->pTableCheckInfo == NULL) { -// tsdbCleanupQueryHandle(pTsdbReadHandle); +// tsdbCleanupReadHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } @@ -453,10 +453,10 @@ tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroup tsdbDebug("%p total numOfTable:%" PRIzu " in this query, group %"PRIzu" %s", pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), taosArrayGetSize(groupList->pGroupList), pTsdbReadHandle->idStr); - return (tsdbReadHandleT) pTsdbReadHandle; + return (tsdbReaderT) pTsdbReadHandle; } -void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { +void tsdbResetQueryHandle(tsdbReaderT queryHandle, STsdbQueryCond *pCond) { STsdbReadHandle* pTsdbReadHandle = queryHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { @@ -493,7 +493,7 @@ void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { +void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { STsdbReadHandle* pTsdbReadHandle = queryHandle; pTsdbReadHandle->order = pCond->order; @@ -525,7 +525,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable); if (pTsdbReadHandle->pTableCheckInfo == NULL) { -// tsdbCleanupQueryHandle(pTsdbReadHandle); +// tsdbCleanupReadHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; } @@ -533,7 +533,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -561,7 +561,7 @@ tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup } #if 0 -tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { +tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); if (pTsdbReadHandle == NULL) { return NULL; @@ -581,7 +581,7 @@ tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGro } #endif -SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) { +SArray* tsdbGetQueriedTableList(tsdbReaderT *pHandle) { assert(pHandle != NULL); STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle; @@ -624,7 +624,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { +tsdbReaderT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -2338,7 +2338,7 @@ static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { cur->blockCompleted = false; } #if 0 -int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT* queryHandle, STableBlockDist* pTableBlockInfo) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; @@ -2494,7 +2494,7 @@ static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { } //todo not unref yet, since it is not support multi-group interpolation query -static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReaderT pHandle) { // filter the queried time stamp in the first place STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; @@ -2635,7 +2635,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { } if (exists) { - tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL); + tsdbRetrieveDataBlock((tsdbReaderT*) pTsdbReadHandle, NULL); if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); @@ -2891,7 +2891,7 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { } // handle data in cache situation -bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { +bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { @@ -3051,11 +3051,11 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { // } // //out_of_memory: -// tsdbCleanupQueryHandle(pSecQueryHandle); +// tsdbCleanupReadHandle(pSecQueryHandle); // return terrno; //} -bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { +bool tsdbGetExternalRow(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; SQueryFilePos* cur = &pTsdbReadHandle->cur; @@ -3112,7 +3112,7 @@ bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { // return code; //} -bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) { +bool isTsdbCacheLastRow(tsdbReaderT* pTsdbReadHandle) { return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } @@ -3230,7 +3230,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { return window; } -void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { +void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; SQueryFilePos* cur = &pHandle->cur; @@ -3254,7 +3254,7 @@ void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) { +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SDataStatis** pBlockStatis) { STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle; SQueryFilePos* c = &pHandle->cur; @@ -3309,7 +3309,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataS return TSDB_CODE_SUCCESS; } -SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) { +SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range @@ -3635,29 +3635,29 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd // return TSDB_CODE_SUCCESS; //} -int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) { - STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid); + STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); if (pTbCfg == NULL) { - tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); +// tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; } if (pTbCfg->type != META_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); +// tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client goto _error; } //NOTE: not add ref count for super table SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); - SSchemaWrapper* pTagSchema = metaGetTableSchema(tsdb->pMeta, uid, 0, true); + SSchemaWrapper* pTagSchema = metaGetTableSchema(pMeta, uid, 0, true); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { - int32_t ret = getAllTableList(tsdb->pMeta, uid, res); + int32_t ret = getAllTableList(pMeta, uid, res); if (ret != TSDB_CODE_SUCCESS) { goto _error; } @@ -3665,8 +3665,8 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, - pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); +// tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, +// pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); taosArrayDestroy(res); return ret; @@ -3722,26 +3722,19 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch return terrno; } -#if 0 -int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { - if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; - - STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); - if (pTable == NULL) { +int32_t tsdbGetOneTableGroup(void* pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { + STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); + if (pTbCfg == NULL) { terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; - tsdbUnlockRepoMeta(tsdb); goto _error; } - assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); - if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; - pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = pTable, .lastKey = startKey}; + STableKeyInfo info = {.lastKey = startKey, .uid = uid}; taosArrayPush(group, &info); taosArrayPush(pGroupInfo->pGroupList, &group); @@ -3751,6 +3744,7 @@ int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGr return terrno; } +#if 0 int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; @@ -3826,7 +3820,7 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { } -void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { +void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; if (pTsdbReadHandle == NULL) { return; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f4776fa3c5..a45a9d5a72 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -26,12 +26,14 @@ int vnodeQueryOpen(SVnode *pVnode) { int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in query queue is processing"); + SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta}; switch (pMsg->msgType) { - case TDMT_VND_QUERY: - return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + case TDMT_VND_QUERY:{ + return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg); + } case TDMT_VND_QUERY_CONTINUE: - return qWorkerProcessCQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 326d99ddbb..40cb02176b 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -16,6 +16,7 @@ #include "tq.h" #include "vnd.h" +#if 0 int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_VND_MQ_SET_CUR: @@ -26,6 +27,7 @@ int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { } return 0; } +#endif int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { SRpcMsg *pMsg; @@ -103,7 +105,12 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { taosArrayDestroy(vCreateTbBatchReq.pArray); break; + case TDMT_VND_ALTER_STB: + vTrace("vgId:%d, process drop stb req", pVnode->vgId); + break; case TDMT_VND_DROP_STB: + vTrace("vgId:%d, process drop stb req", pVnode->vgId); + break; case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { // // TODO: handle error diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7af2e9f14f..41ed1739b9 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -224,12 +224,12 @@ typedef struct STaskAttr { // SFilterInfo *pFilters; void* tsdb; -// SMemRef memRef; STableGroupInfo tableGroupInfo; // table list SArray int32_t vgId; SArray *pUdfInfo; // no need to free } STaskAttr; +typedef int32_t (*__optr_prepare_fn_t)(void* param); typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup); typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); @@ -313,8 +313,9 @@ typedef struct SOperatorInfo { struct SOperatorInfo **pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + __optr_prepare_fn_t prepareFn; __operator_fn_t exec; - __optr_cleanup_fn_t cleanup; + __optr_cleanup_fn_t cleanupFn; } SOperatorInfo; enum { @@ -395,7 +396,7 @@ typedef struct STableScanInfo { int32_t current; int32_t reverseTimes; // 0 by default - SQLFunctionCtx *pCtx; // next operator query context + SqlFunctionCtx *pCtx; // next operator query context SResultRowInfo *pResultRowInfo; int32_t *rowCellInfoOffset; SExprInfo *pExpr; @@ -425,7 +426,7 @@ typedef struct SStreamBlockScanInfo { typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; int32_t *rowCellInfoOffset; // offset value for each row result cell info - SQLFunctionCtx *pCtx; + SqlFunctionCtx *pCtx; SSDataBlock *pRes; } SOptrBasicInfo; @@ -564,7 +565,6 @@ typedef struct SOrderOperatorInfo { SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); @@ -607,11 +607,11 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput); +void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); void freeParam(STaskParam *param); int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, @@ -659,8 +659,8 @@ void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId); +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a32268bf47..c47c83ba29 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -39,7 +39,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) } } -int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { +int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input) { if (tinfo == NULL) { return TSDB_CODE_QRY_APP_ERROR; } @@ -50,7 +50,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, GET_TASKID(pTaskInfo)); + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void*) input, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 8635b2d5c8..f632a84c1f 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -51,25 +51,7 @@ static void freeqinfoFn(void *qhandle) { qDestroyTask(*handle); } -void freeParam(STaskParam *param) { - tfree(param->sql); - tfree(param->tagCond); - tfree(param->tbnameCond); - tfree(param->pTableIdList); - taosArrayDestroy(param->pOperator); - tfree(param->pExprs); - tfree(param->pSecExprs); - - tfree(param->pExpr); - tfree(param->pSecExpr); - - tfree(param->pGroupColIndex); - tfree(param->pTagColumnInfo); - tfree(param->pGroupbyExpr); - tfree(param->prevResult); -} - -int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { assert(readHandle != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 6efd7ef87e..2d3085c82b 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -173,13 +173,13 @@ static void getNextTimeWindow(STaskAttr* pQueryAttr, STimeWindow* tw) { } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes); -static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SQLFunctionCtx* pCtx, +static void setResultOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset); -void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx); +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx); -static void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); +static void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); static bool hasMainOutput(STaskAttr *pQueryAttr); @@ -219,14 +219,14 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); -static void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size); +static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void getAlignQueryTimeWindow(STaskAttr *pQueryAttr, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo); -static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); -static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); +static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); +static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); static void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, - SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); + SqlFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); SArray* getOrderCheckColumns(STaskAttr* pQuery); @@ -354,7 +354,7 @@ void* destroyOutputBuf(SSDataBlock* pBlock) { return NULL; } -static bool isSelectivityWithTagsQuery(SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static bool isSelectivityWithTagsQuery(SqlFunctionCtx *pCtx, int32_t numOfOutput) { return true; // bool hasTags = false; // int32_t numOfSelectivity = 0; @@ -754,14 +754,14 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf } static bool chkWindowOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t groupId, SQLFunctionCtx* pCtx, + bool masterscan, SResultRow **pResult, int64_t groupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, int64_t tid, STimeWindow *win, - bool masterscan, SResultRow **pResult, int64_t tableGroupId, SQLFunctionCtx* pCtx, + bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -940,7 +940,7 @@ static int32_t getNumOfRowsInTimeWindow(STaskRuntimeEnv* pRuntimeEnv, SDataBlock return num; } -static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, +static void doApplyFunctions(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; bool hasAggregates = pCtx[0].isAggSet; @@ -1073,7 +1073,7 @@ static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr *pQueryAttr, STimeWindow *p return ekey; } -static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { +static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { if (type == RESULT_ROW_START_INTERP) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].start.key = INT64_MIN; @@ -1112,9 +1112,9 @@ static TSKEY getStartTsKey(STaskAttr* pQueryAttr, STimeWindow* win, const TSKEY* return ts; } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); +static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1124,7 +1124,7 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SQLFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { // if (pCtx[0].functionId == FUNCTION_ARITHM) { // SScalar* pSupport = (SScalarFunctionSupport*) pCtx[0].param[1].pz; // if (pSupport->colList == NULL) { @@ -1141,7 +1141,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo // } } -static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1205,7 +1205,7 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, } } -static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { +static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunctionCtx* pCtx, SSDataBlock* pSDataBlock) { for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { if (functionNeedToExecute(NULL, &pCtx[k])) { pCtx[k].startTs = startTs;// this can be set during create the struct @@ -1214,7 +1214,7 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction } } -static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static void projectApplyFunctions(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx, int32_t numOfOutput) { STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -1242,7 +1242,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; SExprInfo* pExpr = pOperator->pExpr; - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; for (int32_t k = 0; k < pOperator->numOfOutput; ++k) { int32_t functionId = pCtx[k].functionId; @@ -1302,7 +1302,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } } -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, int32_t pos, +static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1333,7 +1333,7 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SQLF return true; } -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFunctionCtx* pCtx, +static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { STaskRuntimeEnv *pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1365,7 +1365,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SQLFun return true; } -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SQLFunctionCtx* pCtx, +static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -1775,7 +1775,7 @@ static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicI int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; - SQLFunctionCtx *pCtx = binfo->pCtx; + SqlFunctionCtx *pCtx = binfo->pCtx; // not assign result buffer yet, add new result buffer, TODO remove it char* d = pData; @@ -1824,7 +1824,7 @@ static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pD return -1; } -static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SqlFunctionCtx *pCtx) { struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // in case of timestamp column, always generated results. @@ -1855,7 +1855,7 @@ static bool functionNeedToExecute(STaskRuntimeEnv *pRuntimeEnv, SQLFunctionCtx * return true; } -void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { +void setBlockStatisInfo(SqlFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* pColumn) { SColumnDataAgg *pAgg = NULL; if (pSDataBlock->pBlockAgg != NULL && TSDB_COL_IS_NORMAL_COL(pColumn->flag)) { @@ -1879,7 +1879,7 @@ void setBlockStatisInfo(SQLFunctionCtx *pCtx, SSDataBlock* pSDataBlock, SColumn* } // set the output buffer for the selectivity + tag query -static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static int32_t setCtxTagColumnInfo(SqlFunctionCtx *pCtx, int32_t numOfOutput) { if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { return TSDB_CODE_SUCCESS; } @@ -1887,8 +1887,8 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { int32_t num = 0; int16_t tagLen = 0; - SQLFunctionCtx* p = NULL; - SQLFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); + SqlFunctionCtx* p = NULL; + SqlFunctionCtx** pTagCtx = calloc(numOfOutput, POINTER_BYTES); if (pTagCtx == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -1920,11 +1920,11 @@ static int32_t setCtxTagColumnInfo(SQLFunctionCtx *pCtx, int32_t numOfOutput) { return TSDB_CODE_SUCCESS; } -static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, +static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t** rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } @@ -1937,7 +1937,7 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI for (int32_t i = 0; i < numOfOutput; ++i) { SSqlExpr *pSqlExpr = &pExpr[i].base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; + SqlFunctionCtx* pCtx = &pFuncCtx[i]; #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2024,10 +2024,10 @@ static SQLFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } -static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { +static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { size_t numOfOutput = taosArrayGetSize(pExprInfo); - SQLFunctionCtx * pFuncCtx = (SQLFunctionCtx *)calloc(numOfOutput, sizeof(SQLFunctionCtx)); + SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); if (pFuncCtx == NULL) { return NULL; } @@ -2042,7 +2042,7 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); SSqlExpr *pSqlExpr = &pExpr->base; - SQLFunctionCtx* pCtx = &pFuncCtx[i]; + SqlFunctionCtx* pCtx = &pFuncCtx[i]; #if 0 SColIndex *pIndex = &pSqlExpr->colInfo; @@ -2128,7 +2128,7 @@ static SQLFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC return pFuncCtx; } -static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +static void* destroySQLFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { if (pCtx == NULL) { return NULL; } @@ -2371,7 +2371,7 @@ _clean: static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); +// tsdbCleanupReadHandle(pRuntimeEnv->pTsdbReadHandle); pRuntimeEnv->pTsdbReadHandle = NULL; // SMemRef* pMemRef = &pQueryAttr->memRef; @@ -2773,7 +2773,7 @@ static void getIntermediateBufInfo(STaskRuntimeEnv* pRuntimeEnv, int32_t* ps, in #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) -//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SQLFunctionCtx *pCtx, int32_t numOfRows) { +//static FORCE_INLINE bool doFilterByBlockStatistics(STaskRuntimeEnv* pRuntimeEnv, SDataStatis *pDataStatis, SqlFunctionCtx *pCtx, int32_t numOfRows) { // STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; // // if (pDataStatis == NULL || pQueryAttr->pFilters == NULL) { @@ -3053,7 +3053,7 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes); static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) { - SQLFunctionCtx* pCtx = pTableScanInfo->pCtx; + SqlFunctionCtx* pCtx = pTableScanInfo->pCtx; uint32_t status = BLK_DATA_NO_NEEDED; int32_t numOfOutput = pTableScanInfo->numOfOutput; @@ -3334,7 +3334,7 @@ int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order) { } /* - * set tag value in SQLFunctionCtx + * set tag value in SqlFunctionCtx * e.g.,tag information into input buffer */ static void doSetTagValueInParam(void* pTable, int32_t tagColId, SVariant *tag, int16_t type, int16_t bytes) { @@ -3375,7 +3375,7 @@ static SColumnInfo* doGetTagColumnInfoById(SColumnInfo* pTagColList, int32_t num return NULL; } -void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SqlFunctionCtx* pCtx, int32_t numOfOutput) { STaskRuntimeEnv* pRuntimeEnv = pOperatorInfo->pRuntimeEnv; SExprInfo *pExpr = pOperatorInfo->pExpr; @@ -3509,7 +3509,7 @@ static void setupQueryRangeForReverseScan(STableScanInfo* pTableScanInfo) { } -void switchCtxOrder(SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { SWITCH_ORDER(pCtx[i].order); } @@ -3531,7 +3531,7 @@ int32_t initResultRow(SResultRow *pResultRow) { * offset[0] offset[1] offset[2] */ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, int64_t uid, int32_t stage) { - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; @@ -3568,7 +3568,7 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t stage, SExecTaskInfo* pTaskInfo) { SOptrBasicInfo *pInfo = &pAggInfo->binfo; - SQLFunctionCtx* pCtx = pInfo->pCtx; + SqlFunctionCtx* pCtx = pInfo->pCtx; SSDataBlock* pDataBlock = pInfo->pRes; int32_t* rowCellInfoOffset = pInfo->rowCellInfoOffset; SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; @@ -3637,7 +3637,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf } } -void copyTsColoum(SSDataBlock* pRes, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) { bool needCopyTs = false; int32_t tsNum = 0; char *src = NULL; @@ -3680,7 +3680,7 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { } } -void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { +void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (isRowEntryInitialized(pResInfo)) { @@ -3701,7 +3701,7 @@ void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) { } } -static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCtx* pCtx, int32_t numOfOutput) { +static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) { // if (pRuntimeEnv->pTsBuf) { // SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order); // bool ret = tsBufNextPos(pRuntimeEnv->pTsBuf); @@ -3722,7 +3722,7 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt pTableScanInfo->reverseTimes = 0; } -void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { +void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskRuntimeEnv *pRuntimeEnv = pOperator->pRuntimeEnv; // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -3832,7 +3832,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3865,7 +3865,7 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pRes } } -void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, +void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SqlFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; @@ -3905,7 +3905,7 @@ void setExecutionContext(STaskRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, in pRuntimeEnv->prevGroupId = tableGroupId; } -void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFunctionCtx* pCtx, +void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); @@ -3928,7 +3928,7 @@ void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLFu } } -void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { +void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; SSqlExpr* pExpr = &pExprInfo->base; @@ -3996,7 +3996,7 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S } // TODO refactor: this funciton should be merged with setparamForStableStddevColumnData function. -void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { +void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExprInfo) { #if 0 STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -4031,7 +4031,7 @@ void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, #endif } -void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { +void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; #if 0 int32_t numOfExprs = pQueryAttr->numOfOutput; @@ -4199,7 +4199,7 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv* pRuntim } } -static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, +static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; @@ -4953,6 +4953,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { STableScanInfo *pTableScanInfo = pOperator->info; SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + // The read handle is not initialized yet, since no qualified tables exists if (pTableScanInfo->pTsdbReadHandle == NULL) { return NULL; } @@ -5071,6 +5072,7 @@ static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; pBlockInfo->rows = 0; + while (tqNextDataBlock(pInfo->readerHandle)) { pTaskInfo->code = tqRetrieveDataBlockInfo(pInfo->readerHandle, pBlockInfo); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { @@ -5342,37 +5344,6 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { return pResBlock; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { - assert(repeatTime > 0 && numOfOutput > 0); - - STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - tfree(pInfo); - tfree(pOperator); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pInfo->pTsdbReadHandle = pTsdbReadHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = 0; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - - pOperator->name = "TableScanOperator"; - pOperator->operatorType = OP_TableScan; - pOperator->blockingOptr = false; - pOperator->status = OP_IN_EXECUTING; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->exec = doTableScan; - pOperator->pTaskInfo = pTaskInfo; - - return pOperator; -} - SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); @@ -5447,14 +5418,13 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; +// pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols; pOperator->exec = doBlockInfoScan; return pOperator; } -SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, uint64_t uid, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExprInfo, SArray* pTableIdList, SExecTaskInfo* pTaskInfo) { SStreamBlockScanInfo* pInfo = calloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5477,7 +5447,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp // set the extract column id to streamHandle tqReadHandleSetColIdList((STqReadHandle* )streamReadHandle, pColList); - tqReadHandleSetTbUid(streamReadHandle, uid); + tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); pInfo->readerHandle = streamReadHandle; @@ -5701,9 +5671,8 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - // pOperator->exec = doGlobalAggregate; - pOperator->cleanup = destroyGlobalAggOperatorInfo; + pOperator->cleanupFn = destroyGlobalAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -5746,7 +5715,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; // pOperator->exec = doMultiwayMergeSort; - pOperator->cleanup = destroyGlobalAggOperatorInfo; + pOperator->cleanupFn = destroyGlobalAggOperatorInfo; return pOperator; } @@ -5850,7 +5819,7 @@ SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->exec = doSort; - pOperator->cleanup = destroyOrderOperatorInfo; + pOperator->cleanupFn = destroyOrderOperatorInfo; pOperator->pRuntimeEnv = pRuntimeEnv; appendDownstream(pOperator, downstream); @@ -6759,8 +6728,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { return; } - if (pOperator->cleanup != NULL) { - pOperator->cleanup(pOperator->info, pOperator->numOfOutput); + if (pOperator->cleanupFn != NULL) { + pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput); } if (pOperator->pDownstream != NULL) { @@ -6814,7 +6783,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; - pOperator->cleanup = destroyAggOperatorInfo; + pOperator->cleanupFn = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6909,7 +6878,7 @@ SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOp pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableAggregate; - pOperator->cleanup = destroyAggOperatorInfo; + pOperator->cleanupFn = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6939,7 +6908,7 @@ SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doProjectOperation; - pOperator->cleanup = destroyProjectOperatorInfo; + pOperator->cleanupFn = destroyProjectOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -6997,7 +6966,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->exec = doFilter; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyConditionOperatorInfo; + pOperator->cleanupFn = destroyConditionOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7039,7 +7008,7 @@ SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOpe pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7064,7 +7033,7 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAllIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7088,7 +7057,7 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doStateWindowAgg; - pOperator->cleanup = destroyStateWindowOperatorInfo; + pOperator->cleanupFn = destroyStateWindowOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7113,7 +7082,7 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSessionWindowAgg; - pOperator->cleanup = destroySWindowOperatorInfo; + pOperator->cleanupFn = destroySWindowOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7137,7 +7106,7 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7161,7 +7130,7 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doAllSTableIntervalAgg; - pOperator->cleanup = destroyBasicOperatorInfo; + pOperator->cleanupFn = destroyBasicOperatorInfo; appendDownstream(pOperator, downstream); @@ -7194,7 +7163,7 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = hashGroupbyAggregate; - pOperator->cleanup = destroyGroupbyOperatorInfo; + pOperator->cleanupFn = destroyGroupbyOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7233,7 +7202,7 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = doFill; - pOperator->cleanup = destroySFillOperatorInfo; + pOperator->cleanupFn = destroySFillOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7281,7 +7250,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI // pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroySlimitOperatorInfo; + pOperator->cleanupFn = destroySlimitOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7439,7 +7408,7 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->cleanup = destroyTagScanOperatorInfo; + pOperator->cleanupFn = destroyTagScanOperatorInfo; return pOperator; } @@ -7579,7 +7548,7 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->exec = hashDistinct; pOperator->pExpr = pExpr; - pOperator->cleanup = destroyDistinctOperatorInfo; + pOperator->cleanupFn = destroyDistinctOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -7749,31 +7718,43 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { return pTaskInfo; } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId); +static tsdbReaderT doCreateDataReader(STableScanPhyNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId); +static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); -SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { - if (pPhyNode->info.type == OP_TableScan) { + if (pPhyNode->info.type == OP_DataBlocksOptScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + tsdbReaderT pDataReader = doCreateDataReader((STableScanPhyNode*) pPhyNode, pHandle, (uint64_t) queryId, taskId); - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); - - return createTableScanOperatorInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); - } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); - - return createDataBlocksOptScanInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + return createDataBlocksOptScanInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (pPhyNode->info.type == OP_Exchange) { SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); } else if (pPhyNode->info.type == OP_StreamScan) { - SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. - return createStreamScanOperatorInfo(readerHandle, pPhyNode->pTargets, pScanPhyNode->uid, pTaskInfo); + SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; // simple child table. + STableGroupInfo groupInfo = {0}; + + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, &groupInfo, queryId, taskId); + + SArray* pa = taosArrayGetP(groupInfo.pGroupList, 0); + ASSERT(taosArrayGetSize(groupInfo.pGroupList) == 1); + + // Transfer the Array of STableKeyInfo into uid list. + size_t numOfTables = taosArrayGetSize(pa); + SArray* idList = taosArrayInit(numOfTables, sizeof(uint64_t)); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pkeyInfo = taosArrayGet(pa, i); + taosArrayPush(idList, &pkeyInfo->uid); + } + + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pPhyNode->pTargets, idList, pTaskInfo); + taosArrayDestroy(idList); + + //TODO destroy groupInfo + return pOperator; } } @@ -7783,18 +7764,23 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask for (int32_t i = 0; i < size; ++i) { SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i); - SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId, taskId); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId); return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo); } } } -static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { +static tsdbReaderT createDataReaderImpl(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readHandle, uint64_t queryId, uint64_t taskId) { STsdbQueryCond cond = {.loadExternalRows = false}; cond.order = pTableScanNode->scan.order; cond.numOfCols = taosArrayGetSize(pTableScanNode->scan.node.pTargets); cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + if (cond.colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + cond.twindow = pTableScanNode->window; cond.type = BLOCK_LOAD_OFFSET_SEQ_ORDER; @@ -7808,32 +7794,27 @@ static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, S cond.colList[i].colId = pSchema->colId; } - return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, taskId); + return tsdbQueryTables(readHandle, &cond, pGroupInfo, queryId, taskId); } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId) { - int32_t code = 0; +static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { + int32_t code = 0; + if (tableType == TSDB_SUPER_TABLE) { + code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); + } else { // Create one table group. + code = tsdbGetOneTableGroup(metaHandle, tableUid, 0, pGroupInfo); + } + + return code; +} + +static tsdbReaderT doCreateDataReader(STableScanPhyNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { STableGroupInfo groupInfo = {0}; uint64_t uid = pTableScanNode->scan.uid; - STimeWindow window = pTableScanNode->window; - int32_t tableType = pTableScanNode->scan.tableType; - - if (tableType == TSDB_SUPER_TABLE) { - code = - tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId, taskId); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - } else { // Create one table group. - groupInfo.numOfTables = 1; - groupInfo.pGroupList = taosArrayInit(1, POINTER_BYTES); - - SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - - STableKeyInfo info = {.lastKey = 0, .uid = uid}; - taosArrayPush(pa, &info); - taosArrayPush(groupInfo.pGroupList, &pa); + int32_t code = doCreateTableGroup(pHandle->meta, pTableScanNode->scan.tableType, uid, &groupInfo, queryId, taskId); + if (code != TSDB_CODE_SUCCESS) { + goto _error; } if (groupInfo.numOfTables == 0) { @@ -7842,14 +7823,14 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, goto _error; } - return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId, taskId); + return createDataReaderImpl(pTableScanNode, &groupInfo, pHandle->reader, queryId, taskId); _error: terrno = code; return NULL; } -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId) { +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; @@ -7859,7 +7840,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* goto _complete; } - (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId, taskId); + (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId); if ((*pTaskInfo)->pRoot == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 1814e1724b..8381a7c585 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -219,7 +219,7 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - int32_t code = qCreateExecTask((void*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); + int32_t code = qCreateExecTask((SReadHandle*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index 8de66584c6..8b83dbe8cd 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -73,7 +73,7 @@ typedef struct STwaInfo { extern int32_t functionCompatList[]; // compatible check array list -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval); +bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** * the numOfRes should be kept, since it may be used later diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index ff2fb95328..694a1c2ca5 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -59,7 +59,7 @@ #define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ __ctx->tag.i = (ts); \ __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ @@ -71,14 +71,14 @@ #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ aggFunc[FUNCTION_TAG].addInput(__ctx); \ } \ } while (0); -void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} +void noop1(SqlFunctionCtx *UNUSED_PARAM(pCtx)) {} -void doFinalizer(SQLFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } +void doFinalizer(SqlFunctionCtx *pCtx) { cleanupResultRowEntry(GET_RES_INFO(pCtx)); } typedef struct tValuePair { SVariant v; @@ -200,7 +200,7 @@ void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell) { pCell->initialized = false; } -int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { +int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num) { int32_t maxOutput = 0; for (int32_t j = 0; j < num; ++j) { int32_t id = pCtx[j].functionId; @@ -223,7 +223,7 @@ int32_t getNumOfResult(SQLFunctionCtx* pCtx, int32_t num) { return maxOutput; } -void resetResultRowEntryResult(SQLFunctionCtx* pCtx, int32_t num) { +void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num) { for (int32_t j = 0; j < num; ++j) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[j]); pResInfo->numOfRes = 0; @@ -473,7 +473,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { return false; } @@ -490,7 +490,7 @@ static bool function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf * * @param pCtx */ -static void function_finalizer(SQLFunctionCtx *pCtx) { +static void function_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); @@ -503,7 +503,7 @@ static void function_finalizer(SQLFunctionCtx *pCtx) { * count function does need the finalize, if data is missing, the default value, which is 0, is used * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ -static void count_function(SQLFunctionCtx *pCtx) { +static void count_function(SqlFunctionCtx *pCtx) { int32_t numOfElem = 0; /* @@ -537,7 +537,7 @@ static void count_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void count_func_merge(SQLFunctionCtx *pCtx) { +static void count_func_merge(SqlFunctionCtx *pCtx) { int64_t *pData = (int64_t *)GET_INPUT_DATA_LIST(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { *((int64_t *)pCtx->pOutput) += pData[i]; @@ -555,7 +555,7 @@ static void count_func_merge(SQLFunctionCtx *pCtx) { * @param filterCols * @return */ -int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +int32_t countRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) { return BLK_DATA_NO_NEEDED; } else { @@ -563,7 +563,7 @@ int32_t countRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { } } -int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +int32_t noDataRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_NO_NEEDED; } #define LIST_ADD_N_DOUBLE_FLOAT(x, ctx, p, t, numOfElem, tsdbType) \ @@ -635,7 +635,7 @@ int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { LOOPCHECK_N(*_data, _list, ctx, tsdbType, sign, notNullElems); \ } while (0) -static void do_sum(SQLFunctionCtx *pCtx) { +static void do_sum(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; // Only the pre-computing information loaded and actual data does not loaded @@ -698,7 +698,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { } } -static void sum_function(SQLFunctionCtx *pCtx) { +static void sum_function(SqlFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer @@ -710,7 +710,7 @@ static void sum_function(SQLFunctionCtx *pCtx) { } } -static void sum_func_merge(SQLFunctionCtx *pCtx) { +static void sum_func_merge(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); @@ -742,16 +742,16 @@ static void sum_func_merge(SQLFunctionCtx *pCtx) { } } -static int32_t statisRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t statisRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_STATIS_NEEDED; } -static int32_t dataBlockRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t dataBlockRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { return BLK_DATA_ALL_NEEDED; } // todo: if column in current data block are null, opt for this case -static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -764,7 +764,7 @@ static int32_t firstFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t c } } -static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i) { return BLK_DATA_NO_NEEDED; } @@ -776,7 +776,7 @@ static int32_t lastFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t co } } -static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order == TSDB_ORDER_DESC) { return BLK_DATA_NO_NEEDED; } @@ -796,7 +796,7 @@ static int32_t firstDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32 } } -static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { +static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if (pCtx->order != pCtx->param[0].i) { return BLK_DATA_NO_NEEDED; } @@ -822,7 +822,7 @@ static int32_t lastDistFuncRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_ * For super table query, once the avg_function/avg_function_f is finished, copy the intermediate * result into output buffer. */ -static void avg_function(SQLFunctionCtx *pCtx) { +static void avg_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf @@ -885,7 +885,7 @@ static void avg_function(SQLFunctionCtx *pCtx) { } } -static void avg_func_merge(SQLFunctionCtx *pCtx) { +static void avg_func_merge(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->pOutput; @@ -907,7 +907,7 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { /* * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ -static void avg_finalizer(SQLFunctionCtx *pCtx) { +static void avg_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == MERGE_STAGE) { @@ -938,7 +938,7 @@ static void avg_finalizer(SQLFunctionCtx *pCtx) { ///////////////////////////////////////////////////////////////////////////////////////////// -static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { +static void minMax_function(SqlFunctionCtx *pCtx, char *pOutput, int32_t isMin, int32_t *notNullElems) { // data in current data block are qualified to the query if (pCtx->isAggSet) { *notNullElems = pCtx->size - pCtx->agg.numOfNull; @@ -994,7 +994,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, if ((*data < val) ^ isMin) { *data = (int32_t)val; for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; + SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -1089,7 +1089,7 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, } } -static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool min_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1135,7 +1135,7 @@ static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf return true; } -static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool max_func_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized } @@ -1184,7 +1184,7 @@ static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInf /* * the output result of min/max function is the final output buffer, not the intermediate result buffer */ -static void min_function(SQLFunctionCtx *pCtx) { +static void min_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->pOutput, 1, ¬NullElems); @@ -1201,7 +1201,7 @@ static void min_function(SQLFunctionCtx *pCtx) { } } -static void max_function(SQLFunctionCtx *pCtx) { +static void max_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; minMax_function(pCtx, pCtx->pOutput, 0, ¬NullElems); @@ -1218,7 +1218,7 @@ static void max_function(SQLFunctionCtx *pCtx) { } } -static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { +static int32_t minmax_merge_impl(SqlFunctionCtx *pCtx, int32_t bytes, char *output, bool isMin) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); @@ -1247,7 +1247,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp *(int32_t *)output = v; for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; + SqlFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; aggFunc[FUNCTION_TAG].addInput(__ctx); } @@ -1303,7 +1303,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp return notNullElems; } -static void min_func_merge(SQLFunctionCtx *pCtx) { +static void min_func_merge(SqlFunctionCtx *pCtx) { int32_t notNullElems = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 1); SET_VAL(pCtx, notNullElems, 1); @@ -1314,7 +1314,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { } } -static void max_func_merge(SQLFunctionCtx *pCtx) { +static void max_func_merge(SqlFunctionCtx *pCtx) { int32_t numOfElem = minmax_merge_impl(pCtx, pCtx->resDataInfo.bytes, pCtx->pOutput, 0); SET_VAL(pCtx, numOfElem, 1); @@ -1334,7 +1334,7 @@ static void max_func_merge(SQLFunctionCtx *pCtx) { (r) += TPOW2(((type *)d)[i] - (delta)); \ } -static void stddev_function(SQLFunctionCtx *pCtx) { +static void stddev_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); @@ -1419,7 +1419,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { } } -static void stddev_finalizer(SQLFunctionCtx *pCtx) { +static void stddev_finalizer(SqlFunctionCtx *pCtx) { SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { @@ -1445,7 +1445,7 @@ int32_t tsCompare(const void* p1, const void* p2) { } } -static void stddev_dst_function(SQLFunctionCtx *pCtx) { +static void stddev_dst_function(SqlFunctionCtx *pCtx) { SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // the second stage to calculate standard deviation @@ -1536,7 +1536,7 @@ static void stddev_dst_function(SQLFunctionCtx *pCtx) { memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); } -static void stddev_dst_merge(SQLFunctionCtx *pCtx) { +static void stddev_dst_merge(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -1553,7 +1553,7 @@ static void stddev_dst_merge(SQLFunctionCtx *pCtx) { } } -static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { +static void stddev_dst_finalizer(SqlFunctionCtx *pCtx) { SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { @@ -1568,7 +1568,7 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////////// -static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -1581,7 +1581,7 @@ static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* } // todo opt for null block -static void first_function(SQLFunctionCtx *pCtx) { +static void first_function(SqlFunctionCtx *pCtx) { if (pCtx->order == TSDB_ORDER_DESC) { return; } @@ -1612,7 +1612,7 @@ static void first_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void first_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); @@ -1630,7 +1630,7 @@ static void first_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t in * format of intermediate result: "timestamp,value" need to compare the timestamp in the first part (before the comma) * to decide if the value is earlier than current intermediate result */ -static void first_dist_function(SQLFunctionCtx *pCtx) { +static void first_dist_function(SqlFunctionCtx *pCtx) { /* * do not to check data in the following cases: * 1. data block that are not loaded @@ -1661,7 +1661,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void first_dist_func_merge(SQLFunctionCtx *pCtx) { +static void first_dist_func_merge(SqlFunctionCtx *pCtx) { assert(pCtx->stableQuery); char * pData = GET_INPUT_DATA_LIST(pCtx); @@ -1691,7 +1691,7 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { * 2. If numOfNull == pBlock->numOfBlocks, the whole block is empty. Otherwise, there is at * least one data in this block that is not null.(TODO opt for this case) */ -static void last_function(SQLFunctionCtx *pCtx) { +static void last_function(SqlFunctionCtx *pCtx) { if (pCtx->order != pCtx->param[0].i) { return; } @@ -1743,7 +1743,7 @@ static void last_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); } -static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { +static void last_data_assign_impl(SqlFunctionCtx *pCtx, char *pData, int32_t index) { int64_t *timestamp = GET_TS_LIST(pCtx); SFirstLastInfo *pInfo = (SFirstLastInfo *)(pCtx->pOutput + pCtx->inputBytes); @@ -1761,7 +1761,7 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind } } -static void last_dist_function(SQLFunctionCtx *pCtx) { +static void last_dist_function(SqlFunctionCtx *pCtx) { /* * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data @@ -1796,7 +1796,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { * final output size, so the main difference between last_dist_func_merge and second_merge * is: the output data format in computing */ -static void last_dist_func_merge(SQLFunctionCtx *pCtx) { +static void last_dist_func_merge(SqlFunctionCtx *pCtx) { char *pData = GET_INPUT_DATA_LIST(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->resDataInfo.bytes); @@ -1824,7 +1824,7 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { /* * NOTE: last_row does not use the interResultBuf to keep the result */ -static void last_row_function(SQLFunctionCtx *pCtx) { +static void last_row_function(SqlFunctionCtx *pCtx) { assert(pCtx->size >= 1); char *pData = GET_INPUT_DATA_LIST(pCtx); @@ -1849,7 +1849,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); } -static void last_row_finalizer(SQLFunctionCtx *pCtx) { +static void last_row_finalizer(SqlFunctionCtx *pCtx) { // do nothing at the first stage SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { @@ -1873,7 +1873,7 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy(dst->pTags, pTags, (size_t)pTagInfo->tagsLen); } else { // the tags are dumped from the ctx tag fields for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { - SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; + SqlFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; if (ctx->functionId == FUNCTION_TS_DUMMY) { ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; ctx->tag.i = tsKey; @@ -2023,7 +2023,7 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } -static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { +static void copyTopBotRes(SqlFunctionCtx *pCtx, int32_t type) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); @@ -2118,7 +2118,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * * top/bottom use the intermediate result buffer to keep the intermediate result */ -static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { +static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer @@ -2136,7 +2136,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { * |-------------pointer area----------|----ts---+-----+-----n tags-----------|----ts---+-----+-----n tags-----------| * +..[Value Pointer1][Value Pointer2].|timestamp|value|tags1|tags2|....|tagsn|timestamp|value|tags1|tags2|....|tagsn+ */ -static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { +static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; tmp += POINTER_BYTES * pCtx->param[0].i; @@ -2150,7 +2150,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { } } -bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval) { +bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; @@ -2206,7 +2206,7 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha } } -static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool top_bottom_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2216,7 +2216,7 @@ static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* return true; } -static void top_function(SQLFunctionCtx *pCtx) { +static void top_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; STopBotInfo *pRes = getTopBotOutputInfo(pCtx); @@ -2252,7 +2252,7 @@ static void top_function(SQLFunctionCtx *pCtx) { } } -static void top_func_merge(SQLFunctionCtx *pCtx) { +static void top_func_merge(SqlFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data @@ -2275,7 +2275,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { } } -static void bottom_function(SQLFunctionCtx *pCtx) { +static void bottom_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; STopBotInfo *pRes = getTopBotOutputInfo(pCtx); @@ -2309,7 +2309,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { } } -static void bottom_func_merge(SQLFunctionCtx *pCtx) { +static void bottom_func_merge(SqlFunctionCtx *pCtx) { STopBotInfo *pInput = (STopBotInfo *)GET_INPUT_DATA_LIST(pCtx); // construct the input data struct from binary data @@ -2332,7 +2332,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { } } -static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { +static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results @@ -2361,7 +2361,7 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { } /////////////////////////////////////////////////////////////////////////////////////////////// -static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool percentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2375,7 +2375,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* return true; } -static void percentile_function(SQLFunctionCtx *pCtx) { +static void percentile_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -2460,7 +2460,7 @@ static void percentile_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void percentile_finalizer(SQLFunctionCtx *pCtx) { +static void percentile_finalizer(SqlFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i : pCtx->param[0].d; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -2484,7 +2484,7 @@ static void buildHistogramInfo(SAPercentileInfo* pInfo) { pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo)); } -static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { +static SAPercentileInfo *getAPerctInfo(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo* pInfo = NULL; @@ -2498,7 +2498,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { return pInfo; } -static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool apercentile_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2510,7 +2510,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo return true; } -static void apercentile_function(SQLFunctionCtx *pCtx) { +static void apercentile_function(SqlFunctionCtx *pCtx) { int32_t notNullElems = 0; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); @@ -2542,7 +2542,7 @@ static void apercentile_function(SQLFunctionCtx *pCtx) { } } -static void apercentile_func_merge(SQLFunctionCtx *pCtx) { +static void apercentile_func_merge(SqlFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_DATA_LIST(pCtx); pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); @@ -2572,7 +2572,7 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); } -static void apercentile_finalizer(SQLFunctionCtx *pCtx) { +static void apercentile_finalizer(SqlFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i : pCtx->param[0].d; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); @@ -2608,7 +2608,7 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2638,7 +2638,7 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInf LEASTSQR_CAL(param, x, y, i, step); \ } -static void leastsquares_function(SQLFunctionCtx *pCtx) { +static void leastsquares_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2724,7 +2724,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); } -static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { +static void leastsquares_finalizer(SqlFunctionCtx *pCtx) { // no data in query SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -2756,12 +2756,12 @@ static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static void date_col_output_function(SQLFunctionCtx *pCtx) { +static void date_col_output_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); *(int64_t *)(pCtx->pOutput) = pCtx->startTs; } -static void col_project_function(SQLFunctionCtx *pCtx) { +static void col_project_function(SqlFunctionCtx *pCtx) { // the number of output rows should not affect the final number of rows, so set it to be 0 if (pCtx->numOfParams == 2) { return; @@ -2791,7 +2791,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { * @param pCtx * @return */ -static void tag_project_function(SQLFunctionCtx *pCtx) { +static void tag_project_function(SqlFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); assert(pCtx->inputBytes == pCtx->resDataInfo.bytes); @@ -2814,9 +2814,9 @@ static void tag_project_function(SQLFunctionCtx *pCtx) { * @param pCtx * @return */ -static void copy_function(SQLFunctionCtx *pCtx); +static void copy_function(SqlFunctionCtx *pCtx); -static void tag_function(SQLFunctionCtx *pCtx) { +static void tag_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); if (pCtx->currentStage == MERGE_STAGE) { copy_function(pCtx); @@ -2825,7 +2825,7 @@ static void tag_function(SQLFunctionCtx *pCtx) { } } -static void copy_function(SQLFunctionCtx *pCtx) { +static void copy_function(SqlFunctionCtx *pCtx) { SET_VAL(pCtx, pCtx->size, 1); char *pData = GET_INPUT_DATA_LIST(pCtx); @@ -2836,7 +2836,7 @@ enum { INITIAL_VALUE_NOT_ASSIGNED = 0, }; -static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -2846,7 +2846,7 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResI return false; } -static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { +static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (!function_setup(pCtx, pResultInfo)) { return false; } @@ -2861,7 +2861,7 @@ static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pRes return false; } -static void deriv_function(SQLFunctionCtx *pCtx) { +static void deriv_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3056,7 +3056,7 @@ static void deriv_function(SQLFunctionCtx *pCtx) { } while (0); // TODO difference in date column -static void diff_function(SQLFunctionCtx *pCtx) { +static void diff_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); @@ -3236,7 +3236,7 @@ char *getArithColumnData(void *param, const char* name, int32_t colId) { return pSupport->data[index] + pSupport->offset * pSupport->colList[index].bytes; } -static void arithmetic_function(SQLFunctionCtx *pCtx) { +static void arithmetic_function(SqlFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->numOfRes += pCtx->size; SScalarFunctionSupport *pSup = (SScalarFunctionSupport *)pCtx->param[1].pz; @@ -3264,7 +3264,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { } ///////////////////////////////////////////////////////////////////////////////// -static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3283,7 +3283,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pRe return true; } -static void spread_function(SQLFunctionCtx *pCtx) { +static void spread_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -3368,7 +3368,7 @@ static void spread_function(SQLFunctionCtx *pCtx) { * here we set the result value back to the intermediate buffer, to apply the finalize the function * the final result is generated in spread_function_finalizer */ -void spread_func_merge(SQLFunctionCtx *pCtx) { +void spread_func_merge(SqlFunctionCtx *pCtx) { SSpreadInfo *pData = (SSpreadInfo *)GET_INPUT_DATA_LIST(pCtx); if (pData->hasResult != DATA_SET_FLAG) { return; @@ -3385,7 +3385,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) { GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } -void spread_function_finalizer(SQLFunctionCtx *pCtx) { +void spread_function_finalizer(SqlFunctionCtx *pCtx) { /* * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary @@ -3423,7 +3423,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * param[2]: end time * @param pCtx */ -static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool twa_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -3444,7 +3444,7 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { +static int32_t twa_function_impl(SqlFunctionCtx* pCtx, int32_t index, int32_t size) { int32_t notNullElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3685,7 +3685,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si return notNullElems; } -static void twa_function(SQLFunctionCtx *pCtx) { +static void twa_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3719,7 +3719,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { * by next input data. The TWA function only applies to each table, so no merge procedure * is required, we simply copy to the resut ot interResBuffer. */ -void twa_function_copy(SQLFunctionCtx *pCtx) { +void twa_function_copy(SqlFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3727,7 +3727,7 @@ void twa_function_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; } -void twa_function_finalizer(SQLFunctionCtx *pCtx) { +void twa_function_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -3752,7 +3752,7 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { * @param pCtx */ -static void interp_function_impl(SQLFunctionCtx *pCtx) { +static void interp_function_impl(SqlFunctionCtx *pCtx) { int32_t type = (int32_t) pCtx->param[2].i; if (type == TSDB_FILL_NONE) { return; @@ -3875,7 +3875,7 @@ static void interp_function_impl(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, 1, 1); } -static void interp_function(SQLFunctionCtx *pCtx) { +static void interp_function(SqlFunctionCtx *pCtx) { // at this point, the value is existed, return directly if (pCtx->size > 0) { bool ascQuery = (pCtx->order == TSDB_ORDER_ASC); @@ -3918,7 +3918,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } -static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool ts_comp_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; // not initialized since it has been initialized } @@ -3929,7 +3929,7 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pR return true; } -static void ts_comp_function(SQLFunctionCtx *pCtx) { +static void ts_comp_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; @@ -3949,7 +3949,7 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } -static void ts_comp_finalize(SQLFunctionCtx *pCtx) { +static void ts_comp_finalize(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -4006,7 +4006,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; } -static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { +static bool rate_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { return false; } @@ -4023,7 +4023,7 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowEntryInfo* pResI return true; } -static void rate_function(SQLFunctionCtx *pCtx) { +static void rate_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; @@ -4076,7 +4076,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { } } -static void rate_func_copy(SQLFunctionCtx *pCtx) { +static void rate_func_copy(SqlFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); @@ -4084,7 +4084,7 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; } -static void rate_finalizer(SQLFunctionCtx *pCtx) { +static void rate_finalizer(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); @@ -4102,7 +4102,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { doFinalizer(pCtx); } -static void irate_function(SQLFunctionCtx *pCtx) { +static void irate_function(SqlFunctionCtx *pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); int32_t notNullElems = 0; @@ -4184,7 +4184,7 @@ static void blockDistInfoFromBinary(const char* data, int32_t len, STableBlockDi } } -static void blockInfo_func(SQLFunctionCtx* pCtx) { +static void blockInfo_func(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -4232,7 +4232,7 @@ static void mergeTableBlockDist(SResultRowEntryInfo* pResInfo, const STableBlock } } -void block_func_merge(SQLFunctionCtx* pCtx) { +void block_func_merge(SqlFunctionCtx* pCtx) { STableBlockDist info = {0}; int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary(((char*)pCtx->pInput) + sizeof(int32_t), len, &info); @@ -4338,7 +4338,7 @@ void generateBlockDistResult(STableBlockDist *pTableBlockDist, char* result) { UNUSED(sz); } -void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { +void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -4380,12 +4380,12 @@ int32_t functionCompatList[] = { }; //typedef struct SFunctionFpSet { -// bool (*init)(struct SQLFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment -// void (*addInput)(struct SQLFunctionCtx *pCtx); +// bool (*init)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment +// void (*addInput)(struct SqlFunctionCtx *pCtx); // // // finalizer must be called after all exec has been executed to generated final result. -// void (*finalize)(struct SQLFunctionCtx *pCtx); -// void (*combine)(struct SQLFunctionCtx *pCtx); +// void (*finalize)(struct SqlFunctionCtx *pCtx); +// void (*combine)(struct SqlFunctionCtx *pCtx); //} SFunctionFpSet; SFunctionFpSet fpSet[1] = { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 2771a9ecf8..b65e637a57 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -121,7 +121,7 @@ void destroyUdfInfo(SUdfInfo* pUdfInfo) { tfree(pUdfInfo); } -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) { +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type) { int32_t output = 0; if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) { diff --git a/source/libs/parser/inc/astCreateContext.h b/source/libs/parser/inc/astCreateContext.h index dcdc6a045b..5458500a82 100644 --- a/source/libs/parser/inc/astCreateContext.h +++ b/source/libs/parser/inc/astCreateContext.h @@ -28,14 +28,12 @@ typedef struct SAstCreateContext { bool notSupport; bool valid; SNode* pRootNode; + SHashObj* pResourceHash; } SAstCreateContext; -int32_t createAstCreateContext(const SParseContext* pQueryCxt, SAstCreateContext* pCxt); +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt); int32_t destroyAstCreateContext(SAstCreateContext* pCxt); -void* acquireRaii(SAstCreateContext* pCxt, void* p); -void* releaseRaii(SAstCreateContext* pCxt, void* p); - #ifdef __cplusplus } #endif diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index c4c5dcba6c..c31160a27e 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -25,20 +25,46 @@ extern "C" { #endif -bool checkTableName(const SToken* pTableName); -SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); -SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); -SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); -SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName); -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); +extern SToken nil_token; + SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode); +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode); + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName); +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral); +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral); +SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode); +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2); +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull); +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias); +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal); +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol); +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill); +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues); + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere); +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList); +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow); +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList); +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving); +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList); +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit); +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit); SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable); SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight); + SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type); -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias); + #ifdef __cplusplus } diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y index 9237f63f35..451aaddc9e 100644 --- a/source/libs/parser/inc/new_sql.y +++ b/source/libs/parser/inc/new_sql.y @@ -6,7 +6,7 @@ %token_prefix NEW_TK_ %token_type { SToken } %default_type { SNode* } -%default_destructor { nodesDestroyNode($$); } +%default_destructor { PARSER_DESTRUCTOR_TRACE; nodesDestroyNode($$); } %extra_argument { SAstCreateContext* pCxt } @@ -22,7 +22,15 @@ #include "ttokendef.h" #include "astCreateFuncs.h" -#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +#if 0 +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif } %syntax_error { @@ -43,157 +51,295 @@ pCxt->valid = false; } -%parse_accept { printf("parsing complete!\n" );} +%parse_accept { PARSER_COMPLETE; } -//%left OR. -//%left AND. -//%right NOT. +%left OR. +%left AND. %left UNION ALL MINUS EXCEPT INTERSECT. -//%left EQ NE ISNULL NOTNULL IS LIKE MATCH NMATCH GLOB BETWEEN IN. -//%left GT GE LT LE. //%left BITAND BITOR LSHIFT RSHIFT. %left NK_PLUS NK_MINUS. //%left DIVIDE TIMES. -%left NK_STAR NK_SLASH. //REM. +%left NK_STAR NK_SLASH NK_REM. //%left CONCAT. //%right UMINUS UPLUS BITNOT. -cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } +cmd ::= SHOW DATABASES. { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } +cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } -cmd ::= query_expression(A). { PARSER_TRACE; pCxt->pRootNode = A; } +/************************************************ literal *************************************************************/ +literal(A) ::= NK_INTEGER(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B); } +literal(A) ::= NK_FLOAT(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &B); } +literal(A) ::= NK_STRING(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } +literal(A) ::= NK_BOOL(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &B); } +literal(A) ::= TIMESTAMP NK_STRING(B). { PARSER_TRACE; A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); } +literal(A) ::= duration_literal(B). { PARSER_TRACE; A = B; } -//////////////////////// value_function ///////////////////////////////// -value_function ::= NK_ID NK_LP value_expression NK_RP. -value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP. +duration_literal(A) ::= NK_VARIABLE(B). { PARSER_TRACE; A = createDurationValueNode(pCxt, &B); } -//////////////////////// value_expression_primary ///////////////////////////////// -value_expression_primary ::= NK_LP value_expression NK_RP. -value_expression_primary ::= nonparenthesized_value_expression_primary. +%type literal_list { SNodeList* } +%destructor literal_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +literal_list(A) ::= literal(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +literal_list(A) ::= literal_list(B) NK_COMMA literal(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } -nonparenthesized_value_expression_primary ::= literal. -// ? -nonparenthesized_value_expression_primary ::= column_reference. +/************************************************ names and identifiers ***********************************************/ +%type db_name { SToken } +%destructor db_name { PARSER_DESTRUCTOR_TRACE; } +db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -literal ::= NK_LITERAL. +%type table_name { SToken } +%destructor table_name { PARSER_DESTRUCTOR_TRACE; } +table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -column_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } -column_reference(A) ::= table_name(B) NK_DOT NK_ID(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } +%type column_name { SToken } +%destructor column_name { PARSER_DESTRUCTOR_TRACE; } +column_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -//////////////////////// value_expression ///////////////////////////////// -value_expression ::= common_value_expression. +%type function_name { SToken } +%destructor function_name { PARSER_DESTRUCTOR_TRACE; } +function_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -common_value_expression ::= numeric_value_expression. +%type table_alias { SToken } +%destructor table_alias { PARSER_DESTRUCTOR_TRACE; } +table_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -numeric_value_expression ::= numeric_primary. -numeric_value_expression ::= NK_PLUS numeric_primary. -numeric_value_expression ::= NK_MINUS numeric_primary. -numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression. -numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression. +%type column_alias { SToken } +%destructor column_alias { PARSER_DESTRUCTOR_TRACE; } +column_alias(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -numeric_primary ::= value_expression_primary. -numeric_primary ::= value_function. +/************************************************ expression **********************************************************/ +expression(A) ::= literal(B). { PARSER_TRACE; A = B; } +//expression(A) ::= NK_QUESTION(B). { PARSER_TRACE; A = B; } +//expression(A) ::= pseudo_column(B). { PARSER_TRACE; A = B; } +expression(A) ::= column_reference(B). { PARSER_TRACE; A = B; } +expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { PARSER_TRACE; A = createFunctionNode(pCxt, &B, C); } +//expression(A) ::= cast_expression(B). { PARSER_TRACE; A = B; } +//expression(A) ::= case_expression(B). { PARSER_TRACE; A = B; } +expression(A) ::= subquery(B). { PARSER_TRACE; A = B; } +expression(A) ::= NK_LP expression(B) NK_RP. { PARSER_TRACE; A = B; } +expression(A) ::= NK_PLUS expression(B). { PARSER_TRACE; A = B; } +expression(A) ::= NK_MINUS expression(B). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, NULL); } +expression(A) ::= expression(B) NK_PLUS expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_ADD, B, C); } +expression(A) ::= expression(B) NK_MINUS expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_SUB, B, C); } +expression(A) ::= expression(B) NK_STAR expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MULTI, B, C); } +expression(A) ::= expression(B) NK_SLASH expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_DIV, B, C); } +expression(A) ::= expression(B) NK_REM expression(C). { PARSER_TRACE; A = createOperatorNode(pCxt, OP_TYPE_MOD, B, C); } -//////////////////////// query_specification ///////////////////////////////// -query_specification(A) ::= SELECT set_quantifier_opt(B) select_list(C) from_clause(D). { PARSER_TRACE; A = createSelectStmt(pCxt, B, C, D); } +%type expression_list { SNodeList* } +%destructor expression_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +expression_list(A) ::= expression(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +expression_list(A) ::= expression_list(B) NK_COMMA expression(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } -%type set_quantifier_opt { bool } -%destructor set_quantifier_opt {} -set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } -set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } -set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } +column_reference(A) ::= column_name(B). { PARSER_TRACE; A = createColumnNode(pCxt, NULL, &B); } +column_reference(A) ::= table_name(B) NK_DOT column_name(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } -%type select_list { SNodeList* } -%destructor select_list { nodesDestroyNodeList($$); } -select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } -select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } +//pseudo_column(A) ::= NK_NOW. { PARSER_TRACE; A = createFunctionNode(pCxt, NULL, NULL); } -%type select_sublist { SNodeList* } -%destructor select_sublist { nodesDestroyNodeList($$); } -select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } -select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } +/************************************************ predicate ***********************************************************/ +predicate(A) ::= expression(B) compare_op(C) expression(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); } +//predicate(A) ::= expression(B) compare_op sub_type expression(B). +predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createBetweenAnd(pCxt, B, C, D); } +predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createNotBetweenAnd(pCxt, C, B, D); } +predicate(A) ::= expression(B) IS NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, true); } +predicate(A) ::= expression(B) IS NOT NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, B, false); } +predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, B, D); } -select_item(A) ::= value_expression(B). { PARSER_TRACE; A = B; } -select_item(A) ::= value_expression(B) AS NK_ID(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } -select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } +%type compare_op { EOperatorType } +%destructor compare_op { PARSER_DESTRUCTOR_TRACE; } +compare_op(A) ::= NK_LT. { PARSER_TRACE; A = OP_TYPE_LOWER_THAN; } +compare_op(A) ::= NK_GT. { PARSER_TRACE; A = OP_TYPE_GREATER_THAN; } +compare_op(A) ::= NK_LE. { PARSER_TRACE; A = OP_TYPE_LOWER_EQUAL; } +compare_op(A) ::= NK_GE. { PARSER_TRACE; A = OP_TYPE_GREATER_EQUAL; } +compare_op(A) ::= NK_NE. { PARSER_TRACE; A = OP_TYPE_NOT_EQUAL; } +compare_op(A) ::= NK_EQ. { PARSER_TRACE; A = OP_TYPE_EQUAL; } +compare_op(A) ::= LIKE. { PARSER_TRACE; A = OP_TYPE_LIKE; } +compare_op(A) ::= NOT LIKE. { PARSER_TRACE; A = OP_TYPE_NOT_LIKE; } +compare_op(A) ::= MATCH. { PARSER_TRACE; A = OP_TYPE_MATCH; } +compare_op(A) ::= NMATCH. { PARSER_TRACE; A = OP_TYPE_NMATCH; } -from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } +%type in_op { EOperatorType } +%destructor in_op { PARSER_DESTRUCTOR_TRACE; } +in_op(A) ::= IN. { PARSER_TRACE; A = OP_TYPE_IN; } +in_op(A) ::= NOT IN. { PARSER_TRACE; A = OP_TYPE_NOT_IN; } -//%type table_reference_list { SNodeList* } -//%destructor table_reference_list { nodesDestroyNodeList($$); } -table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } -//table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, B, C); } +in_predicate_value(A) ::= NK_LP expression_list(B) NK_RP. { PARSER_TRACE; A = createNodeListNode(pCxt, B); } -//table_reference(A) ::= NK_ID(B). { PARSER_TRACE; A = createRealTableNode(pCxt, ); } -table_reference(A) ::= table_factor(B). { PARSER_TRACE; A = B; } -//table_reference ::= joined_table. +/************************************************ boolean_value_expression ********************************************/ +boolean_value_expression(A) ::= boolean_primary(B). { PARSER_TRACE; A = B; } +boolean_value_expression(A) ::= NOT boolean_primary(B). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, B, NULL); } +boolean_value_expression(A) ::= + boolean_value_expression(B) OR boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, B, C); } +boolean_value_expression(A) ::= + boolean_value_expression(B) AND boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, B, C); } -table_factor(A) ::= table_primary(B). { PARSER_TRACE; A = B; } +boolean_primary(A) ::= predicate(B). { PARSER_TRACE; A = B; } +boolean_primary(A) ::= NK_LP boolean_value_expression(B) NK_RP. { PARSER_TRACE; A = B; } -table_primary(A) ::= table_name(B). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B); } -table_primary(A) ::= db_name(B) NK_DOT table_name(C). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C); } -table_primary ::= derived_table. +/************************************************ from_clause *********************************************************/ +from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } -derived_table ::= table_subquery. +table_reference_list(A) ::= table_reference(B). { PARSER_TRACE; A = B; } +table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { PARSER_TRACE; A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } -%type db_name { SToken } -db_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } -%type table_name { SToken } -table_name(A) ::= NK_ID(B). { PARSER_TRACE; A = B; } +/************************************************ table_reference *****************************************************/ +table_reference(A) ::= table_primary(B). { PARSER_TRACE; A = B; } +table_reference(A) ::= joined_table(B). { PARSER_TRACE; A = B; } -//////////////////////// subquery ///////////////////////////////// -subquery ::= NK_LR query_expression NK_RP. +table_primary(A) ::= table_name(B) alias_opt(C). { PARSER_TRACE; A = createRealTableNode(pCxt, NULL, &B, &C); } +table_primary(A) ::= db_name(B) NK_DOT table_name(C) alias_opt(D). { PARSER_TRACE; A = createRealTableNode(pCxt, &B, &C, &D); } +table_primary(A) ::= subquery(B) alias_opt(C). { PARSER_TRACE; A = createTempTableNode(pCxt, B, &C); } +table_primary ::= parenthesized_joined_table. -table_subquery ::= subquery. +%type alias_opt { SToken } +%destructor alias_opt { PARSER_DESTRUCTOR_TRACE; } +alias_opt(A) ::= . { PARSER_TRACE; A = nil_token; } +alias_opt(A) ::= table_alias(B). { PARSER_TRACE; A = B; } +alias_opt(A) ::= AS table_alias(B). { PARSER_TRACE; A = B; } -// query_expression -query_expression(A) ::= with_clause_opt query_expression_body(B) order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { - PARSER_TRACE; - addOrderByList(pCxt, B, C); - addSlimit(pCxt, B, D); - addLimit(pCxt, B, E); - A = B; - } +parenthesized_joined_table(A) ::= NK_LP joined_table(B) NK_RP. { PARSER_TRACE; A = B; } +parenthesized_joined_table(A) ::= NK_LP parenthesized_joined_table(B) NK_RP. { PARSER_TRACE; A = B; } -// WITH AS -with_clause_opt ::= . {} +/************************************************ joined_table ********************************************************/ +joined_table(A) ::= + table_reference(B) join_type(C) JOIN table_reference(D) ON search_condition(E). { PARSER_TRACE; A = createJoinTableNode(pCxt, C, B, D, E); } -query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } -query_expression_body(A) ::= query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } +%type join_type { EJoinType } +%destructor join_type { PARSER_DESTRUCTOR_TRACE; } +join_type(A) ::= INNER. { PARSER_TRACE; A = JOIN_TYPE_INNER; } -query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } -query_primary(A) ::= NK_LP query_expression_body(B) order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP. { PARSER_TRACE; A = B;} +/************************************************ query_specification *************************************************/ +query_specification(A) ::= + SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) + partition_by_clause_opt(F) twindow_clause_opt(G) + group_by_clause_opt(H) having_clause_opt(I). { + PARSER_TRACE; + A = createSelectStmt(pCxt, B, C, D); + A = addWhereClause(pCxt, A, E); + A = addPartitionByClause(pCxt, A, F); + A = addWindowClauseClause(pCxt, A, G); + A = addGroupByClause(pCxt, A, H); + A = addHavingClause(pCxt, A, I); + } -%type order_by_clause_opt { SNodeList* } -%destructor order_by_clause_opt { nodesDestroyNodeList($$); } -order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } -order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } +%type set_quantifier_opt { bool } +%destructor set_quantifier_opt { PARSER_DESTRUCTOR_TRACE; } +set_quantifier_opt(A) ::= . { PARSER_TRACE; A = false; } +set_quantifier_opt(A) ::= DISTINCT. { PARSER_TRACE; A = true; } +set_quantifier_opt(A) ::= ALL. { PARSER_TRACE; A = false; } -slimit_clause_opt(A) ::= . { A = NULL; } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } +%type select_list { SNodeList* } +%destructor select_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_list(A) ::= NK_STAR. { PARSER_TRACE; A = NULL; } +select_list(A) ::= select_sublist(B). { PARSER_TRACE; A = B; } -limit_clause_opt(A) ::= . { A = NULL; } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { A = createLimitNode(pCxt, &B, &C); } -limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { A = createLimitNode(pCxt, &B, &C); } +%type select_sublist { SNodeList* } +%destructor select_sublist { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } -//////////////////////// sort_specification_list ///////////////////////////////// -%type sort_specification_list { SNodeList* } -%destructor sort_specification_list { nodesDestroyNodeList($$); } -sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } -sort_specification_list(A) ::= sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } +select_item(A) ::= expression(B). { PARSER_TRACE; A = B; } +select_item(A) ::= expression(B) column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= expression(B) AS column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, B, &C); } +select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } -sort_specification(A) ::= value_expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } +where_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +where_clause_opt(A) ::= WHERE search_condition(B). { PARSER_TRACE; A = B; } + +%type partition_by_clause_opt { SNodeList* } +%destructor partition_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +partition_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +partition_by_clause_opt(A) ::= PARTITION BY expression_list(B). { PARSER_TRACE; A = B; } + +twindow_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +twindow_clause_opt(A) ::= + SESSION NK_LP column_reference(B) NK_COMMA NK_INTEGER(C) NK_RP. { PARSER_TRACE; A = createSessionWindowNode(pCxt, B, &C); } +twindow_clause_opt(A) ::= STATE_WINDOW NK_LP column_reference(B) NK_RP. { PARSER_TRACE; A = createStateWindowNode(pCxt, B); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_RP sliding_opt(C) fill_opt(D). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, NULL, C, D); } +twindow_clause_opt(A) ::= + INTERVAL NK_LP duration_literal(B) NK_COMMA duration_literal(C) NK_RP + sliding_opt(D) fill_opt(E). { PARSER_TRACE; A = createIntervalWindowNode(pCxt, B, C, D, E); } + +sliding_opt(A) ::= . { PARSER_TRACE; A = NULL; } +sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. { PARSER_TRACE; A = B; } + +fill_opt(A) ::= . { PARSER_TRACE; A = NULL; } +fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, B, NULL); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { PARSER_TRACE; A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } + +%type fill_mode { EFillMode } +%destructor fill_mode { PARSER_DESTRUCTOR_TRACE; } +fill_mode(A) ::= NONE. { PARSER_TRACE; A = FILL_MODE_NONE; } +fill_mode(A) ::= PREV. { PARSER_TRACE; A = FILL_MODE_PREV; } +fill_mode(A) ::= NULL. { PARSER_TRACE; A = FILL_MODE_NULL; } +fill_mode(A) ::= LINEAR. { PARSER_TRACE; A = FILL_MODE_LINEAR; } +fill_mode(A) ::= NEXT. { PARSER_TRACE; A = FILL_MODE_NEXT; } + +%type group_by_clause_opt { SNodeList* } +%destructor group_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +group_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +group_by_clause_opt(A) ::= GROUP BY expression_list(B). { PARSER_TRACE; A = B; } + +having_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +having_clause_opt(A) ::= HAVING search_condition(B). { PARSER_TRACE; A = B; } + +/************************************************ query_expression ****************************************************/ +query_expression(A) ::= + query_expression_body(B) + order_by_clause_opt(C) slimit_clause_opt(D) limit_clause_opt(E). { + PARSER_TRACE; + A = addOrderByClause(pCxt, B, C); + A = addSlimitClause(pCxt, A, D); + A = addLimitClause(pCxt, A, E); + } + +query_expression_body(A) ::= query_primary(B). { PARSER_TRACE; A = B; } +query_expression_body(A) ::= + query_expression_body(B) UNION ALL query_expression_body(D). { PARSER_TRACE; A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } + +query_primary(A) ::= query_specification(B). { PARSER_TRACE; A = B; } +query_primary(A) ::= + NK_LP query_expression_body(B) + order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP. { PARSER_TRACE; A = B;} + +%type order_by_clause_opt { SNodeList* } +%destructor order_by_clause_opt { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +order_by_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +order_by_clause_opt(A) ::= ORDER BY sort_specification_list(B). { PARSER_TRACE; A = B; } + +slimit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(B) SOFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +slimit_clause_opt(A) ::= SLIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } + +limit_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, NULL); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(B) OFFSET NK_INTEGER(C). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } +limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). { PARSER_TRACE; A = createLimitNode(pCxt, &B, &C); } + +/************************************************ subquery ************************************************************/ +subquery(A) ::= NK_LR query_expression(B) NK_RP. { PARSER_TRACE; A = B; } + +/************************************************ search_condition ****************************************************/ +search_condition(A) ::= boolean_value_expression(B). { PARSER_TRACE; A = B; } + +/************************************************ sort_specification_list *********************************************/ +%type sort_specification_list { SNodeList* } +%destructor sort_specification_list { PARSER_DESTRUCTOR_TRACE; nodesDestroyList($$); } +sort_specification_list(A) ::= sort_specification(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } +sort_specification_list(A) ::= + sort_specification_list(B) NK_COMMA sort_specification(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } + +sort_specification(A) ::= + expression(B) ordering_specification_opt(C) null_ordering_opt(D). { PARSER_TRACE; A = createOrderByExprNode(pCxt, B, C, D); } %type ordering_specification_opt EOrder -%destructor ordering_specification_opt {} -ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } -ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } -ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } +%destructor ordering_specification_opt { PARSER_DESTRUCTOR_TRACE; } +ordering_specification_opt(A) ::= . { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= ASC. { PARSER_TRACE; A = ORDER_ASC; } +ordering_specification_opt(A) ::= DESC. { PARSER_TRACE; A = ORDER_DESC; } %type null_ordering_opt ENullOrder -%destructor null_ordering_opt {} -null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } -null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } -null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } +%destructor null_ordering_opt { PARSER_DESTRUCTOR_TRACE; } +null_ordering_opt(A) ::= . { PARSER_TRACE; A = NULL_ORDER_DEFAULT; } +null_ordering_opt(A) ::= NULLS FIRST. { PARSER_TRACE; A = NULL_ORDER_FIRST; } +null_ordering_opt(A) ::= NULLS LAST. { PARSER_TRACE; A = NULL_ORDER_LAST; } diff --git a/source/libs/parser/src/astCreateContext.c b/source/libs/parser/src/astCreateContext.c index 42c5257360..a8c82780ca 100644 --- a/source/libs/parser/src/astCreateContext.c +++ b/source/libs/parser/src/astCreateContext.c @@ -16,25 +16,14 @@ #include "ttoken.h" #include "astCreateContext.h" -void* acquireRaii(SAstCreateContext* pCxt, void* p) { - if (NULL == p) { - return NULL; - } - return p; +int32_t createAstCreateContext(SParseContext* pQueryCxt, SAstCreateContext* pCxt) { + pCxt->pQueryCxt = pQueryCxt; + pCxt->notSupport = false; + pCxt->valid = true; + pCxt->pRootNode = NULL; + return TSDB_CODE_SUCCESS; } -void* releaseRaii(SAstCreateContext* pCxt, void* p) { - if (NULL == p) { - return NULL; - } - return p; +int32_t destroyAstCreateContext(SAstCreateContext* pCxt) { + return TSDB_CODE_SUCCESS; } - -int32_t createAstCreater(const SParseContext* pQueryCxt, SAstCreateContext* pCxt) { - -} - -int32_t destroyAstCreater(SAstCreateContext* pCxt) { - -} - diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index 337ae78d75..f054649e1a 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -16,76 +16,298 @@ #include "astCreateFuncs.h" -#include "astCreateContext.h" +#define CHECK_OUT_OF_MEM(p) \ + do { \ + if (NULL == (p)) { \ + pCxt->valid = false; \ + return NULL; \ + } \ + } while (0) -bool checkTableName(const SToken* pTableName) { - printf("%p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); - return pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; +SToken nil_token = { .type = TK_NIL, .n = 0, .z = NULL }; + +static bool checkDbName(SAstCreateContext* pCxt, const SToken* pDbName) { + if (NULL == pDbName) { + return true; + } + pCxt->valid = pDbName->n < TSDB_DB_NAME_LEN ? true : false; + return pCxt->valid; } -SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { - +static bool checkTableName(SAstCreateContext* pCxt, const SToken* pTableName) { + if (NULL == pTableName) { + return true; + } + pCxt->valid = pTableName->n < TSDB_TABLE_NAME_LEN ? true : false; + return pCxt->valid; } -SNode* addOrderByList(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { - -} - -SNode* addSlimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { - -} - -SNode* addLimit(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { - -} - -SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) { - -} - -SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { - +static bool checkColumnName(SAstCreateContext* pCxt, const SToken* pColumnName) { + if (NULL == pColumnName) { + return true; + } + pCxt->valid = pColumnName->n < TSDB_COL_NAME_LEN ? true : false; + return pCxt->valid; } SNodeList* createNodeList(SAstCreateContext* pCxt, SNode* pNode) { + SNodeList* list = nodesMakeList(); + CHECK_OUT_OF_MEM(list); + return nodesListAppend(list, pNode); +} +SNodeList* addNodeToList(SAstCreateContext* pCxt, SNodeList* pList, SNode* pNode) { + return nodesListAppend(pList, pNode); +} + +SNode* createColumnNode(SAstCreateContext* pCxt, const SToken* pTableName, const SToken* pColumnName) { + if (!checkTableName(pCxt, pTableName) || !checkColumnName(pCxt, pColumnName)) { + return NULL; + } + SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + CHECK_OUT_OF_MEM(col); + if (NULL != pTableName) { + strncpy(col->tableName, pTableName->z, pTableName->n); + } + strncpy(col->colName, pColumnName->z, pColumnName->n); + return (SNode*)col; +} + +SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + // todo + return (SNode*)val; +} + +SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral) { + SValueNode* val = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + CHECK_OUT_OF_MEM(val); + // todo + return (SNode*)val; +} + +SNode* addMinusSign(SAstCreateContext* pCxt, SNode* pNode) { + // todo +} + +SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2) { + SLogicConditionNode* cond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->condType = type; + cond->pParameterList = nodesMakeList(); + nodesListAppend(cond->pParameterList, pParam1); + nodesListAppend(cond->pParameterList, pParam2); + return (SNode*)cond; +} + +SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight) { + SOperatorNode* op = (SOperatorNode*)nodesMakeNode(QUERY_NODE_OPERATOR); + CHECK_OUT_OF_MEM(op); + op->opType = type; + op->pLeft = pLeft; + op->pRight = pRight; + return (SNode*)op; +} + +SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, + createOperatorNode(pCxt, OP_TYPE_GREATER_EQUAL, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_LOWER_EQUAL, pExpr, pRight)); +} + +SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight) { + return createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, + createOperatorNode(pCxt, OP_TYPE_LOWER_THAN, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, pExpr, pRight)); +} + +SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull) { + SIsNullCondNode* cond = (SIsNullCondNode*)nodesMakeNode(QUERY_NODE_IS_NULL_CONDITION); + CHECK_OUT_OF_MEM(cond); + cond->pExpr = pExpr; + cond->isNull = isNull; + return (SNode*)cond; +} + +SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { + SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + CHECK_OUT_OF_MEM(func); + strncpy(func->functionName, pFuncName->z, pFuncName->n); + func->pParameterList = pParameterList; + return (SNode*)func; +} + +SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList) { + SNodeListNode* list = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); + CHECK_OUT_OF_MEM(list); + list->pNodeList = pList; + return (SNode*)list; +} + +SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias) { + if (!checkDbName(pCxt, pDbName) || !checkTableName(pCxt, pTableName)) { + return NULL; + } + SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + CHECK_OUT_OF_MEM(realTable); + if (NULL != pDbName) { + strncpy(realTable->dbName, pDbName->z, pDbName->n); + } + strncpy(realTable->table.tableName, pTableName->z, pTableName->n); + return (SNode*)realTable; +} + +SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias) { + STempTableNode* tempTable = (STempTableNode*)nodesMakeNode(QUERY_NODE_TEMP_TABLE); + CHECK_OUT_OF_MEM(tempTable); + tempTable->pSubquery = pSubquery; + return (SNode*)tempTable; +} + +SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond) { + SJoinTableNode* joinTable = (SJoinTableNode*)nodesMakeNode(QUERY_NODE_JOIN_TABLE); + CHECK_OUT_OF_MEM(joinTable); + joinTable->joinType = type; + joinTable->pLeft = pLeft; + joinTable->pRight = pRight; + joinTable->pOnCond = pJoinCond; + return (SNode*)joinTable; +} + +SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { + SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); + CHECK_OUT_OF_MEM(limitNode); + // limitNode->limit = limit; + // limitNode->offset = offset; + return (SNode*)limitNode; } SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder) { - + SOrderByExprNode* orderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + CHECK_OUT_OF_MEM(orderByExpr); + orderByExpr->pExpr = pExpr; + orderByExpr->order = order; + orderByExpr->nullOrder = nullOrder; + return (SNode*)orderByExpr; } -SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName) { - SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); - if (NULL != pDbName) { - printf("DbName %p : %d, %d, %s\n", pDbName, pDbName->type, pDbName->n, pDbName->z); - strncpy(realTable->dbName, pDbName->z, pDbName->n); +SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, const SToken* pVal) { + SSessionWindowNode* session = (SSessionWindowNode*)nodesMakeNode(QUERY_NODE_SESSION_WINDOW); + CHECK_OUT_OF_MEM(session); + session->pCol = pCol; + // session->gap = getInteger(pVal); + return (SNode*)session; +} + +SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pCol) { + SStateWindowNode* state = (SStateWindowNode*)nodesMakeNode(QUERY_NODE_STATE_WINDOW); + CHECK_OUT_OF_MEM(state); + state->pCol = pCol; + return (SNode*)state; +} + +SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding, SNode* pFill) { + SIntervalWindowNode* interval = (SIntervalWindowNode*)nodesMakeNode(QUERY_NODE_INTERVAL_WINDOW); + CHECK_OUT_OF_MEM(interval); + interval->pInterval = pInterval; + interval->pOffset = pOffset; + interval->pSliding = pSliding; + interval->pFill = pFill; + return (SNode*)interval; +} + +SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues) { + SFillNode* fill = (SFillNode*)nodesMakeNode(QUERY_NODE_FILL); + CHECK_OUT_OF_MEM(fill); + fill->mode = mode; + fill->pValues = pValues; + return (SNode*)fill; +} + +SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { + strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n); + return pNode; +} + +SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWhere = pWhere; } - printf("TableName %p : %d, %d, %s\n", pTableName, pTableName->type, pTableName->n, pTableName->z); - strncpy(realTable->table.tableName, pTableName->z, pTableName->n); - return acquireRaii(pCxt, realTable); + return pStmt; +} + +SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pPartitionByList = pPartitionByList; + } + return pStmt; +} + +SNode* addWindowClauseClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWindow) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pWindow = pWindow; + } + return pStmt; +} + +SNode* addGroupByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pGroupByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pGroupByList = pGroupByList; + } + return pStmt; +} + +SNode* addHavingClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pHaving) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pHaving = pHaving; + } + return pStmt; +} + +SNode* addOrderByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pOrderByList) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pOrderByList = pOrderByList; + } + return pStmt; +} + +SNode* addSlimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pSlimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pSlimit = pSlimit; + } + return pStmt; +} + +SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit) { + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { + ((SSelectStmt*)pStmt)->pLimit = pLimit; + } + return pStmt; } SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable) { SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); + CHECK_OUT_OF_MEM(select); select->isDistinct = isDistinct; if (NULL == pProjectionList) { select->isStar = true; } - select->pProjectionList = releaseRaii(pCxt, pProjectionList); - printf("pTable = %p, name = %s\n", pTable, ((SRealTableNode*)pTable)->table.tableName); - select->pFromTable = releaseRaii(pCxt, pTable); - return acquireRaii(pCxt, select); + select->pProjectionList = pProjectionList; + select->pFromTable = pTable; + return (SNode*)select; } SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight) { - + SSetOperator* setOp = (SSetOperator*)nodesMakeNode(QUERY_NODE_SET_OPERATOR); + CHECK_OUT_OF_MEM(setOp); + setOp->opType = type; + setOp->pLeft = pLeft; + setOp->pRight = pRight; + return (SNode*)setOp; } SNode* createShowStmt(SAstCreateContext* pCxt, EShowStmtType type) { - -} - -SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { - + SShowStmt* show = (SShowStmt*)nodesMakeNode(QUERY_NODE_SHOW_STMT); + CHECK_OUT_OF_MEM(show); + show->showType = type; + return (SNode*)show; } diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c index 7bf29c45f8..20f82c150a 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/new_sql.c @@ -37,7 +37,15 @@ #include "ttokendef.h" #include "astCreateFuncs.h" -#define PARSER_TRACE printf("rule = %s\n", yyRuleName[yyruleno]) +#if 0 +#define PARSER_TRACE printf("lemon rule = %s\n", yyRuleName[yyruleno]) +#define PARSER_DESTRUCTOR_TRACE printf("lemon destroy token = %s\n", yyTokenName[yymajor]) +#define PARSER_COMPLETE printf("parsing complete!\n" ) +#else +#define PARSER_TRACE +#define PARSER_DESTRUCTOR_TRACE +#define PARSER_COMPLETE +#endif /**************** End of %include directives **********************************/ /* These constants specify the various numeric values for terminal symbols ** in a format understandable to "makeheaders". This section is blank unless @@ -101,18 +109,21 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 71 -#define YYACTIONTYPE unsigned char +#define YYNOCODE 125 +#define YYACTIONTYPE unsigned short int #define NewParseTOKENTYPE SToken typedef union { int yyinit; NewParseTOKENTYPE yy0; - bool yy9; - SNodeList* yy30; - SToken yy67; - ENullOrder yy68; - EOrder yy108; - SNode* yy130; + EOperatorType yy40; + EFillMode yy44; + SToken yy79; + ENullOrder yy107; + EJoinType yy162; + SNodeList* yy174; + EOrder yy188; + SNode* yy212; + bool yy237; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -127,17 +138,17 @@ typedef union { #define NewParseCTX_PARAM #define NewParseCTX_FETCH #define NewParseCTX_STORE -#define YYNSTATE 63 -#define YYNRULE 68 -#define YYNTOKEN 35 -#define YY_MAX_SHIFT 62 -#define YY_MIN_SHIFTREDUCE 114 -#define YY_MAX_SHIFTREDUCE 181 -#define YY_ERROR_ACTION 182 -#define YY_ACCEPT_ACTION 183 -#define YY_NO_ACTION 184 -#define YY_MIN_REDUCE 185 -#define YY_MAX_REDUCE 252 +#define YYNSTATE 149 +#define YYNRULE 135 +#define YYNTOKEN 73 +#define YY_MAX_SHIFT 148 +#define YY_MIN_SHIFTREDUCE 245 +#define YY_MAX_SHIFTREDUCE 379 +#define YY_ERROR_ACTION 380 +#define YY_ACCEPT_ACTION 381 +#define YY_NO_ACTION 382 +#define YY_MIN_REDUCE 383 +#define YY_MAX_REDUCE 517 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -204,89 +215,199 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (229) +#define YY_ACTTAB_COUNT (656) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 30, 50, 30, 30, 30, 30, 55, 50, 30, 30, - /* 10 */ 10, 9, 29, 62, 51, 195, 30, 19, 30, 30, - /* 20 */ 30, 30, 57, 19, 30, 30, 30, 19, 30, 30, - /* 30 */ 30, 30, 57, 19, 30, 30, 30, 50, 30, 30, - /* 40 */ 30, 30, 55, 50, 30, 30, 39, 222, 155, 156, - /* 50 */ 23, 196, 12, 11, 10, 9, 25, 223, 30, 58, - /* 60 */ 30, 30, 30, 30, 57, 58, 30, 30, 30, 59, - /* 70 */ 30, 30, 30, 30, 57, 59, 30, 30, 30, 37, - /* 80 */ 30, 30, 30, 30, 57, 37, 30, 30, 204, 38, - /* 90 */ 5, 183, 60, 166, 158, 159, 121, 53, 200, 201, - /* 100 */ 202, 203, 54, 203, 203, 203, 246, 18, 246, 246, - /* 110 */ 246, 246, 57, 120, 246, 246, 245, 17, 245, 245, - /* 120 */ 245, 245, 57, 46, 245, 245, 35, 27, 35, 35, - /* 130 */ 35, 35, 57, 31, 35, 35, 36, 47, 36, 36, - /* 140 */ 36, 36, 57, 128, 36, 36, 242, 117, 242, 242, - /* 150 */ 242, 242, 57, 42, 241, 242, 241, 241, 241, 241, - /* 160 */ 57, 52, 212, 241, 14, 13, 122, 1, 212, 212, - /* 170 */ 38, 5, 14, 13, 166, 40, 224, 20, 38, 5, - /* 180 */ 15, 210, 166, 211, 21, 22, 17, 210, 210, 43, - /* 190 */ 160, 6, 7, 45, 41, 26, 207, 48, 24, 208, - /* 200 */ 136, 44, 205, 8, 189, 3, 142, 2, 16, 127, - /* 210 */ 147, 146, 32, 150, 49, 149, 4, 33, 180, 206, - /* 220 */ 28, 117, 34, 186, 56, 162, 161, 185, 61, + /* 0 */ 392, 390, 94, 494, 20, 67, 393, 390, 27, 25, + /* 10 */ 23, 22, 21, 400, 390, 321, 52, 131, 414, 144, + /* 20 */ 492, 74, 46, 401, 266, 403, 19, 89, 98, 119, + /* 30 */ 284, 285, 286, 287, 288, 289, 290, 292, 293, 294, + /* 40 */ 27, 25, 23, 22, 21, 23, 22, 21, 346, 465, + /* 50 */ 463, 247, 248, 249, 250, 145, 253, 320, 19, 89, + /* 60 */ 98, 132, 284, 285, 286, 287, 288, 289, 290, 292, + /* 70 */ 293, 294, 26, 24, 110, 344, 345, 347, 348, 247, + /* 80 */ 248, 249, 250, 145, 253, 260, 103, 6, 400, 390, + /* 90 */ 494, 8, 143, 414, 144, 3, 317, 41, 401, 111, + /* 100 */ 403, 439, 36, 53, 70, 96, 435, 492, 400, 390, + /* 110 */ 143, 414, 143, 414, 144, 469, 114, 41, 401, 128, + /* 120 */ 403, 439, 450, 148, 54, 96, 435, 450, 143, 414, + /* 130 */ 494, 34, 63, 36, 118, 490, 400, 390, 137, 448, + /* 140 */ 143, 414, 144, 493, 447, 41, 401, 492, 403, 439, + /* 150 */ 26, 24, 322, 96, 435, 54, 17, 247, 248, 249, + /* 160 */ 250, 145, 253, 454, 103, 28, 291, 400, 390, 295, + /* 170 */ 136, 143, 414, 144, 5, 4, 41, 401, 329, 403, + /* 180 */ 439, 138, 400, 390, 438, 435, 143, 414, 144, 415, + /* 190 */ 258, 41, 401, 76, 403, 439, 120, 115, 113, 123, + /* 200 */ 435, 400, 390, 377, 378, 143, 414, 144, 78, 34, + /* 210 */ 40, 401, 42, 403, 439, 343, 400, 390, 88, 435, + /* 220 */ 131, 414, 144, 26, 24, 46, 401, 7, 403, 134, + /* 230 */ 247, 248, 249, 250, 145, 253, 112, 103, 6, 476, + /* 240 */ 27, 25, 23, 22, 21, 109, 72, 26, 24, 130, + /* 250 */ 31, 140, 104, 462, 247, 248, 249, 250, 145, 253, + /* 260 */ 450, 103, 28, 400, 390, 55, 253, 143, 414, 144, + /* 270 */ 107, 57, 41, 401, 60, 403, 439, 446, 124, 108, + /* 280 */ 281, 436, 34, 400, 390, 475, 95, 143, 414, 144, + /* 290 */ 5, 4, 47, 401, 34, 403, 141, 59, 400, 390, + /* 300 */ 374, 375, 143, 414, 144, 2, 34, 86, 401, 105, + /* 310 */ 403, 400, 390, 302, 456, 143, 414, 144, 16, 122, + /* 320 */ 86, 401, 121, 403, 27, 25, 23, 22, 21, 133, + /* 330 */ 508, 400, 390, 62, 106, 143, 414, 144, 49, 1, + /* 340 */ 86, 401, 97, 403, 400, 390, 64, 317, 143, 414, + /* 350 */ 144, 13, 29, 47, 401, 296, 403, 400, 390, 421, + /* 360 */ 257, 143, 414, 144, 44, 260, 86, 401, 102, 403, + /* 370 */ 451, 400, 390, 30, 65, 143, 414, 144, 261, 29, + /* 380 */ 83, 401, 264, 403, 400, 390, 510, 466, 143, 414, + /* 390 */ 144, 509, 139, 80, 401, 99, 403, 400, 390, 135, + /* 400 */ 142, 143, 414, 144, 258, 77, 84, 401, 397, 403, + /* 410 */ 395, 400, 390, 75, 491, 143, 414, 144, 10, 29, + /* 420 */ 81, 401, 11, 403, 400, 390, 35, 56, 143, 414, + /* 430 */ 144, 340, 58, 85, 401, 342, 403, 400, 390, 336, + /* 440 */ 48, 143, 414, 144, 61, 38, 411, 401, 335, 403, + /* 450 */ 116, 400, 390, 117, 39, 143, 414, 144, 395, 12, + /* 460 */ 410, 401, 4, 403, 400, 390, 282, 315, 143, 414, + /* 470 */ 144, 314, 32, 409, 401, 69, 403, 400, 390, 33, + /* 480 */ 394, 143, 414, 144, 51, 368, 92, 401, 14, 403, + /* 490 */ 9, 400, 390, 37, 357, 143, 414, 144, 363, 79, + /* 500 */ 91, 401, 362, 403, 400, 390, 100, 367, 143, 414, + /* 510 */ 144, 366, 101, 93, 401, 251, 403, 400, 390, 384, + /* 520 */ 383, 143, 414, 144, 15, 147, 90, 401, 382, 403, + /* 530 */ 382, 400, 390, 382, 382, 143, 414, 144, 382, 382, + /* 540 */ 82, 401, 382, 403, 400, 390, 382, 382, 143, 414, + /* 550 */ 144, 382, 382, 87, 401, 382, 403, 127, 45, 382, + /* 560 */ 382, 382, 127, 45, 382, 382, 43, 382, 382, 382, + /* 570 */ 382, 43, 382, 382, 129, 66, 444, 445, 382, 444, + /* 580 */ 68, 444, 126, 382, 125, 382, 382, 127, 45, 382, + /* 590 */ 382, 382, 127, 45, 382, 382, 43, 382, 382, 382, + /* 600 */ 382, 43, 382, 381, 146, 50, 444, 445, 382, 444, + /* 610 */ 71, 444, 445, 382, 444, 27, 25, 23, 22, 21, + /* 620 */ 27, 25, 23, 22, 21, 382, 382, 382, 382, 261, + /* 630 */ 382, 382, 18, 494, 382, 382, 266, 382, 27, 25, + /* 640 */ 23, 22, 21, 382, 382, 382, 53, 382, 73, 382, + /* 650 */ 492, 27, 25, 23, 22, 21, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, - /* 10 */ 8, 9, 49, 10, 51, 52, 37, 38, 39, 40, - /* 20 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, - /* 30 */ 41, 42, 43, 44, 45, 46, 37, 38, 39, 40, - /* 40 */ 41, 42, 43, 44, 45, 46, 67, 68, 30, 31, - /* 50 */ 69, 52, 6, 7, 8, 9, 63, 68, 37, 38, - /* 60 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, - /* 70 */ 39, 40, 41, 42, 43, 44, 45, 46, 37, 38, - /* 80 */ 39, 40, 41, 42, 43, 44, 45, 46, 43, 12, - /* 90 */ 13, 35, 36, 16, 33, 34, 2, 12, 53, 54, - /* 100 */ 55, 56, 57, 58, 59, 60, 37, 22, 39, 40, - /* 110 */ 41, 42, 43, 19, 45, 46, 37, 61, 39, 40, - /* 120 */ 41, 42, 43, 15, 45, 46, 37, 63, 39, 40, - /* 130 */ 41, 42, 43, 32, 45, 46, 37, 29, 39, 40, - /* 140 */ 41, 42, 43, 8, 45, 46, 37, 12, 39, 40, - /* 150 */ 41, 42, 43, 1, 37, 46, 39, 40, 41, 42, - /* 160 */ 43, 36, 47, 46, 6, 7, 8, 48, 47, 47, - /* 170 */ 12, 13, 6, 7, 16, 23, 70, 62, 12, 13, - /* 180 */ 13, 66, 16, 62, 62, 18, 61, 66, 66, 15, - /* 190 */ 14, 15, 13, 25, 64, 64, 17, 28, 65, 65, - /* 200 */ 12, 27, 43, 21, 50, 15, 14, 24, 2, 12, - /* 210 */ 26, 26, 26, 26, 20, 26, 15, 26, 14, 17, - /* 220 */ 17, 12, 17, 0, 17, 14, 14, 0, 11, 71, - /* 230 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 240 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 250 */ 71, 71, 71, 71, 71, 71, 71, 71, 71, 71, - /* 260 */ 71, 71, 71, 71, + /* 0 */ 75, 76, 77, 103, 89, 90, 75, 76, 8, 9, + /* 10 */ 10, 11, 12, 75, 76, 4, 116, 79, 80, 81, + /* 20 */ 120, 123, 84, 85, 24, 87, 26, 27, 28, 22, + /* 30 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + /* 40 */ 8, 9, 10, 11, 12, 10, 11, 12, 29, 83, + /* 50 */ 112, 15, 16, 17, 18, 19, 20, 46, 26, 27, + /* 60 */ 28, 74, 30, 31, 32, 33, 34, 35, 36, 37, + /* 70 */ 38, 39, 8, 9, 55, 56, 57, 58, 59, 15, + /* 80 */ 16, 17, 18, 19, 20, 22, 22, 23, 75, 76, + /* 90 */ 103, 27, 79, 80, 81, 43, 44, 84, 85, 115, + /* 100 */ 87, 88, 23, 116, 41, 92, 93, 120, 75, 76, + /* 110 */ 79, 80, 79, 80, 81, 102, 85, 84, 85, 101, + /* 120 */ 87, 88, 82, 13, 45, 92, 93, 82, 79, 80, + /* 130 */ 103, 67, 108, 23, 85, 102, 75, 76, 21, 99, + /* 140 */ 79, 80, 81, 116, 99, 84, 85, 120, 87, 88, + /* 150 */ 8, 9, 10, 92, 93, 45, 26, 15, 16, 17, + /* 160 */ 18, 19, 20, 102, 22, 23, 36, 75, 76, 39, + /* 170 */ 3, 79, 80, 81, 1, 2, 84, 85, 10, 87, + /* 180 */ 88, 64, 75, 76, 92, 93, 79, 80, 81, 80, + /* 190 */ 22, 84, 85, 117, 87, 88, 50, 51, 52, 92, + /* 200 */ 93, 75, 76, 71, 72, 79, 80, 81, 117, 67, + /* 210 */ 84, 85, 21, 87, 88, 24, 75, 76, 92, 93, + /* 220 */ 79, 80, 81, 8, 9, 84, 85, 104, 87, 62, + /* 230 */ 15, 16, 17, 18, 19, 20, 54, 22, 23, 114, + /* 240 */ 8, 9, 10, 11, 12, 53, 105, 8, 9, 22, + /* 250 */ 23, 21, 111, 112, 15, 16, 17, 18, 19, 20, + /* 260 */ 82, 22, 23, 75, 76, 113, 20, 79, 80, 81, + /* 270 */ 76, 21, 84, 85, 24, 87, 88, 99, 27, 76, + /* 280 */ 29, 93, 67, 75, 76, 114, 76, 79, 80, 81, + /* 290 */ 1, 2, 84, 85, 67, 87, 66, 113, 75, 76, + /* 300 */ 68, 69, 79, 80, 81, 61, 67, 84, 85, 86, + /* 310 */ 87, 75, 76, 24, 110, 79, 80, 81, 2, 60, + /* 320 */ 84, 85, 86, 87, 8, 9, 10, 11, 12, 121, + /* 330 */ 122, 75, 76, 109, 48, 79, 80, 81, 107, 47, + /* 340 */ 84, 85, 86, 87, 75, 76, 106, 44, 79, 80, + /* 350 */ 81, 23, 21, 84, 85, 24, 87, 75, 76, 91, + /* 360 */ 22, 79, 80, 81, 79, 22, 84, 85, 86, 87, + /* 370 */ 82, 75, 76, 40, 94, 79, 80, 81, 22, 21, + /* 380 */ 84, 85, 24, 87, 75, 76, 124, 83, 79, 80, + /* 390 */ 81, 122, 63, 84, 85, 70, 87, 75, 76, 118, + /* 400 */ 65, 79, 80, 81, 22, 118, 84, 85, 23, 87, + /* 410 */ 25, 75, 76, 119, 119, 79, 80, 81, 21, 21, + /* 420 */ 84, 85, 49, 87, 75, 76, 21, 24, 79, 80, + /* 430 */ 81, 24, 23, 84, 85, 24, 87, 75, 76, 24, + /* 440 */ 23, 79, 80, 81, 23, 23, 84, 85, 24, 87, + /* 450 */ 15, 75, 76, 21, 23, 79, 80, 81, 25, 49, + /* 460 */ 84, 85, 2, 87, 75, 76, 29, 24, 79, 80, + /* 470 */ 81, 24, 42, 84, 85, 25, 87, 75, 76, 21, + /* 480 */ 25, 79, 80, 81, 25, 24, 84, 85, 21, 87, + /* 490 */ 49, 75, 76, 4, 24, 79, 80, 81, 15, 25, + /* 500 */ 84, 85, 15, 87, 75, 76, 15, 15, 79, 80, + /* 510 */ 81, 15, 15, 84, 85, 17, 87, 75, 76, 0, + /* 520 */ 0, 79, 80, 81, 23, 14, 84, 85, 125, 87, + /* 530 */ 125, 75, 76, 125, 125, 79, 80, 81, 125, 125, + /* 540 */ 84, 85, 125, 87, 75, 76, 125, 125, 79, 80, + /* 550 */ 81, 125, 125, 84, 85, 125, 87, 78, 79, 125, + /* 560 */ 125, 125, 78, 79, 125, 125, 87, 125, 125, 125, + /* 570 */ 125, 87, 125, 125, 95, 96, 97, 98, 125, 100, + /* 580 */ 96, 97, 98, 125, 100, 125, 125, 78, 79, 125, + /* 590 */ 125, 125, 78, 79, 125, 125, 87, 125, 125, 125, + /* 600 */ 125, 87, 125, 73, 74, 96, 97, 98, 125, 100, + /* 610 */ 96, 97, 98, 125, 100, 8, 9, 10, 11, 12, + /* 620 */ 8, 9, 10, 11, 12, 125, 125, 125, 125, 22, + /* 630 */ 125, 125, 2, 103, 125, 125, 24, 125, 8, 9, + /* 640 */ 10, 11, 12, 125, 125, 125, 116, 125, 41, 125, + /* 650 */ 120, 8, 9, 10, 11, 12, 125, 125, 125, 125, + /* 660 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 670 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 680 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 690 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 700 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 710 */ 125, 125, 125, 125, 125, 125, 125, 125, 125, 125, + /* 720 */ 125, 125, 125, 125, 125, }; -#define YY_SHIFT_COUNT (62) +#define YY_SHIFT_COUNT (148) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (227) +#define YY_SHIFT_MAX (643) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 3, 158, 166, 166, 166, 166, 166, 166, 85, 166, - /* 10 */ 166, 166, 166, 77, 77, 167, 167, 167, 229, 18, - /* 20 */ 152, 152, 94, 101, 168, 169, 169, 168, 188, 182, - /* 30 */ 46, 61, 174, 108, 135, 2, 2, 176, 179, 190, - /* 40 */ 183, 192, 206, 184, 185, 186, 187, 189, 191, 197, - /* 50 */ 194, 201, 204, 202, 203, 205, 209, 207, 211, 212, - /* 60 */ 223, 227, 217, + /* 0 */ 110, 64, 64, 64, 64, 64, 64, 142, 215, 239, + /* 10 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 20 */ 239, 239, 239, 239, 239, 239, 239, 239, 239, 239, + /* 30 */ 227, 227, 227, 227, 79, 36, 79, 79, 7, 7, + /* 40 */ 0, 32, 36, 63, 63, 63, 607, 232, 19, 146, + /* 50 */ 52, 168, 167, 167, 11, 182, 192, 246, 246, 182, + /* 60 */ 192, 246, 244, 259, 286, 292, 303, 328, 303, 338, + /* 70 */ 343, 303, 333, 356, 325, 329, 335, 335, 329, 382, + /* 80 */ 316, 630, 612, 643, 643, 643, 643, 643, 289, 130, + /* 90 */ 35, 35, 35, 35, 191, 250, 173, 331, 251, 132, + /* 100 */ 117, 230, 358, 385, 397, 398, 373, 403, 407, 409, + /* 110 */ 405, 411, 417, 421, 415, 422, 424, 435, 432, 433, + /* 120 */ 431, 398, 410, 460, 437, 443, 447, 450, 430, 458, + /* 130 */ 455, 459, 461, 467, 441, 470, 489, 483, 487, 491, + /* 140 */ 492, 496, 497, 474, 501, 498, 519, 520, 511, }; -#define YY_REDUCE_COUNT (29) -#define YY_REDUCE_MIN (-37) -#define YY_REDUCE_MAX (159) +#define YY_REDUCE_COUNT (79) +#define YY_REDUCE_MIN (-102) +#define YY_REDUCE_MAX (530) static const short yy_reduce_ofst[] = { - /* 0 */ 56, -37, -21, -11, -1, 21, 31, 41, 45, 69, - /* 10 */ 79, 89, 99, 109, 117, 115, 121, 122, 125, -19, - /* 20 */ -7, 64, 119, 106, 130, 133, 134, 131, 159, 154, + /* 0 */ 530, 13, 33, 61, 92, 107, 126, 141, 188, 208, + /* 10 */ -62, 223, 236, 256, 269, 282, 296, 309, 322, 336, + /* 20 */ 349, 362, 376, 389, 402, 416, 429, 442, 456, 469, + /* 30 */ 479, 484, 509, 514, -13, -75, -100, 27, 31, 49, + /* 40 */ -85, -85, -69, 40, 45, 178, -34, -102, -16, 24, + /* 50 */ 18, 109, 76, 91, 123, 125, 152, 194, 203, 171, + /* 60 */ 184, 210, 204, 224, 231, 240, 18, 268, 18, 285, + /* 70 */ 288, 18, 280, 304, 262, 281, 294, 295, 287, 109, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 209, 182, 182, 182, 182, 182, 182, 182, 182, 182, - /* 10 */ 182, 182, 182, 182, 182, 182, 182, 182, 209, 225, - /* 20 */ 214, 214, 190, 228, 216, 219, 219, 216, 182, 182, - /* 30 */ 239, 182, 182, 182, 182, 244, 243, 182, 187, 215, - /* 40 */ 182, 182, 182, 182, 182, 182, 182, 182, 182, 182, - /* 50 */ 197, 194, 182, 207, 182, 182, 182, 182, 182, 182, - /* 60 */ 182, 182, 182, + /* 0 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 10 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 20 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 30 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 380, + /* 40 */ 380, 380, 380, 449, 449, 449, 464, 511, 380, 472, + /* 50 */ 380, 380, 496, 496, 457, 479, 477, 380, 380, 479, + /* 60 */ 477, 380, 489, 487, 470, 468, 442, 380, 380, 380, + /* 70 */ 380, 443, 380, 380, 514, 498, 502, 502, 498, 380, + /* 80 */ 380, 380, 380, 418, 417, 416, 412, 413, 380, 380, + /* 90 */ 407, 408, 406, 405, 380, 380, 507, 380, 380, 380, + /* 100 */ 499, 503, 380, 396, 461, 471, 380, 380, 380, 380, + /* 110 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, 396, + /* 120 */ 380, 488, 380, 437, 380, 517, 445, 380, 380, 441, + /* 130 */ 395, 380, 380, 497, 380, 380, 380, 380, 380, 380, + /* 140 */ 380, 380, 380, 380, 380, 380, 380, 380, 380, }; /********** End of lemon-generated parsing tables *****************************/ @@ -394,76 +515,130 @@ void NewParseTrace(FILE *TraceFILE, char *zTracePrompt){ ** are required. The following table supplies these names */ static const char *const yyTokenName[] = { /* 0 */ "$", - /* 1 */ "UNION", - /* 2 */ "ALL", - /* 3 */ "MINUS", - /* 4 */ "EXCEPT", - /* 5 */ "INTERSECT", - /* 6 */ "NK_PLUS", - /* 7 */ "NK_MINUS", - /* 8 */ "NK_STAR", - /* 9 */ "NK_SLASH", - /* 10 */ "SHOW", - /* 11 */ "DATABASES", - /* 12 */ "NK_ID", - /* 13 */ "NK_LP", - /* 14 */ "NK_RP", - /* 15 */ "NK_COMMA", - /* 16 */ "NK_LITERAL", - /* 17 */ "NK_DOT", - /* 18 */ "SELECT", - /* 19 */ "DISTINCT", - /* 20 */ "AS", - /* 21 */ "FROM", - /* 22 */ "NK_LR", - /* 23 */ "ORDER", - /* 24 */ "BY", - /* 25 */ "SLIMIT", - /* 26 */ "NK_INTEGER", - /* 27 */ "SOFFSET", - /* 28 */ "LIMIT", - /* 29 */ "OFFSET", - /* 30 */ "ASC", - /* 31 */ "DESC", - /* 32 */ "NULLS", - /* 33 */ "FIRST", - /* 34 */ "LAST", - /* 35 */ "cmd", - /* 36 */ "query_expression", - /* 37 */ "value_function", - /* 38 */ "value_expression", - /* 39 */ "value_expression_primary", - /* 40 */ "nonparenthesized_value_expression_primary", - /* 41 */ "literal", - /* 42 */ "column_reference", - /* 43 */ "table_name", - /* 44 */ "common_value_expression", - /* 45 */ "numeric_value_expression", - /* 46 */ "numeric_primary", - /* 47 */ "query_specification", - /* 48 */ "set_quantifier_opt", - /* 49 */ "select_list", - /* 50 */ "from_clause", - /* 51 */ "select_sublist", - /* 52 */ "select_item", - /* 53 */ "table_reference_list", - /* 54 */ "table_reference", - /* 55 */ "table_factor", - /* 56 */ "table_primary", - /* 57 */ "db_name", - /* 58 */ "derived_table", - /* 59 */ "table_subquery", - /* 60 */ "subquery", - /* 61 */ "with_clause_opt", - /* 62 */ "query_expression_body", - /* 63 */ "order_by_clause_opt", - /* 64 */ "slimit_clause_opt", - /* 65 */ "limit_clause_opt", - /* 66 */ "query_primary", - /* 67 */ "sort_specification_list", - /* 68 */ "sort_specification", - /* 69 */ "ordering_specification_opt", - /* 70 */ "null_ordering_opt", + /* 1 */ "OR", + /* 2 */ "AND", + /* 3 */ "UNION", + /* 4 */ "ALL", + /* 5 */ "MINUS", + /* 6 */ "EXCEPT", + /* 7 */ "INTERSECT", + /* 8 */ "NK_PLUS", + /* 9 */ "NK_MINUS", + /* 10 */ "NK_STAR", + /* 11 */ "NK_SLASH", + /* 12 */ "NK_REM", + /* 13 */ "SHOW", + /* 14 */ "DATABASES", + /* 15 */ "NK_INTEGER", + /* 16 */ "NK_FLOAT", + /* 17 */ "NK_STRING", + /* 18 */ "NK_BOOL", + /* 19 */ "TIMESTAMP", + /* 20 */ "NK_VARIABLE", + /* 21 */ "NK_COMMA", + /* 22 */ "NK_ID", + /* 23 */ "NK_LP", + /* 24 */ "NK_RP", + /* 25 */ "NK_DOT", + /* 26 */ "BETWEEN", + /* 27 */ "NOT", + /* 28 */ "IS", + /* 29 */ "NULL", + /* 30 */ "NK_LT", + /* 31 */ "NK_GT", + /* 32 */ "NK_LE", + /* 33 */ "NK_GE", + /* 34 */ "NK_NE", + /* 35 */ "NK_EQ", + /* 36 */ "LIKE", + /* 37 */ "MATCH", + /* 38 */ "NMATCH", + /* 39 */ "IN", + /* 40 */ "FROM", + /* 41 */ "AS", + /* 42 */ "JOIN", + /* 43 */ "ON", + /* 44 */ "INNER", + /* 45 */ "SELECT", + /* 46 */ "DISTINCT", + /* 47 */ "WHERE", + /* 48 */ "PARTITION", + /* 49 */ "BY", + /* 50 */ "SESSION", + /* 51 */ "STATE_WINDOW", + /* 52 */ "INTERVAL", + /* 53 */ "SLIDING", + /* 54 */ "FILL", + /* 55 */ "VALUE", + /* 56 */ "NONE", + /* 57 */ "PREV", + /* 58 */ "LINEAR", + /* 59 */ "NEXT", + /* 60 */ "GROUP", + /* 61 */ "HAVING", + /* 62 */ "ORDER", + /* 63 */ "SLIMIT", + /* 64 */ "SOFFSET", + /* 65 */ "LIMIT", + /* 66 */ "OFFSET", + /* 67 */ "NK_LR", + /* 68 */ "ASC", + /* 69 */ "DESC", + /* 70 */ "NULLS", + /* 71 */ "FIRST", + /* 72 */ "LAST", + /* 73 */ "cmd", + /* 74 */ "query_expression", + /* 75 */ "literal", + /* 76 */ "duration_literal", + /* 77 */ "literal_list", + /* 78 */ "db_name", + /* 79 */ "table_name", + /* 80 */ "column_name", + /* 81 */ "function_name", + /* 82 */ "table_alias", + /* 83 */ "column_alias", + /* 84 */ "expression", + /* 85 */ "column_reference", + /* 86 */ "expression_list", + /* 87 */ "subquery", + /* 88 */ "predicate", + /* 89 */ "compare_op", + /* 90 */ "in_op", + /* 91 */ "in_predicate_value", + /* 92 */ "boolean_value_expression", + /* 93 */ "boolean_primary", + /* 94 */ "from_clause", + /* 95 */ "table_reference_list", + /* 96 */ "table_reference", + /* 97 */ "table_primary", + /* 98 */ "joined_table", + /* 99 */ "alias_opt", + /* 100 */ "parenthesized_joined_table", + /* 101 */ "join_type", + /* 102 */ "search_condition", + /* 103 */ "query_specification", + /* 104 */ "set_quantifier_opt", + /* 105 */ "select_list", + /* 106 */ "where_clause_opt", + /* 107 */ "partition_by_clause_opt", + /* 108 */ "twindow_clause_opt", + /* 109 */ "group_by_clause_opt", + /* 110 */ "having_clause_opt", + /* 111 */ "select_sublist", + /* 112 */ "select_item", + /* 113 */ "sliding_opt", + /* 114 */ "fill_opt", + /* 115 */ "fill_mode", + /* 116 */ "query_expression_body", + /* 117 */ "order_by_clause_opt", + /* 118 */ "slimit_clause_opt", + /* 119 */ "limit_clause_opt", + /* 120 */ "query_primary", + /* 121 */ "sort_specification_list", + /* 122 */ "sort_specification", + /* 123 */ "ordering_specification_opt", + /* 124 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -473,72 +648,139 @@ static const char *const yyTokenName[] = { static const char *const yyRuleName[] = { /* 0 */ "cmd ::= SHOW DATABASES", /* 1 */ "cmd ::= query_expression", - /* 2 */ "column_reference ::= NK_ID", - /* 3 */ "column_reference ::= table_name NK_DOT NK_ID", - /* 4 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause", - /* 5 */ "set_quantifier_opt ::=", - /* 6 */ "set_quantifier_opt ::= DISTINCT", - /* 7 */ "set_quantifier_opt ::= ALL", - /* 8 */ "select_list ::= NK_STAR", - /* 9 */ "select_list ::= select_sublist", - /* 10 */ "select_sublist ::= select_item", - /* 11 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 12 */ "select_item ::= value_expression", - /* 13 */ "select_item ::= value_expression AS NK_ID", - /* 14 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 15 */ "from_clause ::= FROM table_reference_list", - /* 16 */ "table_reference_list ::= table_reference", - /* 17 */ "table_reference ::= table_factor", - /* 18 */ "table_factor ::= table_primary", - /* 19 */ "table_primary ::= table_name", - /* 20 */ "table_primary ::= db_name NK_DOT table_name", - /* 21 */ "db_name ::= NK_ID", - /* 22 */ "table_name ::= NK_ID", - /* 23 */ "query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 24 */ "with_clause_opt ::=", - /* 25 */ "query_expression_body ::= query_primary", - /* 26 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 27 */ "query_primary ::= query_specification", - /* 28 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", - /* 29 */ "order_by_clause_opt ::=", - /* 30 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 31 */ "slimit_clause_opt ::=", - /* 32 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 33 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 34 */ "limit_clause_opt ::=", - /* 35 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 36 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 37 */ "sort_specification_list ::= sort_specification", - /* 38 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 39 */ "sort_specification ::= value_expression ordering_specification_opt null_ordering_opt", - /* 40 */ "ordering_specification_opt ::=", - /* 41 */ "ordering_specification_opt ::= ASC", - /* 42 */ "ordering_specification_opt ::= DESC", - /* 43 */ "null_ordering_opt ::=", - /* 44 */ "null_ordering_opt ::= NULLS FIRST", - /* 45 */ "null_ordering_opt ::= NULLS LAST", - /* 46 */ "value_function ::= NK_ID NK_LP value_expression NK_RP", - /* 47 */ "value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP", - /* 48 */ "value_expression_primary ::= NK_LP value_expression NK_RP", - /* 49 */ "value_expression_primary ::= nonparenthesized_value_expression_primary", - /* 50 */ "nonparenthesized_value_expression_primary ::= literal", - /* 51 */ "nonparenthesized_value_expression_primary ::= column_reference", - /* 52 */ "literal ::= NK_LITERAL", - /* 53 */ "value_expression ::= common_value_expression", - /* 54 */ "common_value_expression ::= numeric_value_expression", - /* 55 */ "numeric_value_expression ::= numeric_primary", - /* 56 */ "numeric_value_expression ::= NK_PLUS numeric_primary", - /* 57 */ "numeric_value_expression ::= NK_MINUS numeric_primary", - /* 58 */ "numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression", - /* 59 */ "numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression", - /* 60 */ "numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression", - /* 61 */ "numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression", - /* 62 */ "numeric_primary ::= value_expression_primary", - /* 63 */ "numeric_primary ::= value_function", - /* 64 */ "table_primary ::= derived_table", - /* 65 */ "derived_table ::= table_subquery", - /* 66 */ "subquery ::= NK_LR query_expression NK_RP", - /* 67 */ "table_subquery ::= subquery", + /* 2 */ "literal ::= NK_INTEGER", + /* 3 */ "literal ::= NK_FLOAT", + /* 4 */ "literal ::= NK_STRING", + /* 5 */ "literal ::= NK_BOOL", + /* 6 */ "literal ::= TIMESTAMP NK_STRING", + /* 7 */ "literal ::= duration_literal", + /* 8 */ "duration_literal ::= NK_VARIABLE", + /* 9 */ "literal_list ::= literal", + /* 10 */ "literal_list ::= literal_list NK_COMMA literal", + /* 11 */ "db_name ::= NK_ID", + /* 12 */ "table_name ::= NK_ID", + /* 13 */ "column_name ::= NK_ID", + /* 14 */ "function_name ::= NK_ID", + /* 15 */ "table_alias ::= NK_ID", + /* 16 */ "column_alias ::= NK_ID", + /* 17 */ "expression ::= literal", + /* 18 */ "expression ::= column_reference", + /* 19 */ "expression ::= function_name NK_LP expression_list NK_RP", + /* 20 */ "expression ::= subquery", + /* 21 */ "expression ::= NK_LP expression NK_RP", + /* 22 */ "expression ::= NK_PLUS expression", + /* 23 */ "expression ::= NK_MINUS expression", + /* 24 */ "expression ::= expression NK_PLUS expression", + /* 25 */ "expression ::= expression NK_MINUS expression", + /* 26 */ "expression ::= expression NK_STAR expression", + /* 27 */ "expression ::= expression NK_SLASH expression", + /* 28 */ "expression ::= expression NK_REM expression", + /* 29 */ "expression_list ::= expression", + /* 30 */ "expression_list ::= expression_list NK_COMMA expression", + /* 31 */ "column_reference ::= column_name", + /* 32 */ "column_reference ::= table_name NK_DOT column_name", + /* 33 */ "predicate ::= expression compare_op expression", + /* 34 */ "predicate ::= expression BETWEEN expression AND expression", + /* 35 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 36 */ "predicate ::= expression IS NULL", + /* 37 */ "predicate ::= expression IS NOT NULL", + /* 38 */ "predicate ::= expression in_op in_predicate_value", + /* 39 */ "compare_op ::= NK_LT", + /* 40 */ "compare_op ::= NK_GT", + /* 41 */ "compare_op ::= NK_LE", + /* 42 */ "compare_op ::= NK_GE", + /* 43 */ "compare_op ::= NK_NE", + /* 44 */ "compare_op ::= NK_EQ", + /* 45 */ "compare_op ::= LIKE", + /* 46 */ "compare_op ::= NOT LIKE", + /* 47 */ "compare_op ::= MATCH", + /* 48 */ "compare_op ::= NMATCH", + /* 49 */ "in_op ::= IN", + /* 50 */ "in_op ::= NOT IN", + /* 51 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 52 */ "boolean_value_expression ::= boolean_primary", + /* 53 */ "boolean_value_expression ::= NOT boolean_primary", + /* 54 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 55 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 56 */ "boolean_primary ::= predicate", + /* 57 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 58 */ "from_clause ::= FROM table_reference_list", + /* 59 */ "table_reference_list ::= table_reference", + /* 60 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 61 */ "table_reference ::= table_primary", + /* 62 */ "table_reference ::= joined_table", + /* 63 */ "table_primary ::= table_name alias_opt", + /* 64 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 65 */ "table_primary ::= subquery alias_opt", + /* 66 */ "alias_opt ::=", + /* 67 */ "alias_opt ::= table_alias", + /* 68 */ "alias_opt ::= AS table_alias", + /* 69 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 70 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 71 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 72 */ "join_type ::= INNER", + /* 73 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 74 */ "set_quantifier_opt ::=", + /* 75 */ "set_quantifier_opt ::= DISTINCT", + /* 76 */ "set_quantifier_opt ::= ALL", + /* 77 */ "select_list ::= NK_STAR", + /* 78 */ "select_list ::= select_sublist", + /* 79 */ "select_sublist ::= select_item", + /* 80 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 81 */ "select_item ::= expression", + /* 82 */ "select_item ::= expression column_alias", + /* 83 */ "select_item ::= expression AS column_alias", + /* 84 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 85 */ "where_clause_opt ::=", + /* 86 */ "where_clause_opt ::= WHERE search_condition", + /* 87 */ "partition_by_clause_opt ::=", + /* 88 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 89 */ "twindow_clause_opt ::=", + /* 90 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 91 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 92 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 93 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 94 */ "sliding_opt ::=", + /* 95 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 96 */ "fill_opt ::=", + /* 97 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 98 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 99 */ "fill_mode ::= NONE", + /* 100 */ "fill_mode ::= PREV", + /* 101 */ "fill_mode ::= NULL", + /* 102 */ "fill_mode ::= LINEAR", + /* 103 */ "fill_mode ::= NEXT", + /* 104 */ "group_by_clause_opt ::=", + /* 105 */ "group_by_clause_opt ::= GROUP BY expression_list", + /* 106 */ "having_clause_opt ::=", + /* 107 */ "having_clause_opt ::= HAVING search_condition", + /* 108 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 109 */ "query_expression_body ::= query_primary", + /* 110 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 111 */ "query_primary ::= query_specification", + /* 112 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP", + /* 113 */ "order_by_clause_opt ::=", + /* 114 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 115 */ "slimit_clause_opt ::=", + /* 116 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 117 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 118 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 119 */ "limit_clause_opt ::=", + /* 120 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 121 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 122 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 123 */ "subquery ::= NK_LR query_expression NK_RP", + /* 124 */ "search_condition ::= boolean_value_expression", + /* 125 */ "sort_specification_list ::= sort_specification", + /* 126 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 127 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 128 */ "ordering_specification_opt ::=", + /* 129 */ "ordering_specification_opt ::= ASC", + /* 130 */ "ordering_specification_opt ::= DESC", + /* 131 */ "null_ordering_opt ::=", + /* 132 */ "null_ordering_opt ::= NULLS FIRST", + /* 133 */ "null_ordering_opt ::= NULLS LAST", + /* 134 */ "table_primary ::= parenthesized_joined_table", }; #endif /* NDEBUG */ @@ -665,60 +907,92 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 35: /* cmd */ - case 36: /* query_expression */ - case 37: /* value_function */ - case 38: /* value_expression */ - case 39: /* value_expression_primary */ - case 40: /* nonparenthesized_value_expression_primary */ - case 41: /* literal */ - case 42: /* column_reference */ - case 43: /* table_name */ - case 44: /* common_value_expression */ - case 45: /* numeric_value_expression */ - case 46: /* numeric_primary */ - case 47: /* query_specification */ - case 50: /* from_clause */ - case 52: /* select_item */ - case 53: /* table_reference_list */ - case 54: /* table_reference */ - case 55: /* table_factor */ - case 56: /* table_primary */ - case 57: /* db_name */ - case 58: /* derived_table */ - case 59: /* table_subquery */ - case 60: /* subquery */ - case 61: /* with_clause_opt */ - case 62: /* query_expression_body */ - case 64: /* slimit_clause_opt */ - case 65: /* limit_clause_opt */ - case 66: /* query_primary */ - case 68: /* sort_specification */ + case 73: /* cmd */ + case 74: /* query_expression */ + case 75: /* literal */ + case 76: /* duration_literal */ + case 84: /* expression */ + case 85: /* column_reference */ + case 87: /* subquery */ + case 88: /* predicate */ + case 91: /* in_predicate_value */ + case 92: /* boolean_value_expression */ + case 93: /* boolean_primary */ + case 94: /* from_clause */ + case 95: /* table_reference_list */ + case 96: /* table_reference */ + case 97: /* table_primary */ + case 98: /* joined_table */ + case 100: /* parenthesized_joined_table */ + case 102: /* search_condition */ + case 103: /* query_specification */ + case 106: /* where_clause_opt */ + case 108: /* twindow_clause_opt */ + case 110: /* having_clause_opt */ + case 112: /* select_item */ + case 113: /* sliding_opt */ + case 114: /* fill_opt */ + case 116: /* query_expression_body */ + case 118: /* slimit_clause_opt */ + case 119: /* limit_clause_opt */ + case 120: /* query_primary */ + case 122: /* sort_specification */ { - nodesDestroyNode((yypminor->yy130)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy212)); } break; - case 48: /* set_quantifier_opt */ + case 77: /* literal_list */ + case 86: /* expression_list */ + case 105: /* select_list */ + case 107: /* partition_by_clause_opt */ + case 109: /* group_by_clause_opt */ + case 111: /* select_sublist */ + case 117: /* order_by_clause_opt */ + case 121: /* sort_specification_list */ { - + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy174)); } break; - case 49: /* select_list */ - case 51: /* select_sublist */ - case 63: /* order_by_clause_opt */ - case 67: /* sort_specification_list */ + case 78: /* db_name */ + case 79: /* table_name */ + case 80: /* column_name */ + case 81: /* function_name */ + case 82: /* table_alias */ + case 83: /* column_alias */ + case 99: /* alias_opt */ { - nodesDestroyNodeList((yypminor->yy30)); + PARSER_DESTRUCTOR_TRACE; } break; - case 69: /* ordering_specification_opt */ + case 89: /* compare_op */ + case 90: /* in_op */ { - + PARSER_DESTRUCTOR_TRACE; } break; - case 70: /* null_ordering_opt */ + case 101: /* join_type */ { - + PARSER_DESTRUCTOR_TRACE; +} + break; + case 104: /* set_quantifier_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 115: /* fill_mode */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 123: /* ordering_specification_opt */ +{ + PARSER_DESTRUCTOR_TRACE; +} + break; + case 124: /* null_ordering_opt */ +{ + PARSER_DESTRUCTOR_TRACE; } break; /********* End destructor definitions *****************************************/ @@ -1015,74 +1289,141 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 35, -2 }, /* (0) cmd ::= SHOW DATABASES */ - { 35, -1 }, /* (1) cmd ::= query_expression */ - { 42, -1 }, /* (2) column_reference ::= NK_ID */ - { 42, -3 }, /* (3) column_reference ::= table_name NK_DOT NK_ID */ - { 47, -4 }, /* (4) query_specification ::= SELECT set_quantifier_opt select_list from_clause */ - { 48, 0 }, /* (5) set_quantifier_opt ::= */ - { 48, -1 }, /* (6) set_quantifier_opt ::= DISTINCT */ - { 48, -1 }, /* (7) set_quantifier_opt ::= ALL */ - { 49, -1 }, /* (8) select_list ::= NK_STAR */ - { 49, -1 }, /* (9) select_list ::= select_sublist */ - { 51, -1 }, /* (10) select_sublist ::= select_item */ - { 51, -3 }, /* (11) select_sublist ::= select_sublist NK_COMMA select_item */ - { 52, -1 }, /* (12) select_item ::= value_expression */ - { 52, -3 }, /* (13) select_item ::= value_expression AS NK_ID */ - { 52, -3 }, /* (14) select_item ::= table_name NK_DOT NK_STAR */ - { 50, -2 }, /* (15) from_clause ::= FROM table_reference_list */ - { 53, -1 }, /* (16) table_reference_list ::= table_reference */ - { 54, -1 }, /* (17) table_reference ::= table_factor */ - { 55, -1 }, /* (18) table_factor ::= table_primary */ - { 56, -1 }, /* (19) table_primary ::= table_name */ - { 56, -3 }, /* (20) table_primary ::= db_name NK_DOT table_name */ - { 57, -1 }, /* (21) db_name ::= NK_ID */ - { 43, -1 }, /* (22) table_name ::= NK_ID */ - { 36, -5 }, /* (23) query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 61, 0 }, /* (24) with_clause_opt ::= */ - { 62, -1 }, /* (25) query_expression_body ::= query_primary */ - { 62, -4 }, /* (26) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 66, -1 }, /* (27) query_primary ::= query_specification */ - { 66, -6 }, /* (28) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ - { 63, 0 }, /* (29) order_by_clause_opt ::= */ - { 63, -3 }, /* (30) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 64, 0 }, /* (31) slimit_clause_opt ::= */ - { 64, -4 }, /* (32) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 64, -4 }, /* (33) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 65, 0 }, /* (34) limit_clause_opt ::= */ - { 65, -4 }, /* (35) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 65, -4 }, /* (36) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 67, -1 }, /* (37) sort_specification_list ::= sort_specification */ - { 67, -3 }, /* (38) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 68, -3 }, /* (39) sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ - { 69, 0 }, /* (40) ordering_specification_opt ::= */ - { 69, -1 }, /* (41) ordering_specification_opt ::= ASC */ - { 69, -1 }, /* (42) ordering_specification_opt ::= DESC */ - { 70, 0 }, /* (43) null_ordering_opt ::= */ - { 70, -2 }, /* (44) null_ordering_opt ::= NULLS FIRST */ - { 70, -2 }, /* (45) null_ordering_opt ::= NULLS LAST */ - { 37, -4 }, /* (46) value_function ::= NK_ID NK_LP value_expression NK_RP */ - { 37, -6 }, /* (47) value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ - { 39, -3 }, /* (48) value_expression_primary ::= NK_LP value_expression NK_RP */ - { 39, -1 }, /* (49) value_expression_primary ::= nonparenthesized_value_expression_primary */ - { 40, -1 }, /* (50) nonparenthesized_value_expression_primary ::= literal */ - { 40, -1 }, /* (51) nonparenthesized_value_expression_primary ::= column_reference */ - { 41, -1 }, /* (52) literal ::= NK_LITERAL */ - { 38, -1 }, /* (53) value_expression ::= common_value_expression */ - { 44, -1 }, /* (54) common_value_expression ::= numeric_value_expression */ - { 45, -1 }, /* (55) numeric_value_expression ::= numeric_primary */ - { 45, -2 }, /* (56) numeric_value_expression ::= NK_PLUS numeric_primary */ - { 45, -2 }, /* (57) numeric_value_expression ::= NK_MINUS numeric_primary */ - { 45, -3 }, /* (58) numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - { 45, -3 }, /* (59) numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ - { 45, -3 }, /* (60) numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ - { 45, -3 }, /* (61) numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ - { 46, -1 }, /* (62) numeric_primary ::= value_expression_primary */ - { 46, -1 }, /* (63) numeric_primary ::= value_function */ - { 56, -1 }, /* (64) table_primary ::= derived_table */ - { 58, -1 }, /* (65) derived_table ::= table_subquery */ - { 60, -3 }, /* (66) subquery ::= NK_LR query_expression NK_RP */ - { 59, -1 }, /* (67) table_subquery ::= subquery */ + { 73, -2 }, /* (0) cmd ::= SHOW DATABASES */ + { 73, -1 }, /* (1) cmd ::= query_expression */ + { 75, -1 }, /* (2) literal ::= NK_INTEGER */ + { 75, -1 }, /* (3) literal ::= NK_FLOAT */ + { 75, -1 }, /* (4) literal ::= NK_STRING */ + { 75, -1 }, /* (5) literal ::= NK_BOOL */ + { 75, -2 }, /* (6) literal ::= TIMESTAMP NK_STRING */ + { 75, -1 }, /* (7) literal ::= duration_literal */ + { 76, -1 }, /* (8) duration_literal ::= NK_VARIABLE */ + { 77, -1 }, /* (9) literal_list ::= literal */ + { 77, -3 }, /* (10) literal_list ::= literal_list NK_COMMA literal */ + { 78, -1 }, /* (11) db_name ::= NK_ID */ + { 79, -1 }, /* (12) table_name ::= NK_ID */ + { 80, -1 }, /* (13) column_name ::= NK_ID */ + { 81, -1 }, /* (14) function_name ::= NK_ID */ + { 82, -1 }, /* (15) table_alias ::= NK_ID */ + { 83, -1 }, /* (16) column_alias ::= NK_ID */ + { 84, -1 }, /* (17) expression ::= literal */ + { 84, -1 }, /* (18) expression ::= column_reference */ + { 84, -4 }, /* (19) expression ::= function_name NK_LP expression_list NK_RP */ + { 84, -1 }, /* (20) expression ::= subquery */ + { 84, -3 }, /* (21) expression ::= NK_LP expression NK_RP */ + { 84, -2 }, /* (22) expression ::= NK_PLUS expression */ + { 84, -2 }, /* (23) expression ::= NK_MINUS expression */ + { 84, -3 }, /* (24) expression ::= expression NK_PLUS expression */ + { 84, -3 }, /* (25) expression ::= expression NK_MINUS expression */ + { 84, -3 }, /* (26) expression ::= expression NK_STAR expression */ + { 84, -3 }, /* (27) expression ::= expression NK_SLASH expression */ + { 84, -3 }, /* (28) expression ::= expression NK_REM expression */ + { 86, -1 }, /* (29) expression_list ::= expression */ + { 86, -3 }, /* (30) expression_list ::= expression_list NK_COMMA expression */ + { 85, -1 }, /* (31) column_reference ::= column_name */ + { 85, -3 }, /* (32) column_reference ::= table_name NK_DOT column_name */ + { 88, -3 }, /* (33) predicate ::= expression compare_op expression */ + { 88, -5 }, /* (34) predicate ::= expression BETWEEN expression AND expression */ + { 88, -6 }, /* (35) predicate ::= expression NOT BETWEEN expression AND expression */ + { 88, -3 }, /* (36) predicate ::= expression IS NULL */ + { 88, -4 }, /* (37) predicate ::= expression IS NOT NULL */ + { 88, -3 }, /* (38) predicate ::= expression in_op in_predicate_value */ + { 89, -1 }, /* (39) compare_op ::= NK_LT */ + { 89, -1 }, /* (40) compare_op ::= NK_GT */ + { 89, -1 }, /* (41) compare_op ::= NK_LE */ + { 89, -1 }, /* (42) compare_op ::= NK_GE */ + { 89, -1 }, /* (43) compare_op ::= NK_NE */ + { 89, -1 }, /* (44) compare_op ::= NK_EQ */ + { 89, -1 }, /* (45) compare_op ::= LIKE */ + { 89, -2 }, /* (46) compare_op ::= NOT LIKE */ + { 89, -1 }, /* (47) compare_op ::= MATCH */ + { 89, -1 }, /* (48) compare_op ::= NMATCH */ + { 90, -1 }, /* (49) in_op ::= IN */ + { 90, -2 }, /* (50) in_op ::= NOT IN */ + { 91, -3 }, /* (51) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 92, -1 }, /* (52) boolean_value_expression ::= boolean_primary */ + { 92, -2 }, /* (53) boolean_value_expression ::= NOT boolean_primary */ + { 92, -3 }, /* (54) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 92, -3 }, /* (55) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 93, -1 }, /* (56) boolean_primary ::= predicate */ + { 93, -3 }, /* (57) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 94, -2 }, /* (58) from_clause ::= FROM table_reference_list */ + { 95, -1 }, /* (59) table_reference_list ::= table_reference */ + { 95, -3 }, /* (60) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 96, -1 }, /* (61) table_reference ::= table_primary */ + { 96, -1 }, /* (62) table_reference ::= joined_table */ + { 97, -2 }, /* (63) table_primary ::= table_name alias_opt */ + { 97, -4 }, /* (64) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 97, -2 }, /* (65) table_primary ::= subquery alias_opt */ + { 99, 0 }, /* (66) alias_opt ::= */ + { 99, -1 }, /* (67) alias_opt ::= table_alias */ + { 99, -2 }, /* (68) alias_opt ::= AS table_alias */ + { 100, -3 }, /* (69) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 100, -3 }, /* (70) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 98, -6 }, /* (71) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 101, -1 }, /* (72) join_type ::= INNER */ + { 103, -9 }, /* (73) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 104, 0 }, /* (74) set_quantifier_opt ::= */ + { 104, -1 }, /* (75) set_quantifier_opt ::= DISTINCT */ + { 104, -1 }, /* (76) set_quantifier_opt ::= ALL */ + { 105, -1 }, /* (77) select_list ::= NK_STAR */ + { 105, -1 }, /* (78) select_list ::= select_sublist */ + { 111, -1 }, /* (79) select_sublist ::= select_item */ + { 111, -3 }, /* (80) select_sublist ::= select_sublist NK_COMMA select_item */ + { 112, -1 }, /* (81) select_item ::= expression */ + { 112, -2 }, /* (82) select_item ::= expression column_alias */ + { 112, -3 }, /* (83) select_item ::= expression AS column_alias */ + { 112, -3 }, /* (84) select_item ::= table_name NK_DOT NK_STAR */ + { 106, 0 }, /* (85) where_clause_opt ::= */ + { 106, -2 }, /* (86) where_clause_opt ::= WHERE search_condition */ + { 107, 0 }, /* (87) partition_by_clause_opt ::= */ + { 107, -3 }, /* (88) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 108, 0 }, /* (89) twindow_clause_opt ::= */ + { 108, -6 }, /* (90) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 108, -4 }, /* (91) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 108, -6 }, /* (92) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 108, -8 }, /* (93) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 113, 0 }, /* (94) sliding_opt ::= */ + { 113, -4 }, /* (95) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 114, 0 }, /* (96) fill_opt ::= */ + { 114, -4 }, /* (97) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 114, -6 }, /* (98) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 115, -1 }, /* (99) fill_mode ::= NONE */ + { 115, -1 }, /* (100) fill_mode ::= PREV */ + { 115, -1 }, /* (101) fill_mode ::= NULL */ + { 115, -1 }, /* (102) fill_mode ::= LINEAR */ + { 115, -1 }, /* (103) fill_mode ::= NEXT */ + { 109, 0 }, /* (104) group_by_clause_opt ::= */ + { 109, -3 }, /* (105) group_by_clause_opt ::= GROUP BY expression_list */ + { 110, 0 }, /* (106) having_clause_opt ::= */ + { 110, -2 }, /* (107) having_clause_opt ::= HAVING search_condition */ + { 74, -4 }, /* (108) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 116, -1 }, /* (109) query_expression_body ::= query_primary */ + { 116, -4 }, /* (110) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 120, -1 }, /* (111) query_primary ::= query_specification */ + { 120, -6 }, /* (112) query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ + { 117, 0 }, /* (113) order_by_clause_opt ::= */ + { 117, -3 }, /* (114) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 118, 0 }, /* (115) slimit_clause_opt ::= */ + { 118, -2 }, /* (116) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 118, -4 }, /* (117) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 118, -4 }, /* (118) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 119, 0 }, /* (119) limit_clause_opt ::= */ + { 119, -2 }, /* (120) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 119, -4 }, /* (121) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 119, -4 }, /* (122) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 87, -3 }, /* (123) subquery ::= NK_LR query_expression NK_RP */ + { 102, -1 }, /* (124) search_condition ::= boolean_value_expression */ + { 121, -1 }, /* (125) sort_specification_list ::= sort_specification */ + { 121, -3 }, /* (126) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 122, -3 }, /* (127) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 123, 0 }, /* (128) ordering_specification_opt ::= */ + { 123, -1 }, /* (129) ordering_specification_opt ::= ASC */ + { 123, -1 }, /* (130) ordering_specification_opt ::= DESC */ + { 124, 0 }, /* (131) null_ordering_opt ::= */ + { 124, -2 }, /* (132) null_ordering_opt ::= NULLS FIRST */ + { 124, -2 }, /* (133) null_ordering_opt ::= NULLS LAST */ + { 97, -1 }, /* (134) table_primary ::= parenthesized_joined_table */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1173,240 +1514,376 @@ static YYACTIONTYPE yy_reduce( { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } break; case 1: /* cmd ::= query_expression */ -{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy130; } +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy212; } break; - case 2: /* column_reference ::= NK_ID */ -{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 2: /* literal ::= NK_INTEGER */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 3: /* column_reference ::= table_name NK_DOT NK_ID */ - case 14: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==14); -{ PARSER_TRACE; yylhsminor.yy130 = createColumnNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 3: /* literal ::= NK_FLOAT */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 4: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause */ -{ PARSER_TRACE; yymsp[-3].minor.yy130 = createSelectStmt(pCxt, yymsp[-2].minor.yy9, yymsp[-1].minor.yy30, yymsp[0].minor.yy130); } + case 4: /* literal ::= NK_STRING */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 5: /* set_quantifier_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy9 = false; } + case 5: /* literal ::= NK_BOOL */ +{ PARSER_TRACE; yylhsminor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 6: /* set_quantifier_opt ::= DISTINCT */ -{ PARSER_TRACE; yymsp[0].minor.yy9 = true; } + case 6: /* literal ::= TIMESTAMP NK_STRING */ +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 7: /* set_quantifier_opt ::= ALL */ -{ PARSER_TRACE; yymsp[0].minor.yy9 = false; } + case 7: /* literal ::= duration_literal */ + case 17: /* expression ::= literal */ yytestcase(yyruleno==17); + case 18: /* expression ::= column_reference */ yytestcase(yyruleno==18); + case 20: /* expression ::= subquery */ yytestcase(yyruleno==20); + case 52: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==52); + case 56: /* boolean_primary ::= predicate */ yytestcase(yyruleno==56); + case 59: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==59); + case 61: /* table_reference ::= table_primary */ yytestcase(yyruleno==61); + case 62: /* table_reference ::= joined_table */ yytestcase(yyruleno==62); + case 81: /* select_item ::= expression */ yytestcase(yyruleno==81); + case 109: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==109); + case 111: /* query_primary ::= query_specification */ yytestcase(yyruleno==111); + case 124: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==124); +{ PARSER_TRACE; yylhsminor.yy212 = yymsp[0].minor.yy212; } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 8: /* select_list ::= NK_STAR */ -{ PARSER_TRACE; yymsp[0].minor.yy30 = NULL; } + case 8: /* duration_literal ::= NK_VARIABLE */ +{ PARSER_TRACE; yylhsminor.yy212 = createDurationValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy212 = yylhsminor.yy212; break; - case 9: /* select_list ::= select_sublist */ -{ PARSER_TRACE; yylhsminor.yy30 = yymsp[0].minor.yy30; } - yymsp[0].minor.yy30 = yylhsminor.yy30; + case 9: /* literal_list ::= literal */ + case 29: /* expression_list ::= expression */ yytestcase(yyruleno==29); + case 79: /* select_sublist ::= select_item */ yytestcase(yyruleno==79); + case 125: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==125); +{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, yymsp[0].minor.yy212); } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 10: /* select_sublist ::= select_item */ - case 37: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==37); -{ PARSER_TRACE; yylhsminor.yy30 = createNodeList(pCxt, yymsp[0].minor.yy130); } - yymsp[0].minor.yy30 = yylhsminor.yy30; + case 10: /* literal_list ::= literal_list NK_COMMA literal */ + case 30: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==30); + case 80: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==80); + case 126: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==126); +{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy174 = yylhsminor.yy174; break; - case 11: /* select_sublist ::= select_sublist NK_COMMA select_item */ - case 38: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==38); -{ PARSER_TRACE; yylhsminor.yy30 = addNodeToList(pCxt, yymsp[-2].minor.yy30, yymsp[0].minor.yy130); } - yymsp[-2].minor.yy30 = yylhsminor.yy30; + case 11: /* db_name ::= NK_ID */ + case 12: /* table_name ::= NK_ID */ yytestcase(yyruleno==12); + case 13: /* column_name ::= NK_ID */ yytestcase(yyruleno==13); + case 14: /* function_name ::= NK_ID */ yytestcase(yyruleno==14); + case 15: /* table_alias ::= NK_ID */ yytestcase(yyruleno==15); + case 16: /* column_alias ::= NK_ID */ yytestcase(yyruleno==16); +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy79 = yylhsminor.yy79; break; - case 12: /* select_item ::= value_expression */ - case 16: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==16); - case 17: /* table_reference ::= table_factor */ yytestcase(yyruleno==17); - case 18: /* table_factor ::= table_primary */ yytestcase(yyruleno==18); - case 25: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==25); - case 27: /* query_primary ::= query_specification */ yytestcase(yyruleno==27); -{ PARSER_TRACE; yylhsminor.yy130 = yymsp[0].minor.yy130; } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 19: /* expression ::= function_name NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yylhsminor.yy212 = createFunctionNode(pCxt, &yymsp[-3].minor.yy79, yymsp[-1].minor.yy174); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 13: /* select_item ::= value_expression AS NK_ID */ -{ PARSER_TRACE; yylhsminor.yy130 = setProjectionAlias(pCxt, yymsp[-2].minor.yy130, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 21: /* expression ::= NK_LP expression NK_RP */ + case 57: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==57); + case 69: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==69); + case 70: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==70); + case 123: /* subquery ::= NK_LR query_expression NK_RP */ yytestcase(yyruleno==123); +{ PARSER_TRACE; yymsp[-2].minor.yy212 = yymsp[-1].minor.yy212; } break; - case 15: /* from_clause ::= FROM table_reference_list */ -{ PARSER_TRACE; yymsp[-1].minor.yy130 = yymsp[0].minor.yy130; } + case 22: /* expression ::= NK_PLUS expression */ + case 58: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==58); + case 86: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==86); + case 107: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==107); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = yymsp[0].minor.yy212; } break; - case 19: /* table_primary ::= table_name */ -{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy67); } - yymsp[0].minor.yy130 = yylhsminor.yy130; + case 23: /* expression ::= NK_MINUS expression */ +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createOperatorNode(pCxt, OP_TYPE_SUB, yymsp[0].minor.yy212, NULL); } break; - case 20: /* table_primary ::= db_name NK_DOT table_name */ -{ PARSER_TRACE; yylhsminor.yy130 = createRealTableNode(pCxt, &yymsp[-2].minor.yy67, &yymsp[0].minor.yy67); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 24: /* expression ::= expression NK_PLUS expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_ADD, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 21: /* db_name ::= NK_ID */ - case 22: /* table_name ::= NK_ID */ yytestcase(yyruleno==22); -{ PARSER_TRACE; yylhsminor.yy67 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy67 = yylhsminor.yy67; + case 25: /* expression ::= expression NK_MINUS expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_SUB, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 23: /* query_expression ::= with_clause_opt query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ -{ yy_destructor(yypParser,61,&yymsp[-4].minor); + case 26: /* expression ::= expression NK_STAR expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_MULTI, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 27: /* expression ::= expression NK_SLASH expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_DIV, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 28: /* expression ::= expression NK_REM expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, OP_TYPE_MOD, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 31: /* column_reference ::= column_name */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy79); } + yymsp[0].minor.yy212 = yylhsminor.yy212; + break; + case 32: /* column_reference ::= table_name NK_DOT column_name */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 33: /* predicate ::= expression compare_op expression */ + case 38: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==38); +{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 34: /* predicate ::= expression BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createBetweenAnd(pCxt, yymsp[-4].minor.yy212, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-4].minor.yy212 = yylhsminor.yy212; + break; + case 35: /* predicate ::= expression NOT BETWEEN expression AND expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createNotBetweenAnd(pCxt, yymsp[-2].minor.yy212, yymsp[-5].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; + break; + case 36: /* predicate ::= expression IS NULL */ +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, yymsp[-2].minor.yy212, true); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 37: /* predicate ::= expression IS NOT NULL */ +{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, yymsp[-3].minor.yy212, false); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; + break; + case 39: /* compare_op ::= NK_LT */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_THAN; } + break; + case 40: /* compare_op ::= NK_GT */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_THAN; } + break; + case 41: /* compare_op ::= NK_LE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_EQUAL; } + break; + case 42: /* compare_op ::= NK_GE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_EQUAL; } + break; + case 43: /* compare_op ::= NK_NE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NOT_EQUAL; } + break; + case 44: /* compare_op ::= NK_EQ */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_EQUAL; } + break; + case 45: /* compare_op ::= LIKE */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LIKE; } + break; + case 46: /* compare_op ::= NOT LIKE */ +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_LIKE; } + break; + case 47: /* compare_op ::= MATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_MATCH; } + break; + case 48: /* compare_op ::= NMATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NMATCH; } + break; + case 49: /* in_op ::= IN */ +{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_IN; } + break; + case 50: /* in_op ::= NOT IN */ +{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_IN; } + break; + case 51: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yymsp[-2].minor.yy212 = createNodeListNode(pCxt, yymsp[-1].minor.yy174); } + break; + case 53: /* boolean_value_expression ::= NOT boolean_primary */ +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy212, NULL); } + break; + case 54: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 55: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ +{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 60: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy212, yymsp[0].minor.yy212, NULL); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; + break; + case 63: /* table_primary ::= table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; + break; + case 64: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, &yymsp[-3].minor.yy79, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; + break; + case 65: /* table_primary ::= subquery alias_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createTempTableNode(pCxt, yymsp[-1].minor.yy212, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; + break; + case 66: /* alias_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy79 = nil_token; } + break; + case 67: /* alias_opt ::= table_alias */ +{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy79; } + yymsp[0].minor.yy79 = yylhsminor.yy79; + break; + case 68: /* alias_opt ::= AS table_alias */ +{ PARSER_TRACE; yymsp[-1].minor.yy79 = yymsp[0].minor.yy79; } + break; + case 71: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, yymsp[-4].minor.yy162, yymsp[-5].minor.yy212, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-5].minor.yy212 = yylhsminor.yy212; + break; + case 72: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy162 = JOIN_TYPE_INNER; } + break; + case 73: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - PARSER_TRACE; - addOrderByList(pCxt, yymsp[-3].minor.yy130, yymsp[-2].minor.yy30); - addSlimit(pCxt, yymsp[-3].minor.yy130, yymsp[-1].minor.yy130); - addLimit(pCxt, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); - yymsp[-4].minor.yy130 = yymsp[-3].minor.yy130; - } -} + PARSER_TRACE; + yymsp[-8].minor.yy212 = createSelectStmt(pCxt, yymsp[-7].minor.yy237, yymsp[-6].minor.yy174, yymsp[-5].minor.yy212); + yymsp[-8].minor.yy212 = addWhereClause(pCxt, yymsp[-8].minor.yy212, yymsp[-4].minor.yy212); + yymsp[-8].minor.yy212 = addPartitionByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-3].minor.yy174); + yymsp[-8].minor.yy212 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy212, yymsp[-2].minor.yy212); + yymsp[-8].minor.yy212 = addGroupByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-1].minor.yy174); + yymsp[-8].minor.yy212 = addHavingClause(pCxt, yymsp[-8].minor.yy212, yymsp[0].minor.yy212); + } break; - case 24: /* with_clause_opt ::= */ -{} + case 74: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy237 = false; } break; - case 26: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ PARSER_TRACE; yylhsminor.yy130 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy130, yymsp[0].minor.yy130); } - yymsp[-3].minor.yy130 = yylhsminor.yy130; + case 75: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = true; } break; - case 28: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy130 = yymsp[-4].minor.yy130;} - yy_destructor(yypParser,63,&yymsp[-3].minor); - yy_destructor(yypParser,65,&yymsp[-2].minor); - yy_destructor(yypParser,64,&yymsp[-1].minor); + case 76: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy237 = false; } break; - case 29: /* order_by_clause_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy30 = NULL; } + case 77: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy174 = NULL; } break; - case 30: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ -{ PARSER_TRACE; yymsp[-2].minor.yy30 = yymsp[0].minor.yy30; } + case 78: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy174 = yymsp[0].minor.yy174; } + yymsp[0].minor.yy174 = yylhsminor.yy174; break; - case 31: /* slimit_clause_opt ::= */ - case 34: /* limit_clause_opt ::= */ yytestcase(yyruleno==34); -{ yymsp[1].minor.yy130 = NULL; } + case 82: /* select_item ::= expression column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, yymsp[-1].minor.yy212, &yymsp[0].minor.yy79); } + yymsp[-1].minor.yy212 = yylhsminor.yy212; break; - case 32: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 35: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==35); -{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 83: /* select_item ::= expression AS column_alias */ +{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, yymsp[-2].minor.yy212, &yymsp[0].minor.yy79); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 33: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 36: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==36); -{ yymsp[-3].minor.yy130 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 84: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 39: /* sort_specification ::= value_expression ordering_specification_opt null_ordering_opt */ -{ PARSER_TRACE; yylhsminor.yy130 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy130, yymsp[-1].minor.yy108, yymsp[0].minor.yy68); } - yymsp[-2].minor.yy130 = yylhsminor.yy130; + case 85: /* where_clause_opt ::= */ + case 89: /* twindow_clause_opt ::= */ yytestcase(yyruleno==89); + case 94: /* sliding_opt ::= */ yytestcase(yyruleno==94); + case 96: /* fill_opt ::= */ yytestcase(yyruleno==96); + case 106: /* having_clause_opt ::= */ yytestcase(yyruleno==106); + case 115: /* slimit_clause_opt ::= */ yytestcase(yyruleno==115); + case 119: /* limit_clause_opt ::= */ yytestcase(yyruleno==119); +{ PARSER_TRACE; yymsp[1].minor.yy212 = NULL; } break; - case 40: /* ordering_specification_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy108 = ORDER_ASC; } + case 87: /* partition_by_clause_opt ::= */ + case 104: /* group_by_clause_opt ::= */ yytestcase(yyruleno==104); + case 113: /* order_by_clause_opt ::= */ yytestcase(yyruleno==113); +{ PARSER_TRACE; yymsp[1].minor.yy174 = NULL; } break; - case 41: /* ordering_specification_opt ::= ASC */ -{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_ASC; } + case 88: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 105: /* group_by_clause_opt ::= GROUP BY expression_list */ yytestcase(yyruleno==105); + case 114: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==114); +{ PARSER_TRACE; yymsp[-2].minor.yy174 = yymsp[0].minor.yy174; } break; - case 42: /* ordering_specification_opt ::= DESC */ -{ PARSER_TRACE; yymsp[0].minor.yy108 = ORDER_DESC; } + case 90: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createSessionWindowNode(pCxt, yymsp[-3].minor.yy212, &yymsp[-1].minor.yy0); } break; - case 43: /* null_ordering_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy68 = NULL_ORDER_DEFAULT; } + case 91: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createStateWindowNode(pCxt, yymsp[-1].minor.yy212); } break; - case 44: /* null_ordering_opt ::= NULLS FIRST */ -{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_FIRST; } + case 92: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy212, NULL, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } break; - case 45: /* null_ordering_opt ::= NULLS LAST */ -{ PARSER_TRACE; yymsp[-1].minor.yy68 = NULL_ORDER_LAST; } + case 93: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-7].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy212, yymsp[-3].minor.yy212, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } break; - case 46: /* value_function ::= NK_ID NK_LP value_expression NK_RP */ - case 48: /* value_expression_primary ::= NK_LP value_expression NK_RP */ yytestcase(yyruleno==48); -{ -} - yy_destructor(yypParser,38,&yymsp[-1].minor); + case 95: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = yymsp[-1].minor.yy212; } break; - case 47: /* value_function ::= NK_ID NK_LP value_expression NK_COMMA value_expression NK_RP */ -{ -} - yy_destructor(yypParser,38,&yymsp[-3].minor); - yy_destructor(yypParser,38,&yymsp[-1].minor); + case 97: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createFillNode(pCxt, yymsp[-1].minor.yy44, NULL); } break; - case 49: /* value_expression_primary ::= nonparenthesized_value_expression_primary */ -{ yy_destructor(yypParser,40,&yymsp[0].minor); -{ -} -} + case 98: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } break; - case 50: /* nonparenthesized_value_expression_primary ::= literal */ -{ yy_destructor(yypParser,41,&yymsp[0].minor); -{ -} -} + case 99: /* fill_mode ::= NONE */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NONE; } break; - case 51: /* nonparenthesized_value_expression_primary ::= column_reference */ -{ yy_destructor(yypParser,42,&yymsp[0].minor); -{ -} -} + case 100: /* fill_mode ::= PREV */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_PREV; } break; - case 53: /* value_expression ::= common_value_expression */ -{ yy_destructor(yypParser,44,&yymsp[0].minor); -{ -} -} + case 101: /* fill_mode ::= NULL */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NULL; } break; - case 54: /* common_value_expression ::= numeric_value_expression */ -{ yy_destructor(yypParser,45,&yymsp[0].minor); -{ -} -} + case 102: /* fill_mode ::= LINEAR */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_LINEAR; } break; - case 55: /* numeric_value_expression ::= numeric_primary */ -{ yy_destructor(yypParser,46,&yymsp[0].minor); -{ -} -} + case 103: /* fill_mode ::= NEXT */ +{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NEXT; } break; - case 56: /* numeric_value_expression ::= NK_PLUS numeric_primary */ - case 57: /* numeric_value_expression ::= NK_MINUS numeric_primary */ yytestcase(yyruleno==57); -{ -} - yy_destructor(yypParser,46,&yymsp[0].minor); + case 108: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ +{ + PARSER_TRACE; + yylhsminor.yy212 = addOrderByClause(pCxt, yymsp[-3].minor.yy212, yymsp[-2].minor.yy174); + yylhsminor.yy212 = addSlimitClause(pCxt, yylhsminor.yy212, yymsp[-1].minor.yy212); + yylhsminor.yy212 = addLimitClause(pCxt, yylhsminor.yy212, yymsp[0].minor.yy212); + } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 58: /* numeric_value_expression ::= numeric_value_expression NK_PLUS numeric_value_expression */ - case 59: /* numeric_value_expression ::= numeric_value_expression NK_MINUS numeric_value_expression */ yytestcase(yyruleno==59); - case 60: /* numeric_value_expression ::= numeric_value_expression NK_STAR numeric_value_expression */ yytestcase(yyruleno==60); - case 61: /* numeric_value_expression ::= numeric_value_expression NK_SLASH numeric_value_expression */ yytestcase(yyruleno==61); -{ yy_destructor(yypParser,45,&yymsp[-2].minor); -{ -} - yy_destructor(yypParser,45,&yymsp[0].minor); -} + case 110: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy212 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy212, yymsp[0].minor.yy212); } + yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 62: /* numeric_primary ::= value_expression_primary */ -{ yy_destructor(yypParser,39,&yymsp[0].minor); -{ -} -} + case 112: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt limit_clause_opt slimit_clause_opt NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy212 = yymsp[-4].minor.yy212;} + yy_destructor(yypParser,117,&yymsp[-3].minor); + yy_destructor(yypParser,119,&yymsp[-2].minor); + yy_destructor(yypParser,118,&yymsp[-1].minor); break; - case 63: /* numeric_primary ::= value_function */ -{ yy_destructor(yypParser,37,&yymsp[0].minor); -{ -} -} + case 116: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 120: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==120); +{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 64: /* table_primary ::= derived_table */ -{ yy_destructor(yypParser,58,&yymsp[0].minor); -{ -} -} + case 117: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 121: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==121); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 65: /* derived_table ::= table_subquery */ -{ yy_destructor(yypParser,59,&yymsp[0].minor); -{ -} -} + case 118: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 122: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==122); +{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 66: /* subquery ::= NK_LR query_expression NK_RP */ -{ -} - yy_destructor(yypParser,36,&yymsp[-1].minor); + case 127: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy212 = createOrderByExprNode(pCxt, yymsp[-2].minor.yy212, yymsp[-1].minor.yy188, yymsp[0].minor.yy107); } + yymsp[-2].minor.yy212 = yylhsminor.yy212; break; - case 67: /* table_subquery ::= subquery */ -{ yy_destructor(yypParser,60,&yymsp[0].minor); + case 128: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy188 = ORDER_ASC; } + break; + case 129: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_ASC; } + break; + case 130: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_DESC; } + break; + case 131: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy107 = NULL_ORDER_DEFAULT; } + break; + case 132: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_FIRST; } + break; + case 133: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_LAST; } + break; + case 134: /* table_primary ::= parenthesized_joined_table */ +{ yy_destructor(yypParser,100,&yymsp[0].minor); { } } break; default: - /* (52) literal ::= NK_LITERAL */ yytestcase(yyruleno==52); break; /********** End reduce actions ************************************************/ }; @@ -1507,7 +1984,7 @@ static void yy_accept( /* Here code is inserted which will be executed whenever the ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ - printf("parsing complete!\n" ); + PARSER_COMPLETE; /*********** End %parse_accept code *******************************************/ NewParseARG_STORE /* Suppress warning about unused %extra_argument variable */ NewParseCTX_STORE diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index 8902da9274..fdedafcbb4 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -80,20 +80,21 @@ uint32_t getToken(const char* z, uint32_t* tokenId) { } int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { - SAstCreateContext cxt = { .pQueryCxt = pParseCxt, .valid = true, .pRootNode = NULL }; + SAstCreateContext cxt; + createAstCreateContext(pParseCxt, &cxt); void *pParser = NewParseAlloc(malloc); int32_t i = 0; while (1) { SToken t0 = {0}; - printf("===========================\n"); + // printf("===========================\n"); if (cxt.pQueryCxt->pSql[i] == 0) { NewParse(pParser, 0, t0, &cxt); goto abort_parse; } - printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); + // printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); t0.z = (char *)(cxt.pQueryCxt->pSql + i); - printf("token %p : %d %d [%s]\n", &t0, t0.type, t0.n, t0.z); + // printf("token : %d %d [%s]\n", t0.type, t0.n, t0.z); i += t0.n; switch (t0.type) { @@ -130,7 +131,9 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { } abort_parse: + // printf("doParse completed.\n"); NewParseFree(pParser, free); + destroyAstCreateContext(&cxt); pQuery->pRoot = cxt.pRootNode; return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index 5223c9f02a..706deb443c 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -38,22 +38,13 @@ protected: } - int32_t run() { + bool run(int32_t expectCode = TSDB_CODE_SUCCESS) { int32_t code = doParse(&cxt_, &query_); if (code != TSDB_CODE_SUCCESS) { - cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - return code; + cout << "sql:[" << cxt_.pSql << "] code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; + return (code == expectCode); } - cout << nodeType(query_.pRoot) << endl; if (NULL != query_.pRoot && QUERY_NODE_SELECT_STMT == nodeType(query_.pRoot)) { - // SNode* pWhereCond; - // SNodeList* pPartitionByList; // SNode - // SNode* pWindowClause; - // SNodeList* pGroupByList; // SGroupingSetNode - // SNodeList* pOrderByList; // SOrderByExprNode - // SLimitNode limit; - // SLimitNode slimit; - SSelectStmt* select = (SSelectStmt*)query_.pRoot; string sql("SELECT "); if (select->isDistinct) { @@ -68,15 +59,7 @@ protected: tableToSql(select->pFromTable, sql); cout << sql << endl; } - // char* pStr = NULL; - // int32_t len = 0; - // code = nodesNodeToString(query_.pRoot, &pStr, &len); - // if (code != TSDB_CODE_SUCCESS) { - // cout << "code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; - // return code; - // } - // cout << "node tree:\n" << pStr << endl; - return TSDB_CODE_SUCCESS; + return (code == expectCode); } private: @@ -85,7 +68,6 @@ private: void tableToSql(const SNode* node, string& sql) { const STableNode* table = (const STableNode*)node; - cout << "node : " << nodeType(node) << endl; switch (nodeType(node)) { case QUERY_NODE_REAL_TABLE: { SRealTableNode* realTable = (SRealTableNode*)table; @@ -108,12 +90,14 @@ private: if (!firstNode) { sql.append(", "); } + firstNode = false; switch (nodeType(node)) { case QUERY_NODE_COLUMN: sql.append(((SColumnNode*)node)->colName); break; } } + sql.append(" "); } void reset() { @@ -136,8 +120,30 @@ TEST_F(NewParserTest, selectStar) { setDatabase("root", "test"); bind("SELECT * FROM t1"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + ASSERT_TRUE(run()); bind("SELECT * FROM test.t1"); - ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + ASSERT_TRUE(run()); + + bind("SELECT ts FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT ts, tag1, c1 FROM t1"); + ASSERT_TRUE(run()); +} + +TEST_F(NewParserTest, syntaxError) { + setDatabase("root", "test"); + + bind("SELECTT * FROM t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT *, * FROM test.t1"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); + + bind("SELECT * FROM test.t1 t WHER"); + ASSERT_TRUE(run(TSDB_CODE_FAILED)); } diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 0be97f1ce0..20e8378c26 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -216,7 +216,8 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } else if (needSeqScan(pPlanNode)) { return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan); } - return createUserTableScanNode(pPlanNode, pTable, OP_DataBlocksOptScan); + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; + return createUserTableScanNode(pPlanNode, pTable, type); } static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) { @@ -287,7 +288,7 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) { static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) { SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList; vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode); - int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan; + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; return createUserTableScanNode(pPlanNode, pTableInfo, type); } @@ -296,6 +297,7 @@ static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNo if (needMultiNodeScan(pTable)) { return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } + return createSingleTableScanNode(pPlanNode, pTable, pCxt->pCurrentSubplan); } @@ -386,6 +388,33 @@ static void createSubplanByLevel(SPlanContext* pCxt, SQueryPlanNode* pRoot) { // todo deal subquery } +static void postCreateDag(SQueryPlanNode* pQueryNode, SQueryDag* pDag, SArray* pNodeList) { + // The exchange operator is not necessary, in case of the stream scan. + // Here we need to remove it from the DAG. + if (pQueryNode->info.type == QNODE_STREAMSCAN) { + SArray* pRootLevel = taosArrayGetP(pDag->pSubplans, 0); + SSubplan *pSubplan = taosArrayGetP(pRootLevel, 0); + + if (pSubplan->pNode->info.type == OP_Exchange) { + ASSERT(taosArrayGetSize(pRootLevel) == 1); + + taosArrayRemove(pDag->pSubplans, 0); + // And then update the number of the subplans. + pDag->numOfSubplans -= 1; + } + } else { + // Traverse the dag again to acquire the execution node. + if (pNodeList != NULL) { + SArray** pSubLevel = taosArrayGetLast(pDag->pSubplans); + size_t num = taosArrayGetSize(*pSubLevel); + for (int32_t j = 0; j < num; ++j) { + SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); + taosArrayPush(pNodeList, &pPlan->execNode); + } + } + } +} + int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryDag** pDag, SArray* pNodeList, uint64_t requestId) { TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { @@ -407,16 +436,7 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD return TSDB_CODE_FAILED; } END_TRY - // traverse the dag again to acquire the execution node. - if (pNodeList != NULL) { - SArray** pSubLevel = taosArrayGetLast((*pDag)->pSubplans); - size_t num = taosArrayGetSize(*pSubLevel); - for (int32_t j = 0; j < num; ++j) { - SSubplan* pPlan = taosArrayGetP(*pSubLevel, j); - taosArrayPush(pNodeList, &pPlan->execNode); - } - } - + postCreateDag(pQueryNode, *pDag, pNodeList); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 5d96ab47c3..e367f2e74b 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -88,7 +88,6 @@ static const char* jkPnodeType = "Type"; static int32_t getPnodeTypeSize(cJSON* json) { switch (getNumber(json, jkPnodeType)) { case OP_StreamScan: - case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return sizeof(STableScanPhyNode); @@ -831,7 +830,6 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { const SPhyNode* phyNode = (const SPhyNode*)obj; switch (phyNode->info.type) { case OP_StreamScan: - case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return tableScanNodeToJson(obj, json); @@ -869,7 +867,6 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* phyNode = (SPhyNode*)obj; switch (phyNode->info.type) { - case OP_TableScan: case OP_StreamScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index cfe2ed96ad..9f678b4528 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -128,7 +128,7 @@ int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransp if (NULL == pMsg) { qError("0x%"PRIx64" msg:%s malloc failed", pInfo->requestId, TMSG_INFO(pInfo->msgType)); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return -1; + return terrno; } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index a19a02069b..de2940846c 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -182,9 +182,9 @@ typedef struct SQWorkerMgmt { #define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) #define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9b9b6a73ca..8705bc23b9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -479,7 +479,6 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { if (NULL == pRes) { QW_TASK_DLOG("task query done, useconds:%"PRIu64, useconds); - dsEndPut(sinkHandle, useconds); if (TASK_TYPE_TEMP == ctx->taskType) { @@ -493,6 +492,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { break; } + ASSERT(pRes->info.rows > 0); + SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); if (code) { diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 11783335b3..368a590918 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -282,18 +282,21 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); + msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->contentLen = ntohl(msg->contentLen); - + msg->taskId = be64toh(msg->taskId); + msg->phyLen = ntohl(msg->phyLen); + msg->sqlLen = ntohl(msg->sqlLen); + uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; - SQWMsg qwMsg = {.node = node, .msg = msg->msg, .msgLen = msg->contentLen, .connection = pMsg}; + SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen, .connection = pMsg}; - QW_SCH_TASK_DLOG("processQuery start, node:%p", node); + char* sql = strndup(msg->msg, msg->sqlLen); + QW_SCH_TASK_DLOG("processQuery start, node:%p, sql:%s", node, sql); + tfree(sql); QW_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, msg->taskType)); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 4c81123bf2..38890150ee 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -113,7 +113,8 @@ void qwtBuildQueryReqMsg(SRpcMsg *queryRpc) { qwtqueryMsg.queryId = htobe64(atomic_add_fetch_64(&qwtTestQueryId, 1)); qwtqueryMsg.sId = htobe64(1); qwtqueryMsg.taskId = htobe64(1); - qwtqueryMsg.contentLen = htonl(100); + qwtqueryMsg.phyLen = htonl(100); + qwtqueryMsg.sqlLen = 0; queryRpc->msgType = TDMT_VND_QUERY; queryRpc->pCont = &qwtqueryMsg; queryRpc->contLen = sizeof(SSubQueryMsg) + 100; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index df62e72770..94c833307f 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -128,6 +128,7 @@ typedef struct SSchJob { int32_t errCode; void *res; //TODO free it or not int32_t resNumOfRows; + const char *sql; SQueryProfileSummary summary; } SSchJob; @@ -150,11 +151,11 @@ typedef struct SSchJob { #define SCH_JOB_DLOG(param, ...) qDebug("QID:0x%" PRIx64 " " param, pJob->queryId, __VA_ARGS__) #define SCH_TASK_ELOG(param, ...) \ - qError("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ - qDebug("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ - qWarn("QID:0x%" PRIx64 ",TID:%" PRId64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) + qWarn("QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, pJob->queryId, pTask->taskId, __VA_ARGS__) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ab35993966..04505d33cd 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -345,7 +345,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { } pLevel = taosArrayGet(pJob->levels, i); - pLevel->level = i; plans = taosArrayGetP(pDag->pSubplans, i); @@ -375,7 +374,7 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SSchTask task = {0}; SSchTask *pTask = &task; - + SCH_ERR_JRET(schInitTask(pJob, &task, plan, pLevel)); void *p = taosArrayPush(pLevel->subTasks, &task); @@ -388,8 +387,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - - SCH_TASK_DLOG("task initialized, level:%d", pLevel->level); } SCH_JOB_DLOG("level initialized, taskNum:%d", taskNum); @@ -716,8 +713,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { int32_t code = 0; SSchTask *pErrTask = pTask; - SCH_TASK_DLOG("taskOnSuccess, status:%d", SCH_GET_TASK_STATUS(pTask)); - code = schMoveTaskToSuccList(pJob, pTask, &moved); if (code && moved) { SCH_ERR_RET(code); @@ -739,7 +734,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { if (taskDone < pTask->level->taskNum) { SCH_TASK_DLOG("wait all tasks, done:%d, all:%d", taskDone, pTask->level->taskNum); - return TSDB_CODE_SUCCESS; } else if (taskDone > pTask->level->taskNum) { SCH_TASK_ELOG("taskDone number invalid, done:%d, total:%d", taskDone, pTask->level->taskNum); @@ -1031,22 +1025,18 @@ int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t t pMsgSendInfo->fp = fp; int64_t transporterId = 0; - code = asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo); if (code) { - qError("QID:%"PRIx64 ",TID:%"PRIx64 " asyncSendMsgToServer failed, code:%x", qId, tId, code); SCH_ERR_JRET(code); } - qDebug("QID:%"PRIx64 ",TID:%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); - + qDebug("QID:0x%"PRIx64 ",TID:0x%"PRIx64 " req msg sent, type:%d, %s", qId, tId, msgType, TMSG_INFO(msgType)); return TSDB_CODE_SUCCESS; _return: tfree(param); tfree(pMsgSendInfo); - SCH_RET(code); } @@ -1057,7 +1047,6 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, bool isCandidateAddr = false; if (NULL == addr) { addr = taosArrayGet(pTask->candidateAddrs, atomic_load_8(&pTask->candidateIdx)); - isCandidateAddr = true; } @@ -1078,7 +1067,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_VND_QUERY: { - msgSize = sizeof(SSubQueryMsg) + pTask->msgLen; + uint32_t len = strlen(pJob->sql); + + msgSize = sizeof(SSubQueryMsg) + pTask->msgLen + len; msg = calloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); @@ -1086,15 +1077,16 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } SSubQueryMsg *pMsg = msg; - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->taskType = TASK_TYPE_TEMP; - pMsg->contentLen = htonl(pTask->msgLen); - memcpy(pMsg->msg, pTask->msg, pTask->msgLen); + pMsg->sId = htobe64(schMgmt.sId); + pMsg->queryId = htobe64(pJob->queryId); + pMsg->taskId = htobe64(pTask->taskId); + pMsg->taskType = TASK_TYPE_TEMP; + pMsg->phyLen = htonl(pTask->msgLen); + pMsg->sqlLen = htonl(len); + + memcpy(pMsg->msg, pJob->sql, len); + memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); break; } @@ -1205,7 +1197,7 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); SCH_ERR_JRET(code); } else { - SCH_TASK_DLOG(" ===physical plan=== len:%d, %s", pTask->msgLen, pTask->msg); + SCH_TASK_DLOG("physical plan len:%d, %s", pTask->msgLen, pTask->msg); } } @@ -1214,7 +1206,6 @@ int32_t schLaunchTask(SSchJob *pJob, SSchTask *pTask) { // NOTE: race condition: the task should be put into the hash table before send msg to server if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { SCH_ERR_JRET(schPushTaskToExecList(pJob, pTask)); - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_EXECUTING); } @@ -1281,7 +1272,7 @@ void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->failTasks); } -int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { +static int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** job, const char* sql, bool syncSchedule) { qDebug("QID:0x%"PRIx64" job started", pDag->queryId); if (pNodeList == NULL || (pNodeList && taosArrayGetSize(pNodeList) <= 0)) { @@ -1297,6 +1288,7 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru pJob->attr.syncSchedule = syncSchedule; pJob->transport = transport; + pJob->sql = sql; if (pNodeList != NULL) { pJob->nodeList = taosArrayDup(pNodeList); @@ -1336,7 +1328,6 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru } pJob->status = JOB_TASK_STATUS_NOT_START; - SCH_ERR_JRET(schLaunchJob(pJob)); *(SSchJob **)job = pJob; @@ -1347,15 +1338,12 @@ int32_t schExecJobImpl(void *transport, SArray *pNodeList, SQueryDag* pDag, stru } SCH_JOB_DLOG("job exec done, job status:%d", SCH_GET_JOB_STATUS(pJob)); - return TSDB_CODE_SUCCESS; _return: *(SSchJob **)job = NULL; - schedulerFreeJob(pJob); - SCH_RET(code); } @@ -1399,29 +1387,24 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { +int32_t schedulerExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, const char* sql, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = NULL; - - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, true)); - - job = *pJob; - - pRes->code = atomic_load_32(&job->errCode); - pRes->numOfRows = job->resNumOfRows; + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, sql, true)); + pRes->code = atomic_load_32(&(*pJob)->errCode); + pRes->numOfRows = (*pJob)->resNumOfRows; return TSDB_CODE_SUCCESS; } -int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, struct SSchJob** pJob) { +int32_t schedulerAsyncExecJob(void *transport, SArray *pNodeList, SQueryDag* pDag, const char* sql, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, false)); + SCH_ERR_RET(schExecJobImpl(transport, pNodeList, pDag, pJob, sql, false)); return TSDB_CODE_SUCCESS; } @@ -1456,7 +1439,6 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { for (int32_t i = 0; i < taskNum; ++i) { SSubplan *plan = taosArrayGetP(plans, i); - tInfo.addr = plan->execNode; code = qSubPlanToString(plan, &msg, &msgLen); @@ -1472,16 +1454,16 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { } SSubQueryMsg* pMsg = calloc(1, msgSize); - /*SSubQueryMsg *pMsg = (SSubQueryMsg*) msg;*/ memcpy(pMsg->msg, msg, msgLen); pMsg->header.vgId = tInfo.addr.nodeId; - pMsg->sId = schMgmt.sId; - pMsg->queryId = plan->id.queryId; - pMsg->taskId = schGenUUID(); + pMsg->sId = schMgmt.sId; + pMsg->queryId = plan->id.queryId; + pMsg->taskId = schGenUUID(); pMsg->taskType = TASK_TYPE_PERSISTENT; - pMsg->contentLen = msgLen; + pMsg->phyLen = msgLen; + pMsg->sqlLen = 0; /*memcpy(pMsg->msg, ((SSubQueryMsg*)msg)->msg, msgLen);*/ tInfo.msg = pMsg; @@ -1514,7 +1496,7 @@ int32_t schedulerCopyTask(STaskInfo *src, SArray **dst, int32_t copyNum) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - int32_t msgSize = src->msg->contentLen + sizeof(*src->msg); + int32_t msgSize = src->msg->phyLen + sizeof(*src->msg); STaskInfo info = {0}; info.addr = src->addr; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6e44d99c15..2c75683ecb 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -378,7 +378,7 @@ void* schtRunJobThread(void *aa) { qnodeAddr.port = 6031; taosArrayPush(qnodeList, &qnodeAddr); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &job); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &job); assert(code == 0); execTasks = taosHashInit(5, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); @@ -539,7 +539,7 @@ TEST(queryTest, normalCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, &pJob); + code = schedulerAsyncExecJob(mockPointer, qnodeList, &dag, "select * from tb", &pJob); ASSERT_EQ(code, 0); SSchJob *job = (SSchJob *)pJob; @@ -649,7 +649,7 @@ TEST(insertTest, normalCase) { pthread_create(&(thread1), &thattr, schtSendRsp, &pInsertJob); SQueryResult res = {0}; - code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, &res); + code = schedulerExecJob(mockPointer, qnodeList, &dag, &pInsertJob, "insert into tb values(now,1)", &res); ASSERT_EQ(code, 0); ASSERT_EQ(res.numOfRows, 20); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 3f2aa1170e..f4fe0b1f79 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -211,6 +211,18 @@ typedef struct SConnBuffer { int left; } SConnBuffer; +typedef void (*AsyncCB)(uv_async_t* handle); + +typedef struct { + int index; + int nAsync; + uv_async_t* asyncs; +} SAsyncPool; + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb); +void transDestroyAsyncPool(SAsyncPool* pool); +int transSendAsync(SAsyncPool* pool); + int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 069ebaeb8a..5037de1407 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -42,9 +42,10 @@ typedef struct SCliMsg { } SCliMsg; typedef struct SCliThrdObj { - pthread_t thread; - uv_loop_t* loop; - uv_async_t* cliAsync; // + pthread_t thread; + uv_loop_t* loop; + // uv_async_t* cliAsync; // + SAsyncPool* asyncPool; uv_timer_t* timer; void* pool; // conn pool queue msg; @@ -379,7 +380,7 @@ static void clientConnCb(uv_connect_t* req, int status) { static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { tDebug("thread %p start to quit", pThrd); destroyCmsg(pMsg); - uv_close((uv_handle_t*)pThrd->cliAsync, NULL); + // transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL); uv_timer_stop(pThrd->timer); pThrd->quit = true; // uv__async_stop(pThrd->cliAsync); @@ -501,6 +502,7 @@ static void destroyCmsg(SCliMsg* pMsg) { destroyUserdata(&pMsg->msg); free(pMsg); } + static SCliThrdObj* createThrdObj() { SCliThrdObj* pThrd = (SCliThrdObj*)calloc(1, sizeof(SCliThrdObj)); QUEUE_INIT(&pThrd->msg); @@ -509,9 +511,7 @@ static SCliThrdObj* createThrdObj() { pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); uv_loop_init(pThrd->loop); - pThrd->cliAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->cliAsync, clientAsyncCb); - pThrd->cliAsync->data = pThrd; + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, clientAsyncCb); pThrd->timer = malloc(sizeof(uv_timer_t)); uv_timer_init(pThrd->loop, pThrd->timer); @@ -529,7 +529,8 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { uv_stop(pThrd->loop); pthread_join(pThrd->thread, NULL); pthread_mutex_destroy(&pThrd->msgMtx); - free(pThrd->cliAsync); + transDestroyAsyncPool(pThrd->asyncPool); + // free(pThrd->cliAsync); free(pThrd->timer); free(pThrd->loop); free(pThrd); @@ -551,7 +552,8 @@ static void clientSendQuit(SCliThrdObj* thrd) { QUEUE_PUSH(&thrd->msg, &msg->q); pthread_mutex_unlock(&thrd->msgMtx); - uv_async_send(thrd->cliAsync); + transSendAsync(thrd->asyncPool); + // uv_async_send(thrd->cliAsync); } void taosCloseClient(void* arg) { // impl later @@ -600,6 +602,10 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* QUEUE_PUSH(&thrd->msg, &cliMsg->q); pthread_mutex_unlock(&thrd->msgMtx); - uv_async_send(thrd->cliAsync); + int start = taosGetTimestampUs(); + transSendAsync(thrd->asyncPool); + // uv_async_send(thrd->cliAsync); + int end = taosGetTimestampUs() - start; + // tError("client sent to rpc, time cost: %d", (int)end); } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index ca39f85eb3..47eabd4320 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -245,4 +245,36 @@ int transDestroyBuffer(SConnBuffer* buf) { } transClearBuffer(buf); } + +SAsyncPool* transCreateAsyncPool(uv_loop_t* loop, void* arg, AsyncCB cb) { + static int sz = 20; + + SAsyncPool* pool = calloc(1, sizeof(SAsyncPool)); + pool->index = 0; + pool->nAsync = sz; + pool->asyncs = calloc(1, sizeof(uv_async_t) * pool->nAsync); + + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + uv_async_init(loop, async, cb); + async->data = arg; + } + return pool; +} +void transDestroyAsyncPool(SAsyncPool* pool) { + for (int i = 0; i < pool->nAsync; i++) { + uv_async_t* async = &(pool->asyncs[i]); + } + free(pool->asyncs); + free(pool); +} +int transSendAsync(SAsyncPool* pool) { + int idx = pool->index; + idx = idx % pool->nAsync; + // no need mutex here + if (pool->index++ > pool->nAsync) { + pool->index = 0; + } + return uv_async_send(&(pool->asyncs[idx])); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 475ef32b46..826b91dc02 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -22,7 +22,7 @@ typedef struct SSrvConn { uv_write_t* pWriter; uv_timer_t* pTimer; - uv_async_t* pWorkerAsync; + // uv_async_t* pWorkerAsync; queue queue; int ref; int persist; // persist connection or not @@ -50,11 +50,12 @@ typedef struct SSrvMsg { } SSrvMsg; typedef struct SWorkThrdObj { - pthread_t thread; - uv_pipe_t* pipe; - int fd; - uv_loop_t* loop; - uv_async_t* workerAsync; // + pthread_t thread; + uv_pipe_t* pipe; + int fd; + uv_loop_t* loop; + SAsyncPool* asyncPool; + // uv_async_t* workerAsync; // queue msg; pthread_mutex_t msgMtx; void* pTransInst; @@ -469,7 +470,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { pConn->pTimer->data = pConn; pConn->hostThrd = pThrd; - pConn->pWorkerAsync = pThrd->workerAsync; // thread safty + // pConn->pWorkerAsync = pThrd->workerAsync; // thread safty // init client handle pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); @@ -512,10 +513,7 @@ static bool addHandleToWorkloop(void* arg) { QUEUE_INIT(&pThrd->msg); pthread_mutex_init(&pThrd->msgMtx, NULL); - pThrd->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); - pThrd->workerAsync->data = pThrd; - + pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, pThrd, uvWorkerAsyncCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true; } @@ -665,7 +663,9 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { } pthread_join(pThrd->thread, NULL); free(pThrd->loop); - free(pThrd->workerAsync); + transDestroyAsyncPool(pThrd->asyncPool); + + // free(pThrd->workerAsync); free(pThrd); } void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { @@ -676,7 +676,8 @@ void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { pthread_mutex_unlock(&pThrd->msgMtx); tDebug("send quit msg to work thread"); - uv_async_send(pThrd->workerAsync); + transSendAsync(pThrd->asyncPool); + // uv_async_send(pThrd->workerAsync); } void taosCloseServer(void* arg) { @@ -716,8 +717,8 @@ void rpcSendResponse(const SRpcMsg* pMsg) { pthread_mutex_unlock(&pThrd->msgMtx); tDebug("conn %p start to send resp", pConn); - - uv_async_send(pThrd->workerAsync); + transSendAsync(pThrd->asyncPool); + // uv_async_send(pThrd->workerAsync); } #endif diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 6767e704b5..308b7b54bd 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include #include #include "os.h" @@ -53,6 +54,12 @@ static void *sendRequest(void *param) { tDebug("thread:%d, start to send request", pInfo->index); + tDebug("thread:%d, reqs: %d", pInfo->index, pInfo->numOfReqs); + int u100 = 0; + int u500 = 0; + int u1000 = 0; + int u10000 = 0; + while (pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) { pInfo->num++; rpcMsg.pCont = rpcMallocCont(pInfo->msgSize); @@ -60,15 +67,28 @@ static void *sendRequest(void *param) { rpcMsg.ahandle = pInfo; rpcMsg.msgType = 1; // tDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num); + int64_t start = taosGetTimestampUs(); rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg, NULL); if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); + int64_t end = taosGetTimestampUs() - start; + if (end <= 100) { + u100++; + } else if (end > 100 && end <= 500) { + u500++; + } else if (end > 500 && end < 1000) { + u1000++; + } else { + u10000++; + } + tDebug("recv response succefully"); // usleep(100000000); } + tError("send and recv sum: %d, %d, %d, %d", u100, u500, u1000, u10000); tDebug("thread:%d, it is over", pInfo->index); tcount++; @@ -161,8 +181,8 @@ int main(int argc, char *argv[]) { tInfo("client is initialized"); tInfo("threads:%d msgSize:%d requests:%d", appThreads, msgSize, numOfReqs); - // gettimeofday(&systemTime, NULL); - // startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + gettimeofday(&systemTime, NULL); + startTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo) * appThreads); @@ -184,13 +204,12 @@ int main(int argc, char *argv[]) { usleep(1); } while (tcount < appThreads); - // gettimeofday(&systemTime, NULL); - // endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; - // float usedTime = (endTime - startTime) / 1000.0f; // mseconds + gettimeofday(&systemTime, NULL); + endTime = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + float usedTime = (endTime - startTime) / 1000.0f; // mseconds - // tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); - // tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, - // msgSize); + tInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs * appThreads); + tInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0 * numOfReqs * appThreads / usedTime, msgSize); int ch = getchar(); UNUSED(ch); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 96d0c6c385..1d9201f69d 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -169,10 +169,17 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { return -1; } - ASSERT(pRead->pHead->head.version == ver); + if (pRead->pHead->head.version != ver) { + wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } code = walValidBodyCksum(pRead->pHead); if (code != 0) { + wError("unexpected wal log version: checksum not passed"); + pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } diff --git a/source/nodes/src/nodesEqualFuncs.c b/source/nodes/src/nodesEqualFuncs.c index f2752fe7f9..41d1e5b05d 100644 --- a/source/nodes/src/nodesEqualFuncs.c +++ b/source/nodes/src/nodesEqualFuncs.c @@ -91,7 +91,7 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCondNode* b) { COMPARE_NODE_FIELD(pExpr); - COMPARE_SCALAR_FIELD(isNot); + COMPARE_SCALAR_FIELD(isNull); return true; } diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/nodes/src/nodesUtilFuncs.c index 535bd8e171..bf4c4a83cc 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/nodes/src/nodesUtilFuncs.c @@ -15,17 +15,13 @@ #include "nodes.h" #include "nodesShowStmts.h" - -bool nodesIsTimeorderQuery(const SNode* pQuery) { - -} - -bool nodesIsTimelineQuery(const SNode* pQuery) { - -} +#include "taoserror.h" static SNode* makeNode(ENodeType type, size_t size) { SNode* p = calloc(1, size); + if (NULL == p) { + return NULL; + } setNodeType(p, type); return p; } @@ -78,6 +74,42 @@ void nodesDestroyNode(SNode* pNode) { } -void nodesDestroyNodeList(SNodeList* pList) { +SNodeList* nodesMakeList() { + SNodeList* p = calloc(1, sizeof(SNodeList)); + if (NULL == p) { + return NULL; + } + return p; +} + +SNodeList* nodesListAppend(SNodeList* pList, SNode* pNode) { + if (NULL == pList || NULL == pNode) { + return NULL; + } + SListCell* p = calloc(1, sizeof(SListCell)); + if (NULL == p) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pList; + } + p->pNode = pNode; + if (NULL == pList->pHead) { + pList->pHead = p; + } + if (NULL != pList->pTail) { + pList->pTail->pNext = p; + } + pList->pTail = p; + return pList; +} + +void nodesDestroyList(SNodeList* pList) { } + +bool nodesIsTimeorderQuery(const SNode* pQuery) { + +} + +bool nodesIsTimelineQuery(const SNode* pQuery) { + +} \ No newline at end of file diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 04efbef5b3..8999646f6a 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "os.h" #include "osString.h" diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index ed936e90f6..760a66a4fb 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -9,7 +9,6 @@ target_include_directories( target_link_libraries( util PRIVATE os - PUBLIC zlib PUBLIC lz4_static PUBLIC api ) diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index ddd283ae0f..0456d6a2ee 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "tbuffer.h" #include "exception.h" #include "os.h" diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 489fff5d64..ee5bea0ab7 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -254,6 +254,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retriev TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NOT_EXIST, "Transaction not exists") +// mnode-topic +TAOS_DEFINE_ERROR(TSDB_CODE_MND_UNSUPPORTED_TOPIC, "Topic with STable not supported yet") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_ACTION_IN_PROGRESS, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OFFLINE, "Dnode is offline") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index c7b1350591..12aa77214f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,7 +19,6 @@ #include "tnote.h" #include "tutil.h" #include "ulog.h" -//#include "zlib.h" #define MAX_LOGLINE_SIZE (1000) #define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) @@ -92,7 +91,7 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; -int32_t tqDebugFlag = 131; +int32_t tqDebugFlag = 135; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; diff --git a/source/util/src/tpagedfile.c b/source/util/src/tpagedfile.c index 3373d09876..3cdba580d4 100644 --- a/source/util/src/tpagedfile.c +++ b/source/util/src/tpagedfile.c @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE + #include "tpagedfile.h" #include "thash.h" #include "stddef.h" diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 383a00232a..ee0ade03b8 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -45,4 +45,10 @@ target_link_libraries(freelistTest os util gtest gtest_main) add_executable(encodeTest "encodeTest.cpp") target_link_libraries(encodeTest os util gtest gtest_main) - +# queueTest +add_executable(queue_test "queueTest.cpp") +target_link_libraries(queue_test os util gtest_main) +add_test( + NAME queue_test + COMMAND queue_test +) \ No newline at end of file diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp new file mode 100644 index 0000000000..310ae4350e --- /dev/null +++ b/source/util/test/queueTest.cpp @@ -0,0 +1,29 @@ +/** + * @file queue.cpp + * @author slguan (slguan@taosdata.com) + * @brief UTIL module queue tests + * @version 1.0 + * @date 2022-01-27 + * + * @copyright Copyright (c) 2022 + * + */ + +#include + +#include "os.h" +#include "tqueue.h" + +class UtilTestQueue : public ::testing::Test { + public: + void SetUp() override {} + void TearDown() override {} + + public: + static void SetUpTestSuite() {} + static void TearDownTestSuite() {} +}; + +TEST_F(UtilTestQueue, 01_ReadQitemFromQsetByThread) { + EXPECT_EQ(0, 0); +} \ No newline at end of file diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin new file mode 160000 index 0000000000..4a4d79099b --- /dev/null +++ b/src/connector/grafanaplugin @@ -0,0 +1 @@ +Subproject commit 4a4d79099b076b8ff12d5b4fdbcba54049a6866d diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index b785ed2707..4186f0dae1 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -966,7 +966,7 @@ int isCommentLine(char *line) { void source_file(TAOS *con, char *fptr) { wordexp_t full_path; int read_len = 0; - char *cmd = calloc(1, tsMaxSQLStringLen + 1); + char *cmd = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); size_t cmd_len = 0; char *line = NULL; size_t line_len = 0; @@ -998,7 +998,7 @@ void source_file(TAOS *con, char *fptr) { } while ((read_len = tgetline(&line, &line_len, f)) != -1) { - if (read_len >= tsMaxSQLStringLen) continue; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -1015,7 +1015,7 @@ void source_file(TAOS *con, char *fptr) { memcpy(cmd + cmd_len, line, read_len); printf("%s%s\n", PROMPT_HEADER, cmd); shellRunCommand(con, cmd); - memset(cmd, 0, tsMaxSQLStringLen); + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; }