diff --git a/CMakeLists.txt b/CMakeLists.txt index e1fbd54be3..0436f5b259 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project( ) if (NOT DEFINED TD_SOURCE_DIR) - set( TD_SOURCE_DIR ${CMAKE_SOURCE_DIR} ) + set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} ) endif() set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") diff --git a/Jenkinsfile b/Jenkinsfile index fc2b3562c1..4b84e1f88e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,7 +113,7 @@ pipeline { ''' sh''' cd ${WKC}/debug - ctest + ctest -VV ''' } } diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 441cf89626..c9c9c3a7ca 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -88,6 +88,12 @@ def pre_test(){ cmake .. > /dev/null make -j4> /dev/null ''' + sh''' + cd ${WKPY} + git reset --hard + git pull + pip3 install . + ''' return 1 } @@ -97,6 +103,7 @@ pipeline { environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDengine' + WKPY= '/var/lib/jenkins/workspace/taos-connector-python' } stages { stage('pre_build'){ @@ -113,12 +120,17 @@ pipeline { timeout(time: 45, unit: 'MINUTES'){ pre_test() sh''' - cd ${WKC}/tests - ./test-all.sh b1fq + cd ${WKC}/debug + ctest -VV ''' sh''' - cd ${WKC}/debug - ctest + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + sh''' + cd ${WKC}/tests + ./test-all.sh b1fq ''' } } diff --git a/cmake/cmake.define b/cmake/cmake.define index c9a188600a..fb6ba8cc2e 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.16) -set(CMAKE_VERBOSE_MAKEFILE ON) +set(CMAKE_VERBOSE_MAKEFILE OFF) #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 19923a5ad6..1ddc765c5c 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,24 +14,9 @@ if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() -# iconv -if(${BUILD_WITH_ICONV}) - cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# msvc regex -if(${BUILD_MSVCREGEX}) - cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# wcwidth -if(${BUILD_WCWIDTH}) - cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# wingetopt -if(${BUILD_WINGETOPT}) - cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# gnu regex +if(${BUILD_GNUREGEX}) + cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() # googletest @@ -114,27 +99,8 @@ if(${BUILD_TEST}) target_include_directories( gtest PUBLIC $ + PUBLIC $ ) - if(${TD_WINDOWS}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_WINDOWS}) - if(${TD_LINUX}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_LINUX}) - if(${TD_DARWIN}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_DARWIN}) - - endif(${BUILD_TEST}) # cJson @@ -216,53 +182,6 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) -# pthread -if(${BUILD_PTHREAD}) - set(CMAKE_BUILD_TYPE release) - add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) - set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) - add_library(pthread STATIC IMPORTED GLOBAL) - SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) -endif() - -# iconv -if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv) -endif(${BUILD_WITH_ICONV}) - -# wingetopt -if(${BUILD_WINGETOPT}) - add_subdirectory(wingetopt) -endif(${BUILD_WINGETOPT}) - -# msvcregex -if(${BUILD_MSVCREGEX}) - add_library(msvcregex STATIC "") - target_sources(msvcregex - PRIVATE "msvcregex/regex.c" - ) - target_include_directories(msvcregex - PRIVATE "msvcregex" - ) - target_link_libraries(msvcregex - INTERFACE Shell32 - ) - SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex) -endif(${BUILD_MSVCREGEX}) - -# msvcregex -if(${BUILD_WCWIDTH}) - add_library(wcwidth STATIC "") - target_sources(wcwidth - PRIVATE "wcwidth/wcwidth.c" - ) - target_include_directories(wcwidth - PRIVATE "wcwidth" - ) - SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth) -endif(${BUILD_WCWIDTH}) - # CRAFT if(${BUILD_WITH_CRAFT}) add_library(craft STATIC IMPORTED GLOBAL) @@ -319,12 +238,8 @@ if(${BUILD_WITH_SQLITE}) target_link_libraries(sqlite INTERFACE m INTERFACE pthread + INTERFACE dl ) - if(NOT TD_WINDOWS) - target_link_libraries(sqlite - INTERFACE dl - ) - endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) # pthread diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index ec8b44b673..e0f6ae64bd 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,9 +1,2 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) -if(${BUILD_WINGETOPT}) - target_link_libraries(simulate_vnode PUBLIC wingetopt) - target_include_directories( - simulate_vnode - PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" - ) -endif() \ No newline at end of file +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file diff --git a/contrib/test/tdev/src/main.c b/contrib/test/tdev/src/main.c index e40040ce97..5e1de83e88 100644 --- a/contrib/test/tdev/src/main.c +++ b/contrib/test/tdev/src/main.c @@ -6,39 +6,43 @@ #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) -static inline void tPutA(void **buf, uint64_t val) { - memcpy(buf, &val, sizeof(val)); - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutA(buf, val) \ + ({ \ + memcpy(buf, &val, sizeof(val)); \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) -static inline void tPutB(void **buf, uint64_t val) { - ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; - ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; - ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; - ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; - ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; - ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; - ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; - ((uint8_t *)buf)[0] = (val)&0xff; - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutB(buf, val) \ + ({ \ + ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \ + ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \ + ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \ + ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \ + ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ + ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ + ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ + ((uint8_t *)buf)[0] = (val)&0xff; \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) -static inline void tPutC(void **buf, uint64_t val) { - if (buf) { - ((uint64_t *)buf)[0] = (val); - POINTER_SHIFT(buf, sizeof(val)); - } - *buf = NULL; -} +#define tPutC(buf, val) \ + ({ \ + if (buf) { \ + ((uint64_t *)buf)[0] = (val); \ + POINTER_SHIFT(buf, sizeof(val)); \ + } \ + NULL; \ + }) -static inline void tPutD(void **buf, uint64_t val) { - uint64_t tmp = val; - for (size_t i = 0; i < sizeof(val); i++) { - ((uint8_t *)buf)[i] = tmp & 0xff; - tmp >>= 8; - } - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutD(buf, val) \ + ({ \ + uint64_t tmp = val; \ + for (size_t i = 0; i < sizeof(val); i++) { \ + ((uint8_t *)buf)[i] = tmp & 0xff; \ + tmp >>= 8; \ + } \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) static inline void tPutE(void **buf, uint64_t val) { if (buf) { @@ -57,7 +61,7 @@ static void func(T t) { switch (t) { case A: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutA(pBuf, val); + pBuf = tPutA(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -65,7 +69,7 @@ static void func(T t) { break; case B: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutB(pBuf, val); + pBuf = tPutB(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -73,7 +77,7 @@ static void func(T t) { break; case C: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutC(pBuf, val); + pBuf = tPutC(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -81,7 +85,7 @@ static void func(T t) { break; case D: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutD(pBuf, val); + pBuf = tPutD(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } diff --git a/example/src/tmq.c b/example/src/tmq.c index 2abf915fd8..56f210081b 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "taos.h" static int running = 1; @@ -47,6 +48,7 @@ int32_t init_env() { return -1; } taos_free_result(pRes); + sleep(1); pRes = taos_query(pConn, "use abc1"); if (taos_errno(pRes) != 0) { @@ -101,8 +103,8 @@ int32_t create_topic() { } taos_free_result(pRes); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1"); + pRes = taos_query(pConn, "create topic topic_ctb_column as abc1"); + /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");*/ if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -160,9 +162,10 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "group.id", "tg2"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); - tmq_conf_set(conf, "td.connect.db", "abc1"); + /*tmq_conf_set(conf, "td.connect.db", "abc1");*/ tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); return tmq; } @@ -265,10 +268,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) { } int main(int argc, char* argv[]) { - int code; if (argc > 1) { printf("env init\n"); - code = init_env(); + if (init_env() < 0) { + return -1; + } create_topic(); } tmq_t* tmq = build_consumer(); diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index e94de3cbca..17a9257c49 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -1,7 +1,7 @@ PROJECT(TDengine) IF (TD_LINUX) - INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(demo apitest.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) @@ -13,7 +13,7 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) ENDIF () IF (TD_DARWIN) - INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) + INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(demo demo.c) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua) diff --git a/examples/c/asyncdemo.c b/examples/c/asyncdemo.c index 9e214e0966..07e61b871e 100644 --- a/examples/c/asyncdemo.c +++ b/examples/c/asyncdemo.c @@ -45,7 +45,7 @@ typedef struct { void taos_insert_call_back(void *param, TAOS_RES *tres, int code); void taos_select_call_back(void *param, TAOS_RES *tres, int code); -void taos_error(TAOS *taos); +void shellPrintError(TAOS *taos); static void queryDB(TAOS *taos, char *command) { int i; @@ -102,7 +102,7 @@ int main(int argc, char *argv[]) taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); if (taos == NULL) - taos_error(taos); + shellPrintError(taos); printf("success to connect to server\n"); @@ -193,7 +193,7 @@ int main(int argc, char *argv[]) return 0; } -void taos_error(TAOS *con) +void shellPrintError(TAOS *con) { fprintf(stderr, "TDengine error: %s\n", taos_errstr(con)); taos_close(con); diff --git a/include/client/taos.h b/include/client/taos.h index 6afbcee6f1..72cb7bfa96 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -93,12 +93,12 @@ typedef struct taosField { typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); typedef struct TAOS_BIND_v2 { - int buffer_type; - void *buffer; - int32_t buffer_length; - int32_t *length; - char *is_null; - int num; + int buffer_type; + void *buffer; + int32_t buffer_length; + int32_t *length; + char *is_null; + int num; } TAOS_BIND_v2; typedef enum { @@ -128,35 +128,35 @@ DLL_EXPORT void taos_close(TAOS *taos); const char *taos_data_type(int type); -DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); -DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags); -DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); +DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); -DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); +DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); +DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); -DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result -DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); @@ -234,7 +234,7 @@ DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errst DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ -DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); +DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index fa203e231a..71e2e4fba3 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -74,8 +74,8 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, } char *data = colDataGetVarData(pColumnInfoData, row); return (*data == TSDB_DATA_TYPE_NULL); - } - + } + if (!pColumnInfoData->hasNull) { return false; } @@ -238,10 +238,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { - int32_t* colSizes = (int32_t*)data; + int32_t* actualLen = (int32_t*) data; + data += sizeof(int32_t); + uint64_t* groupId = (uint64_t*) data; + data += sizeof(uint64_t); + + int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); - *dataLen = (numOfCols * sizeof(int32_t)); + + *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); int32_t numOfRows = pBlock->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { @@ -273,6 +279,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da colSizes[col] = htonl(colSizes[col]); } + + *actualLen = *dataLen; + *groupId = pBlock->info.groupId; } #ifdef __cplusplus diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8a8e3cd223..fc3d575317 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -34,13 +34,9 @@ extern int32_t tsVersion; extern int32_t tsStatusInterval; // common -extern int32_t tsRpcTimer; -extern int32_t tsRpcMaxTime; -extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled extern int32_t tsMaxConnections; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; -extern int32_t tsMaxTmrCtrl; extern int32_t tsCompressMsgSize; extern int32_t tsCompressColData; extern int32_t tsMaxNumOfDistinctResults; @@ -98,9 +94,6 @@ extern bool tsDeadLockKillQuery; extern int32_t tsQueryPolicy; // client -extern int32_t tsMaxWildCardsLen; -extern int32_t tsMaxRegexStringLen; -extern int32_t tsMaxNumOfOrderedResults; extern int32_t tsMinSlidingTime; extern int32_t tsMinIntervalTime; extern int32_t tsMaxStreamComputDelay; @@ -130,9 +123,9 @@ extern SDiskCfg tsDiskCfg[]; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) -int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, - const char *apolloUrl, SArray *pArgs, bool tsc); -int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc); +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, + char *apolloUrl, SArray *pArgs, bool tsc); +int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); void taosCleanupCfg(); void taosCfgDynamicOptions(const char *option, const char *value); void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 757fa8e74b..a21e30c900 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -227,8 +227,16 @@ typedef struct { } SSubmitBlkIter; typedef struct { - int32_t totalLen; - int32_t len; + int32_t totalLen; + int32_t len; + // head of SSubmitBlk + // int64_t uid; // table unique id + // int64_t suid; // stable id + // int32_t sversion; // data schema version + // int32_t dataLen; // data part length, not including the SSubmitBlk head + // int32_t schemaLen; // schema length, if length is 0, no schema exists + // int16_t numOfRows; // total number of rows in current submit block + // head of SSubmitBlk const void* pMsg; } SSubmitMsgIter; @@ -237,6 +245,15 @@ int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); +// TODO: KEEP one suite of iterator API finally. +// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts +// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later +// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter +// int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); +// int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); +// int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); +// STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter); + typedef struct { int32_t index; // index of failed block in submit blocks int32_t vnode; // vnode index of failed block @@ -275,7 +292,6 @@ typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igExists; float xFilesFactor; - int32_t aggregationMethod; int32_t delay; int32_t ttl; int32_t numOfColumns; @@ -1314,30 +1330,34 @@ typedef struct { } SMqConsumerLostMsg; typedef struct { - int32_t topicNum; int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; - SArray* topicNames; // SArray + SArray* topicNames; // SArray } SCMSubscribeReq; static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->cgroup); - for (int32_t i = 0; i < pReq->topicNum; i++) { + int32_t topicNum = taosArrayGetSize(pReq->topicNames); + tlen += taosEncodeFixedI32(buf, topicNum); + + for (int32_t i = 0; i < topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } return tlen; } static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { - buf = taosDecodeFixedI32(buf, &pReq->topicNum); buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->cgroup); - pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); - for (int32_t i = 0; i < pReq->topicNum; i++) { + + int32_t topicNum; + buf = taosDecodeFixedI32(buf, &topicNum); + + pReq->topicNames = taosArrayInit(topicNum, sizeof(void*)); + for (int32_t i = 0; i < topicNum; i++) { char* name; buf = taosDecodeString(buf, &name); taosArrayPush(pReq->topicNames, &name); @@ -1491,10 +1511,8 @@ typedef struct { int32_t delay; int32_t qmsg1Len; int32_t qmsg2Len; - func_id_t* pFuncIds; - char* qmsg1; // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1 - char* qmsg2; // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2 - int8_t nFuncIds; + char* qmsg1; // pAst1:qmsg1:SRetention1 => trigger aggr task1 + char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2 } SRSmaParam; typedef struct SVCreateTbReq { @@ -1969,7 +1987,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; char* qmsg; } SMqRebVgReq; @@ -1984,7 +2001,6 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR tlen += taosEncodeFixedI8(buf, pReq->withTbName); tlen += taosEncodeFixedI8(buf, pReq->withSchema); tlen += taosEncodeFixedI8(buf, pReq->withTag); - tlen += taosEncodeFixedI8(buf, pReq->withTagSchema); if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { tlen += taosEncodeString(buf, pReq->qmsg); } @@ -2001,7 +2017,6 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) buf = taosDecodeFixedI8(buf, &pReq->withTbName); buf = taosDecodeFixedI8(buf, &pReq->withSchema); buf = taosDecodeFixedI8(buf, &pReq->withTag); - buf = taosDecodeFixedI8(buf, &pReq->withTagSchema); if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { buf = taosDecodeString(buf, &pReq->qmsg); } @@ -2051,80 +2066,6 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { return buf; } -typedef struct { - int64_t leftForVer; - int32_t vgId; - int32_t epoch; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; -} SMqCancelConnReq; - -static FORCE_INLINE int32_t tEncodeSMqCancelConnReq(void** buf, const SMqCancelConnReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->leftForVer); - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI32(buf, pReq->epoch); - tlen += taosEncodeFixedI64(buf, pReq->consumerId); - tlen += taosEncodeString(buf, pReq->topicName); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqCancelConnReq(void* buf, SMqCancelConnReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->leftForVer); - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI32(buf, &pReq->epoch); - buf = taosDecodeFixedI64(buf, &pReq->consumerId); - buf = taosDecodeStringTo(buf, pReq->topicName); - return buf; -} - -typedef struct { - int8_t reserved; -} SMqCancelConnRsp; - -typedef struct { - int64_t leftForVer; - int32_t vgId; - int64_t oldConsumerId; - int64_t newConsumerId; - char* topic; -} SMqMVRebReq; - -static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pReq->leftForVer); - tlen += taosEncodeFixedI32(buf, pReq->vgId); - tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); - tlen += taosEncodeString(buf, pReq->topic); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqMVRebReq(void* buf, SMqMVRebReq* pReq) { - buf = taosDecodeFixedI64(buf, &pReq->leftForVer); - buf = taosDecodeFixedI32(buf, &pReq->vgId); - buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); - buf = taosDecodeString(buf, &pReq->topic); - return buf; -} - -typedef struct { - SMsgHead header; - int32_t vgId; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; -} SMqSetCVgRsp; - -typedef struct { - SMsgHead header; - int32_t vgId; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; -} SMqMVRebRsp; - typedef struct { int32_t vgId; int64_t offset; @@ -2151,6 +2092,24 @@ typedef struct { SSchema* pSchema; } SSchemaWrapper; +static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); + if (pSW == NULL) return pSW; + pSW->nCols = pSchemaWrapper->nCols; + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); + if (pSW->pSchema == NULL) { + taosMemoryFree(pSW); + return NULL; + } + memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema)); + return pSW; +} + +static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { + taosMemoryFree(pSchemaWrapper->pSchema); + taosMemoryFree(pSchemaWrapper); +} + static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); @@ -2161,13 +2120,13 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema return tlen; } -static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { +static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) { buf = taosDecodeFixedI8(buf, &pSchema->type); buf = taosDecodeFixedI8(buf, &pSchema->flags); buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeStringTo(buf, pSchema->name); - return buf; + return (void*)buf; } static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { @@ -2197,7 +2156,7 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr return tlen; } -static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { +static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->nCols); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { @@ -2207,7 +2166,7 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pS for (int32_t i = 0; i < pSW->nCols; i++) { buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); } - return buf; + return (void*)buf; } static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) { @@ -2590,7 +2549,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; SArray* blockDataLen; // SArray SArray* blockData; // SArray SArray* blockTbName; // SArray @@ -2609,13 +2567,16 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp tlen += taosEncodeFixedI8(buf, pRsp->withTbName); tlen += taosEncodeFixedI8(buf, pRsp->withSchema); tlen += taosEncodeFixedI8(buf, pRsp->withTag); - tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema); for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); void* data = taosArrayGetP(pRsp->blockData, i); tlen += taosEncodeFixedI32(buf, bLen); tlen += taosEncodeBinary(buf, data, bLen); + if (pRsp->withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); + tlen += taosEncodeSSchemaWrapper(buf, pSW); + } } } return tlen; @@ -2628,11 +2589,11 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeFixedI32(buf, &pRsp->blockNum); pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); + pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); if (pRsp->blockNum != 0) { buf = taosDecodeFixedI8(buf, &pRsp->withTbName); buf = taosDecodeFixedI8(buf, &pRsp->withSchema); buf = taosDecodeFixedI8(buf, &pRsp->withTag); - buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema); for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = 0; @@ -2641,6 +2602,11 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeBinary(buf, &data, bLen); taosArrayPush(pRsp->blockDataLen, &bLen); taosArrayPush(pRsp->blockData, &data); + if (pRsp->withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); + buf = taosDecodeSSchemaWrapper(buf, pSW); + taosArrayPush(pRsp->blockSchema, &pSW); + } } } return (void*)buf; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index b37001bde9..97ee66a2da 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -85,6 +85,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL) // Requests handled by MNODE TD_NEW_MSG_SEG(TDMT_MND_MSG) @@ -144,7 +145,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) + TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-mq-ask-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b88afcb39d..30d1bd0a51 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -51,7 +51,7 @@ typedef struct SMetaData { SArray *pTableMeta; // STableMeta array SArray *pVgroupInfo; // SVgroupInfo list SArray *pUdfList; // udf info list - SArray *pEpSetList; // qnode epset list, SArray + SArray *pQnodeList; // qnode list, SArray } SMetaData; typedef struct SCatalogCfg { diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 004d834287..ef32533e5f 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -37,7 +37,7 @@ typedef struct SFuncExecEnv { typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); -typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId); +typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef struct SScalarFuncExecFuncs { @@ -141,8 +141,7 @@ struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SSubsidiaryResInfo { - int16_t bufLen; // keep the tags data for top/bottom query result - int16_t numOfCols; + int16_t num; struct SqlFunctionCtx **pCtx; } SSubsidiaryResInfo; @@ -187,8 +186,8 @@ typedef struct SqlFunctionCtx { uint8_t currentStage; // record current running step, default: 0 bool isAggSet; int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it - ///////////////////////////////////////////////////////////////// bool stableQuery; + ///////////////////////////////////////////////////////////////// int16_t functionId; // function id char * pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; @@ -198,11 +197,15 @@ typedef struct SqlFunctionCtx { int32_t offset; SVariant tag; struct SResultRowEntryInfo *resultInfo; - SSubsidiaryResInfo subsidiaryRes; - SPoint1 start; - SPoint1 end; - SFuncExecFuncs fpSet; - SScalarFuncExecFuncs sfp; + SSubsidiaryResInfo subsidiaries; + SPoint1 start; + SPoint1 end; + SFuncExecFuncs fpSet; + SScalarFuncExecFuncs sfp; + SExprInfo *pExpr; + struct SDiskbasedBuf *pBuf; + struct SSDataBlock *pSrcBlock; + int32_t curBufPage; } SqlFunctionCtx; enum { @@ -319,6 +322,20 @@ struct SUdfInfo; void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); +/** + * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf + * @return error code + */ +int32_t udfcOpen(); + +/** + * destroy udfd proxy + * @return error code + */ +int32_t udfcClose(); + +typedef void *UdfcFuncHandle; + #ifdef __cplusplus } #endif diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 2fff819f54..56e25d49c1 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -125,15 +125,15 @@ typedef struct SFmGetFuncInfoParam { struct SCatalog* pCtg; void *pRpc; const SEpSet* pMgmtEps; + char* pErrBuf; + int32_t errBufLen; } SFmGetFuncInfoParam; int32_t fmFuncMgtInit(); void fmFuncMgtDestroy(); -int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); - -int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len); +int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc); bool fmIsAggFunc(int32_t funcId); bool fmIsScalarFunc(int32_t funcId); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b4a290cbfc..829770ed1b 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -46,7 +46,7 @@ typedef struct SScanLogicNode { struct STableMeta* pMeta; SVgroupsInfo* pVgroupList; EScanType scanType; - uint8_t scanFlag; // denotes reversed scan of data or not + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count STimeWindow scanRange; SName tableName; bool showRewrite; @@ -189,9 +189,6 @@ typedef struct SScanPhysiNode { SNodeList* pScanCols; uint64_t uid; // unique id of the table int8_t tableType; - int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC - int32_t count; // repeat count - int32_t reverse; // reverse scan count SName tableName; } SScanPhysiNode; @@ -207,7 +204,7 @@ typedef struct SSystemTableScanPhysiNode { typedef struct STableScanPhysiNode { SScanPhysiNode scan; - uint8_t scanFlag; // denotes reversed scan of data or not + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count STimeWindow scanRange; double ratio; int32_t dataRequired; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index d8e2354e8e..c71fb266b1 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -46,7 +46,7 @@ typedef struct SExprNode { ENodeType type; SDataType resType; char aliasName[TSDB_COL_NAME_LEN]; - SNodeList* pAssociationList; + SArray* pAssociation; } SExprNode; typedef enum EColumnType { @@ -81,7 +81,6 @@ typedef struct SValueNode { char* literal; bool isDuration; bool translate; - bool genByCalc; int16_t placeholderNo; union { bool b; @@ -119,6 +118,7 @@ typedef struct SFunctionNode { int32_t funcId; int32_t funcType; SNodeList* pParameterList; + int32_t udfBufSize; } SFunctionNode; typedef struct STableNode { @@ -135,7 +135,7 @@ typedef struct SRealTableNode { STableNode table; // QUERY_NODE_REAL_TABLE struct STableMeta* pMeta; SVgroupsInfo* pVgroupList; - char useDbName[TSDB_DB_NAME_LEN]; + char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES double ratio; } SRealTableNode; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 2f6b9e1866..78d0911502 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -37,6 +37,8 @@ typedef struct SPlanContext { bool isStmtQuery; void* pTransporter; struct SCatalog* pCatalog; + char* pMsg; + int32_t msgLen; } SPlanContext; // Create the physical plan for the query, according to the AST. diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 65d3bd0171..7b9f68103d 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -38,11 +38,12 @@ typedef struct SRpcConnInfo { typedef struct SRpcMsg { tmsg_t msgType; - void *pCont; + void * pCont; int contLen; int32_t code; - void *handle; // rpc handle returned to app - void *ahandle; // app handle set by client + void * handle; // rpc handle returned to app + void * ahandle; // app handle set by client + int64_t refId; // refid, used by server int noResp; // has response or not(default 0, 0: resp, 1: no resp); int persistHandle; // persist handle or not @@ -54,8 +55,8 @@ typedef struct { uint16_t clientPort; SRpcMsg rpcMsg; int32_t rspLen; - void *pRsp; - void *pNode; + void * pRsp; + void * pNode; } SNodeMsg; typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); @@ -64,7 +65,7 @@ typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *); typedef struct SRpcInit { uint16_t localPort; // local port - char *label; // for debug purpose + char * label; // for debug purpose int numOfThreads; // number of threads to handle connections int sessions; // number of sessions allowed int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS @@ -97,23 +98,23 @@ typedef struct { typedef struct { int32_t msgType; - void *val; + void * val; int32_t (*clone)(void *src, void **dst); void (*freeFunc)(const void *arg); } SRpcBrokenlinkVal; typedef struct { - SHashObj *args; + SHashObj * args; SRpcBrokenlinkVal brokenVal; } SRpcCtx; int32_t rpcInit(); void rpcCleanup(); -void *rpcOpen(const SRpcInit *pRpc); +void * rpcOpen(const SRpcInit *pRpc); void rpcClose(void *); -void *rpcMallocCont(int contLen); +void * rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); -void *rpcReallocCont(void *ptr, int contLen); +void * rpcReallocCont(void *ptr, int contLen); // Because taosd supports multi-process mode // These functions should not be used on the server side diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index d41e797536..7b0d70b769 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -192,7 +192,13 @@ int32_t walEndSnapshot(SWal *); SWalReadHandle *walOpenReadHandle(SWal *); void walCloseReadHandle(SWalReadHandle *); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); -int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead); + +// only for tq usage +// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead); +void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity); +int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead); +int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead); +int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead); // deprecated #if 0 diff --git a/include/os/osSystem.h b/include/os/osSystem.h index 33b0a46ee9..6770be6e46 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -31,19 +31,19 @@ extern "C" { typedef struct TdCmd *TdCmdPtr; -TdCmdPtr taosOpenCmd(const char *cmd); -int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf); -int32_t taosEOFCmd(TdCmdPtr pCmd); -int64_t taosCloseCmd(TdCmdPtr *ppCmd); +TdCmdPtr taosOpenCmd(const char* cmd); +int64_t taosGetLineCmd(TdCmdPtr pCmd, char** __restrict ptrBuf); +int32_t taosEOFCmd(TdCmdPtr pCmd); +int64_t taosCloseCmd(TdCmdPtr* ppCmd); void* taosLoadDll(const char* filename); void* taosLoadSym(void* handle, char* name); void taosCloseDll(void* handle); int32_t taosSetConsoleEcho(bool on); -void setTerminalMode(); -int32_t getOldTerminalMode(); -void resetTerminalMode(); +void taosSetTerminalMode(); +int32_t taosGetOldTerminalMode(); +void taosResetTerminalMode(); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 03fb43a46b..f0c3cc4b14 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -252,7 +252,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) // mnode-infoSchema -#define TSDB_CODE_MND_INVALID_INFOS_TBL TAOS_DEF_ERROR_CODE(0, 0x03B0) +#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x03B0) // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) @@ -573,7 +573,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) #define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) -#define TSDB_CODE_PAR_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2605) #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) @@ -623,6 +622,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INVALID_DAYS_VALUE TAOS_DEF_ERROR_CODE(0, 0x2636) #define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637) #define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638) +#define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) @@ -632,6 +632,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801) #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) +#define TSDB_CODE_FUNC_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2604) #ifdef __cplusplus } diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 012368d9cb..06fa9fd9aa 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -30,6 +30,7 @@ typedef enum { CFG_STYPE_CFG_FILE, CFG_STYPE_ENV_FILE, CFG_STYPE_ENV_VAR, + CFG_STYPE_ENV_CMD, CFG_STYPE_APOLLO_URL, CFG_STYPE_ARG_LIST, CFG_STYPE_TAOS_OPTIONS @@ -82,7 +83,7 @@ typedef struct SConfig { } SConfig; SConfig *cfgInit(); -int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr); int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair void cfgCleanup(SConfig *pCfg); @@ -105,6 +106,8 @@ const char *cfgDtypeStr(ECfgDataType type); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); +int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl); + #ifdef __cplusplus } #endif diff --git a/include/util/tdef.h b/include/util/tdef.h index 31fbe42b6c..aa0d0bd8ff 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -76,8 +76,6 @@ extern const int32_t TYPE_BYTES[15]; #define TSDB_DEFAULT_PASS "taosdata" #endif -#define SHELL_MAX_PASSWORD_LEN 20 - #define TSDB_TRUE 1 #define TSDB_FALSE 0 #define TSDB_OK 0 diff --git a/tools/shell/src/backup/tnettest.h b/include/util/tenv.h similarity index 75% rename from tools/shell/src/backup/tnettest.h rename to include/util/tenv.h index 8a03b67628..1410e23f90 100644 --- a/tools/shell/src/backup/tnettest.h +++ b/include/util/tenv.h @@ -1,3 +1,4 @@ + /* * Copyright (c) 2019 TAOS Data, Inc. * @@ -13,17 +14,20 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TNETTEST_H -#define TDENGINE_TNETTEST_H +#ifndef _TD_ENV_H_ +#define _TD_ENV_H_ + +#include "os.h" #ifdef __cplusplus extern "C" { #endif -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType); +int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t cfgNameMaxLen); +int32_t taosEnvToCfg(const char *envStr, char *cfgStr); #ifdef __cplusplus } #endif -#endif // TDENGINE_TNETTEST_H +#endif /*_TD_ENV_H_*/ diff --git a/include/util/tjson.h b/include/util/tjson.h index 286a7e65fc..d23f7b402e 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -49,6 +49,8 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem); int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem); SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName); +int32_t tjsonGetObjectName(const SJson* pJson, char** pName); +int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue); int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal); int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal); int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal); @@ -81,6 +83,7 @@ char* tjsonToUnformattedString(const SJson* pJson); SJson* tjsonParse(const char* pStr); bool tjsonValidateJson(const char* pJson); +const char* tjsonGetError(); #ifdef __cplusplus } diff --git a/include/util/tprocess.h b/include/util/tprocess.h index 2b0fd89aa5..7e1767441e 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -53,8 +53,8 @@ int32_t taosProcRun(SProcObj *pProc); void taosProcStop(SProcObj *pProc); int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - void *handle, EProcFuncType ftype); -void taosProcRemoveHandle(SProcObj *pProc, void *handle); + void *handle, int64_t handleRef, EProcFuncType ftype); +int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle); void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)); void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, EProcFuncType ftype); diff --git a/include/util/ttimer.h b/include/util/ttimer.h index f2ee825c4e..1022259631 100644 --- a/include/util/ttimer.h +++ b/include/util/ttimer.h @@ -41,10 +41,6 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *han void taosTmrCleanUp(void *handle); -int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms); - -void taosUninitTimer(); - #ifdef __cplusplus } #endif diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 814caf330a..c0237a184a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -231,6 +231,10 @@ static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool conver msg->resIter++; if (msg->resIter < msg->rsp.blockNum) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter); + if (msg->rsp.withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter); + setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols); + } setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4); return &msg->resInfo; } diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9a34334466..596c3d3fbf 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -254,12 +254,12 @@ void taos_init_imp(void) { deltaToUtcInitOnce(); - if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, 1) != 0) { + if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { tscInitRes = -1; return; } - if (taosInitCfg(configDir, NULL, NULL, NULL, 1) != 0) { + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) { tscInitRes = -1; return; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6159da9cb1..b11a49fa1a 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -14,9 +14,9 @@ */ #include "catalog.h" -#include "scheduler.h" #include "clientInt.h" #include "clientLog.h" +#include "scheduler.h" #include "trpc.h" static SClientHbMgr clientHbMgr = {0}; @@ -110,7 +110,8 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); if (NULL == info) { - tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, pRsp->connKey.connType); + tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, + pRsp->connKey.connType); return TSDB_CODE_SUCCESS; } @@ -121,7 +122,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { } else { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pRsp->query->epSet); pTscObj->connId = pRsp->query->connId; - + if (pRsp->query->killRid) { SRequestObj *pRequest = acquireRequest(pRsp->query->killRid); if (NULL == pRequest) { @@ -131,7 +132,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { releaseRequest(pRsp->query->killRid); } } - + if (pRsp->query->killConnection) { taos_close(pTscObj); } @@ -139,7 +140,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { releaseTscObj(pRsp->connKey.tscRid); } } - + int32_t kvNum = pRsp->info ? taosArrayGetSize(pRsp->info) : 0; tscDebug("hb got %d rsp kv", kvNum); @@ -236,24 +237,24 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) } int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { - int64_t now = taosGetTimestampUs(); + int64_t now = taosGetTimestampUs(); SQueryDesc desc = {0}; - int32_t code = 0; + int32_t code = 0; - void *pIter = taosHashIterate(pObj->pRequests, NULL); + void *pIter = taosHashIterate(pObj->pRequests, NULL); while (pIter != NULL) { - int64_t *rid = pIter; + int64_t *rid = pIter; SRequestObj *pRequest = acquireRequest(*rid); if (NULL == pRequest) { continue; } tstrncpy(desc.sql, pRequest->sqlstr, sizeof(desc.sql)); - desc.stime = pRequest->metric.start; - desc.queryId = pRequest->requestId; + desc.stime = pRequest->metric.start; + desc.queryId = pRequest->requestId; desc.useconds = now - pRequest->metric.start; - desc.reqRid = pRequest->self; - desc.pid = hbBasic->pid; + desc.reqRid = pRequest->self; + desc.pid = hbBasic->pid; taosGetFqdn(desc.fqdn); desc.subPlanNum = pRequest->body.pDag ? pRequest->body.pDag->numOfSubplans : 0; @@ -271,9 +272,9 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { } } - releaseRequest(*rid); + releaseRequest(*rid); taosArrayPush(hbBasic->queryDesc, &desc); - + pIter = taosHashIterate(pObj->pRequests, pIter); } @@ -286,14 +287,14 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); return TSDB_CODE_QRY_APP_ERROR; } - + int32_t numOfQueries = pTscObj->pRequests ? taosHashGetSize(pTscObj->pRequests) : 0; if (numOfQueries <= 0) { releaseTscObj(connKey->tscRid); tscDebug("no queries on connection"); return TSDB_CODE_QRY_APP_ERROR; } - + SQueryHbReqBasic *hbBasic = (SQueryHbReqBasic *)taosMemoryCalloc(1, sizeof(SQueryHbReqBasic)); if (NULL == hbBasic) { tscError("calloc %d failed", (int32_t)sizeof(SQueryHbReqBasic)); @@ -308,7 +309,7 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { taosMemoryFree(hbBasic); return TSDB_CODE_QRY_OUT_OF_MEMORY; } - + hbBasic->connId = pTscObj->connId; hbBasic->pid = taosGetPId(); taosGetAppName(hbBasic->app, NULL); @@ -405,7 +406,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req } hbGetQueryBasicInfo(connKey, req); - + code = hbGetExpiredDBInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; @@ -471,10 +472,10 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); } -// if (code) { -// taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq); -// taosMemoryFreeClear(pBatchReq); -// } + // if (code) { + // taosArrayDestroyEx(pBatchReq->reqs, hbFreeReq); + // taosMemoryFreeClear(pBatchReq); + // } return pBatchReq; } @@ -630,24 +631,23 @@ void appHbMgrCleanup(void) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); - - void *pIter = taosHashIterate(pTarget->activeInfo, NULL); + + void *pIter = taosHashIterate(pTarget->activeInfo, NULL); while (pIter != NULL) { SClientHbReq *pOneReq = pIter; hbFreeReq(pOneReq); taosHashCleanup(pOneReq->info); pIter = taosHashIterate(pTarget->activeInfo, pIter); - } + } taosHashCleanup(pTarget->activeInfo); pTarget->activeInfo = NULL; - pIter = taosHashIterate(pTarget->connInfo, NULL); while (pIter != NULL) { SHbConnInfo *info = pIter; taosMemoryFree(info->param); pIter = taosHashIterate(pTarget->connInfo, pIter); - } + } taosHashCleanup(pTarget->connInfo); pTarget->connInfo = NULL; @@ -668,13 +668,13 @@ int hbMgrInit() { hbMgrInitHandle(); // init backgroud thread - //hbCreateThread(); + /*hbCreateThread();*/ return 0; } void hbMgrCleanUp() { - //hbStopThread(); + // hbStopThread(); // destroy all appHbMgr int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 1, 0); @@ -747,11 +747,11 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { taosMemoryFree(info->param); taosHashRemove(pAppHbMgr->connInfo, &connKey, sizeof(SClientHbKey)); } - + if (NULL == pReq || NULL == info) { return; } - + atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fc4192d818..a40edbb65a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -105,9 +105,9 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, epSet.epSet.eps[0].port = port; } - char* key = getClusterKey(user, secretEncrypt, ip, port); - SAppInstInfo** pInst = NULL; + char* key = getClusterKey(user, secretEncrypt, ip, port); + SAppInstInfo** pInst = NULL; taosThreadMutexLock(&appInfo.mutex); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); @@ -226,15 +226,15 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) { pRequest->type = pQuery->msgType; - SPlanContext cxt = { - .queryId = pRequest->requestId, - .acctId = pRequest->pTscObj->acctId, - .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), - .pAstRoot = pQuery->pRoot, - .showRewrite = pQuery->showRewrite, - .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter - }; - int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog); + SPlanContext cxt = {.queryId = pRequest->requestId, + .acctId = pRequest->pTscObj->acctId, + .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), + .pAstRoot = pQuery->pRoot, + .showRewrite = pQuery->showRewrite, + .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter, + .pMsg = pRequest->msgBuf, + .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; + int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog); if (TSDB_CODE_SUCCESS == code) { code = qCreateQueryPlan(&cxt, pPlan, pNodeList); } @@ -245,6 +245,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t ASSERT(pSchema != NULL && numOfCols > 0); pResInfo->numOfCols = numOfCols; + // TODO handle memory leak pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); @@ -280,7 +281,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, - pRequest->metric.start, &res); + pRequest->metric.start, &res); if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); @@ -838,10 +839,21 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 return code; } - int32_t* colLength = (int32_t*)pResultInfo->pData; - char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols; + char* p = (char*)pResultInfo->pData; + + int32_t dataLen = *(int32_t*)p; + p += sizeof(int32_t); + + uint64_t groupId = *(uint64_t*)p; + p += sizeof(uint64_t); + + int32_t* colLength = (int32_t*)p; + p += sizeof(int32_t) * numOfCols; + + char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { colLength[i] = htonl(colLength[i]); + ASSERT(colLength[i] < dataLen); if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { pResultInfo->pCol[i].offset = (int32_t*)pStart; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 27efcee76e..a9efcced40 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -120,7 +120,7 @@ const char *taos_errstr(TAOS_RES *res) { return (const char *)tstrerror(terrno); } - if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { + if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { return (const char *)tstrerror(pRequest->code); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index a33351373e..8096ce395a 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tdef.h" -#include "tname.h" +#include "catalog.h" #include "clientInt.h" #include "clientLog.h" -#include "catalog.h" +#include "os.h" #include "query.h" +#include "tdef.h" +#include "tname.h" int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); @@ -50,7 +50,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SConnectRsp connectRsp = {0}; tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp); - assert(connectRsp.epSet.numOfEps > 0); + /*assert(connectRsp.epSet.numOfEps > 0);*/ + if (connectRsp.epSet.numOfEps == 0) { + taosMemoryFree(pMsg->pData); + setErrno(pRequest, TSDB_CODE_MND_APP_ERROR); + tsem_post(&pRequest->body.rspSem); + return code; + } if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet); @@ -82,18 +88,20 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { +SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); pMsgSendInfo->requestObjRefId = pRequest->self; - pMsgSendInfo->requestId = pRequest->requestId; - pMsgSendInfo->param = pRequest; - pMsgSendInfo->msgType = pRequest->type; + pMsgSendInfo->requestId = pRequest->requestId; + pMsgSendInfo->param = pRequest; + pMsgSendInfo->msgType = pRequest->type; assert(pRequest != NULL); pMsgSendInfo->msgInfo = pRequest->body.requestMsg; - pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; + pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL) + ? genericRspCallback + : handleRequestRspFp[TMSG_INDEX(pRequest->type)]; return pMsgSendInfo; } @@ -114,7 +122,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (TSDB_CODE_MND_DB_NOT_EXIST == code) { SUseDbRsp usedbRsp = {0}; tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); - struct SCatalog *pCatalog = NULL; + struct SCatalog* pCatalog = NULL; if (usedbRsp.vgVersion >= 0) { int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 699fa7ecf5..f9540bc8fc 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -23,6 +23,9 @@ #include "tmsgtype.h" #include "tqueue.h" #include "tref.h" +#include "ttimer.h" + +int32_t tmqAskEp(tmq_t* tmq, bool sync); typedef struct { int8_t tmqRspType; @@ -61,29 +64,40 @@ struct tmq_conf_t { tmq_commit_cb* commit_cb; }; +typedef struct { + int8_t inited; + tmr_h timer; +} SMqMgmt; + +static SMqMgmt tmqMgmt = {0}; + struct tmq_t { // conf - char groupId[TSDB_CGROUP_LEN]; - char clientId[256]; - int8_t autoCommit; - /*int8_t inWaiting;*/ + char groupId[TSDB_CGROUP_LEN]; + char clientId[256]; + int8_t autoCommit; int64_t consumerId; - int32_t epoch; int32_t resetOffsetCfg; - int64_t status; - STscObj* pTscObj; tmq_commit_cb* commit_cb; - /*int32_t nextTopicIdx;*/ + + // status + int8_t status; int8_t epStatus; + int32_t epoch; int32_t epSkipCnt; - /*int32_t waitingRequest;*/ - /*int32_t readyRequest;*/ - SArray* clientTopics; // SArray - STaosQueue* mqueue; // queue of tmq_message_t - STaosQall* qall; - tsem_t rspSem; - // stat int64_t pollCnt; + + // connection + STscObj* pTscObj; + + // container + SArray* clientTopics; // SArray + STaosQueue* mqueue; // queue of rsp + STaosQall* qall; + STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit + + // ctl + tsem_t rspSem; }; enum { @@ -93,6 +107,7 @@ enum { enum { TMQ_CONSUMER_STATUS__INIT = 0, + TMQ_CONSUMER_STATUS__SUBSCRIBED, TMQ_CONSUMER_STATUS__READY, }; @@ -110,13 +125,11 @@ typedef struct { typedef struct { // subscribe info - int32_t sqlLen; - char* sql; - char* topicName; - int64_t topicId; - SArray* vgs; // SArray + char* topicName; + + SArray* vgs; // SArray + int8_t isSchemaAdaptive; - int32_t numOfFields; SSchemaWrapper schema; } SMqClientTopic; @@ -156,7 +169,6 @@ typedef struct { int32_t async; tsem_t rspSem; tmq_resp_err_t rspErr; - /*SMqClientVg* pVg;*/ } SMqCommitCbParam; tmq_conf_t* tmq_conf_new() { @@ -251,13 +263,7 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { void tmq_list_destroy(tmq_list_t* list) { SArray* container = &list->container; - /*taosArrayDestroy(container);*/ - int32_t sz = taosArrayGetSize(container); - for (int32_t i = 0; i < sz; i++) { - char* str = taosArrayGetP(container, i); - taosMemoryFree(str); - } - taosArrayDestroy(container); + taosArrayDestroyP(container, taosMemoryFree); } int32_t tmq_list_get_size(const tmq_list_t* list) { @@ -298,6 +304,8 @@ void tmqClearUnhandleMsg(tmq_t* tmq) { int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; + tmq_t* tmq = pParam->tmq; + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__SUBSCRIBED); tsem_post(&pParam->rspSem); return 0; } @@ -335,12 +343,9 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs return NULL; } pTmq->pTscObj = (STscObj*)conn; - /*pTmq->inWaiting = 0;*/ pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; - /*pTmq->waitingRequest = 0;*/ - /*pTmq->readyRequest = 0;*/ pTmq->epStatus = 0; pTmq->epSkipCnt = 0; // set conf @@ -367,29 +372,45 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs #endif tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { + // init timer + int8_t inited = atomic_val_compare_exchange_8(&tmqMgmt.inited, 0, 1); + if (inited == 0) { + tmqMgmt.timer = taosTmrInit(1000, 100, 360000, "TMQ"); + if (tmqMgmt.timer == NULL) { + atomic_store_8(&tmqMgmt.inited, 0); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + } + tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); if (pTmq == NULL) { return NULL; } + const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user; const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; ASSERT(user); ASSERT(pass); - ASSERT(conf->db); ASSERT(conf->groupId[0]); - pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ); - if (pTmq->pTscObj == NULL) return NULL; + pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); + pTmq->mqueue = taosOpenQueue(); + pTmq->qall = taosAllocateQall(); + pTmq->delayedTask = taosOpenQueue(); - /*pTmq->inWaiting = 0;*/ - pTmq->status = 0; + if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL) { + goto FAIL; + } + + // init status + pTmq->status = TMQ_CONSUMER_STATUS__INIT; pTmq->pollCnt = 0; pTmq->epoch = 0; - /*pTmq->waitingRequest = 0;*/ - /*pTmq->readyRequest = 0;*/ pTmq->epStatus = 0; pTmq->epSkipCnt = 0; + // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); @@ -397,19 +418,30 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->commit_cb = conf->commit_cb; pTmq->resetOffsetCfg = conf->resetOffset; + // assign consumerId pTmq->consumerId = tGenIdPI64(); - pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); - if (pTmq->clientTopics == NULL) { - taosMemoryFree(pTmq); - return NULL; + + // init semaphore + if (tsem_init(&pTmq->rspSem, 0, 0) != 0) { + goto FAIL; } - pTmq->mqueue = taosOpenQueue(); - pTmq->qall = taosAllocateQall(); - - tsem_init(&pTmq->rspSem, 0, 0); + // init connection + pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ); + if (pTmq->pTscObj == NULL) { + tsem_destroy(&pTmq->rspSem); + goto FAIL; + } return pTmq; + +FAIL: + if (pTmq->clientTopics) taosArrayDestroy(pTmq->clientTopics); + if (pTmq->mqueue) taosCloseQueue(pTmq->mqueue); + if (pTmq->delayedTask) taosCloseQueue(pTmq->delayedTask); + if (pTmq->qall) taosFreeQall(pTmq->qall); + taosMemoryFree(pTmq); + return NULL; } tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { @@ -500,81 +532,64 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in return resp; } -tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { - SRequestObj* pRequest = NULL; - SArray* container = &topic_list->container; - int32_t sz = taosArrayGetSize(container); - // destroy ex - taosArrayDestroy(tmq->clientTopics); - tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); +tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { + const SArray* container = &topic_list->container; + int32_t sz = taosArrayGetSize(container); + void* buf = NULL; + SCMSubscribeReq req = {0}; + int32_t code = -1; - SCMSubscribeReq req; - req.topicNum = sz; req.consumerId = tmq->consumerId; - strcpy(req.cgroup, tmq->groupId); + tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); + if (req.topicNames == NULL) goto FAIL; - for (int i = 0; i < sz; i++) { - /*char* topicName = topic_list->elems[i];*/ - char* topicName = taosArrayGetP(container, i); + for (int32_t i = 0; i < sz; i++) { + char* topic = taosArrayGetP(container, i); SName name = {0}; - char* dbName = getDbOfConnection(tmq->pTscObj); - if (dbName == NULL) { - return TMQ_RESP_ERR__FAIL; - } - tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); - tNameFromString(&name, topicName, T_NAME_TABLE); + tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic)); - char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); - if (topicFname == NULL) { - goto _return; + char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); + if (topicFName == NULL) { + goto FAIL; } - tNameExtractFullName(&name, topicFname); - tscDebug("subscribe topic: %s", topicFname); - SMqClientTopic topic = { - .sql = NULL, - .sqlLen = 0, - .topicId = 0, - .topicName = topicFname, - .vgs = NULL, - }; - topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); - taosArrayPush(tmq->clientTopics, &topic); - taosArrayPush(req.topicNames, &topicFname); - taosMemoryFree(dbName); + tNameExtractFullName(&name, topicFName); + + tscDebug("subscribe topic: %s", topicFName); + + taosArrayPush(req.topicNames, &topicFName); } - int tlen = tSerializeSCMSubscribeReq(NULL, &req); - void* buf = taosMemoryMalloc(tlen); - if (buf == NULL) { - goto _return; - } + int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req); + buf = taosMemoryMalloc(tlen); + if (buf == NULL) goto FAIL; void* abuf = buf; tSerializeSCMSubscribeReq(&abuf, &req); - /*printf("formatted: %s\n", dagStr);*/ - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); - if (pRequest == NULL) { - tscError("failed to malloc request"); - } + SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); + if (sendInfo == NULL) goto FAIL; SMqSubscribeCbParam param = { .rspErr = TMQ_RESP_ERR__SUCCESS, .tmq = tmq, }; - tsem_init(¶m.rspSem, 0, 0); - pRequest->body.requestMsg = (SDataBuf){ + if (tsem_init(¶m.rspSem, 0, 0) != 0) goto FAIL; + + sendInfo->msgInfo = (SDataBuf){ .pData = buf, .len = tlen, .handle = NULL, }; - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; sendInfo->param = ¶m; sendInfo->fp = tmqSubscribeCb; + sendInfo->msgType = TDMT_MND_SUBSCRIBE; + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int64_t transporterId = 0; @@ -583,15 +598,28 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tsem_wait(¶m.rspSem); tsem_destroy(¶m.rspSem); -_return: - /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ - /*}*/ + code = param.rspErr; + if (code != 0) goto FAIL; - return param.rspErr; + // TODO: add max retry cnt + while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, true)) { + tscDebug("not ready, retry\n"); + taosMsleep(500); + } + + code = 0; +FAIL: + if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree); + if (code != 0) { + taosMemoryFree(buf); + } + return code; } -void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { + // + conf->commit_cb = cb; +} TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) { STscObj* pTscObj = (STscObj*)taos; @@ -625,9 +653,6 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa int32_t code = 0; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return); - - // todo check for invalid sql statement and return with error code - CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); /*printf("%s\n", pStr);*/ @@ -651,7 +676,6 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa } tSerializeSCMCreateStreamReq(buf, tlen, &req); - /*printf("formatted: %s\n", dagStr);*/ pRequest->body.requestMsg = (SDataBuf){ .pData = buf, @@ -682,94 +706,6 @@ _return: return pRequest; } -#if 0 -TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { - STscObj* pTscObj = (STscObj*)taos; - SRequestObj* pRequest = NULL; - SQuery* pQueryNode = NULL; - char* astStr = NULL; - - terrno = TSDB_CODE_SUCCESS; - if (taos == NULL || topicName == NULL || sql == NULL) { - tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) { - tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1); - terrno = TSDB_CODE_TSC_INVALID_INPUT; - goto _return; - } - - if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) { - tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); - terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - goto _return; - } - - tscDebug("start to create topic: %s", topicName); - - int32_t code = TSDB_CODE_SUCCESS; - CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); - CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return); - - // todo check for invalid sql statement and return with error code - - CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); - - /*printf("%s\n", pStr);*/ - - SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T}; - strcpy(name.dbname, pRequest->pDb); - strcpy(name.tname, topicName); - - SCMCreateTopicReq req = { - .igExists = 1, - .ast = astStr, - .sql = (char*)sql, - }; - tNameExtractFullName(&name, req.name); - - int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req); - void* buf = taosMemoryMalloc(tlen); - if (buf == NULL) { - goto _return; - } - - tSerializeSCMCreateTopicReq(buf, tlen, &req); - /*printf("formatted: %s\n", dagStr);*/ - - pRequest->body.requestMsg = (SDataBuf){ - .pData = buf, - .len = tlen, - .handle = NULL, - }; - pRequest->type = TDMT_MND_CREATE_TOPIC; - - SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); - SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - - int64_t transporterId = 0; - asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - - tsem_wait(&pRequest->body.rspSem); - -_return: - taosMemoryFreeClear(astStr); - qDestroyQuery(pQueryNode); - /*if (sendInfo != NULL) {*/ - /*destroySendMsgInfo(sendInfo);*/ - /*}*/ - - if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) { - pRequest->code = terrno; - } - - return pRequest; -} -#endif - #if 0 int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; @@ -952,7 +888,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ if (tmqUpdateEp(tmq, head->epoch, &rsp)) { - atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY); + atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); } tDeleteSMqCMGetSubEpRsp(&rsp); } else { @@ -1116,7 +1052,9 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); + if (!pWrapper->msg.withSchema) { + setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); + } taosFreeQitem(pWrapper); return pRspObj; @@ -1185,7 +1123,6 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int64_t transporterId = 0; /*printf("send poll\n");*/ - /*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/ tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ @@ -1264,12 +1201,14 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { SMqRspObj* rspObj; int64_t startTime = taosGetTimestampMs(); - // TODO: put into another thread or delayed queue - int64_t status = atomic_load_64(&tmq->status); - while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) { + // TODO: put into delayed queue +#if 0 + int8_t status = atomic_load_8(&tmq->status); + while (0 != tmqAskEp(tmq, status != TMQ_CONSUMER_STATUS__READY)) { tscDebug("not ready, retry\n"); taosSsleep(1); } +#endif rspObj = tmqHandleAllRsp(tmq, blocking_time, false); if (rspObj) { @@ -1277,7 +1216,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { } while (1) { - /*printf("cycle\n");*/ tmqAskEp(tmq, false); tmqPollImpl(tmq, blocking_time); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 800bcd70ce..09452b584d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -644,12 +644,12 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { /** * @refitem blockDataToBuf for the meta size - * * @param pBlock * @return */ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { - return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t); + // | total rows/total length | block group id | each column length | + return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -1219,7 +1219,6 @@ void colDataDestroy(SColumnInfoData* pColData) { taosMemoryFree(pColData->pData); } - static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { int32_t len = BitmapLen(total); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46e9654a69..bbf14b2fdc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -32,13 +32,9 @@ int32_t tsVersion = 30000000; int32_t tsStatusInterval = 1; // second // common -int32_t tsRpcTimer = 300; -int32_t tsRpcMaxTime = 600; // seconds; -bool tsRpcForceTcp = true; // disable this, means query, show command use udp protocol as default int32_t tsMaxShellConns = 50000; int32_t tsMaxConnections = 50000; int32_t tsShellActivityTimer = 3; // second -int32_t tsMaxBinaryDisplayWidth = 30; bool tsEnableSlaveQuery = true; bool tsPrintAuth = false; @@ -105,14 +101,6 @@ int32_t tsCompressColData = -1; */ int32_t tsCompatibleModel = 1; -// client -int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN; -int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN; - -// the maximum number of results for projection query on super table that are returned from -// one virtual node, to order according to timestamp -int32_t tsMaxNumOfOrderedResults = 100000; - // 10 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 10; @@ -232,7 +220,7 @@ struct SConfig *taosGetCfg() { return tsCfg; } -static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { +static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { char cfgDir[PATH_MAX] = {0}; char cfgFile[PATH_MAX + 100] = {0}; @@ -243,6 +231,8 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e tstrncpy(cfgFile, cfgDir, sizeof(cfgDir)); } + if (apolloUrl == NULL || apolloUrl[0] == '\0') cfgGetApollUrl(envCmd, envFile, apolloUrl); + if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr()); return -1; @@ -263,6 +253,11 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e return -1; } + if (cfgLoad(pCfg, CFG_STYPE_ENV_CMD, envCmd) != 0) { + uError("failed to load from cmd env variables since %s", terrstr()); + return -1; + } + return 0; } @@ -314,19 +309,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1; - if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0) - return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 4; @@ -346,8 +332,8 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) { if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; - if (cfgAddInt64(pCfg, "totalMemory(KB)", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; @@ -508,18 +494,10 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } - tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32; - tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32; - tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32; - tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; - tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32; - tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32; - tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; - tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; return 0; @@ -601,8 +579,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { return 0; } -int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, - const char *apolloUrl, SArray *pArgs, bool tsc) { +int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, + char *apolloUrl, SArray *pArgs, bool tsc) { osDefaultInit(); SConfig *pCfg = cfgInit(); @@ -617,7 +595,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi if (taosAddServerLogCfg(pCfg) != 0) return -1; } - if (taosLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { + if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { uError("failed to load cfg since %s", terrstr()); cfgCleanup(pCfg); return -1; @@ -654,7 +632,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return 0; } -int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc) { +int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg != NULL) return 0; tsCfg = cfgInit(); @@ -669,7 +647,7 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU } taosAddSystemCfg(tsCfg); - if (taosLoadCfg(tsCfg, cfgDir, envFile, apolloUrl) != 0) { + if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) { uError("failed to load cfg since %s", terrstr()); cfgCleanup(tsCfg); tsCfg = NULL; @@ -710,6 +688,6 @@ void taosCfgDynamicOptions(const char *option, const char *value) { if (strcasecmp(option, "resetlog") == 0) { taosResetLog(); - cfgDumpCfg(tsCfg, 1, false); + cfgDumpCfg(tsCfg, 0, false); } } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index eb7c9a763f..7abe1186c6 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -93,7 +93,87 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { return row; } } +#if 0 +// TODO: KEEP one suite of iterator API finally. +// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts +// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later +// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter +int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) { + if (pMsg == NULL) { + terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; + return -1; + } + + pIter->totalLen = htonl(pMsg->length); + ASSERT(pIter->totalLen > 0); + pIter->len = 0; + pIter->pMsg = pMsg; + if (pIter->totalLen <= sizeof(SSubmitReq)) { + terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; + return -1; + } + + return 0; +} + +int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { + ASSERT(pIter->len >= 0); + + if (pIter->len == 0) { + pIter->len += sizeof(SSubmitReq); + } else { + if (pIter->len >= pIter->totalLen) { + ASSERT(0); + } + + SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); + pIter->len += (sizeof(SSubmitBlk) + pIter->dataLen + pIter->schemaLen); + ASSERT(pIter->len > 0); + } + + if (pIter->len > pIter->totalLen) { + terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; + *pPBlock = NULL; + return -1; + } + + if (pIter->len == pIter->totalLen) { + *pPBlock = NULL; + } else { + *pPBlock = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); + pIter->uid = htobe64((*pPBlock)->uid); + pIter->suid = htobe64((*pPBlock)->suid); + pIter->sversion = htonl((*pPBlock)->sversion); + pIter->dataLen = htonl((*pPBlock)->dataLen); + pIter->schemaLen = htonl((*pPBlock)->schemaLen); + pIter->numOfRows = htons((*pPBlock)->numOfRows); + } + return 0; +} + +int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { + if (pMsgIter->dataLen <= 0) return -1; + pIter->totalLen = pMsgIter->dataLen; + pIter->len = 0; + pIter->row = (STSRow *)(pBlock->data + pMsgIter->schemaLen); + return 0; +} + +STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) { + STSRow *row = pIter->row; + + if (pIter->len >= pIter->totalLen) { + return NULL; + } else { + pIter->len += TD_ROW_LEN(row); + if (pIter->len < pIter->totalLen) { + pIter->row = POINTER_SHIFT(row, TD_ROW_LEN(row)); + } + return row; + } +} +#endif int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) { if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; @@ -126,7 +206,6 @@ int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) { return 0; } - int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pEp->inUse); @@ -443,10 +522,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { SRSmaParam *param = pReq->stbCfg.pRSmaParam; tlen += taosEncodeBinary(buf, (const void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); tlen += taosEncodeFixedI32(buf, param->delay); - tlen += taosEncodeFixedI8(buf, param->nFuncIds); - for (int8_t i = 0; i < param->nFuncIds; ++i) { - tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); - } tlen += taosEncodeFixedI32(buf, param->qmsg1Len); if (param->qmsg1Len > 0) { tlen += taosEncodeString(buf, param->qmsg1); @@ -476,10 +551,6 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { SRSmaParam *param = pReq->ntbCfg.pRSmaParam; tlen += taosEncodeBinary(buf, (const void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); tlen += taosEncodeFixedI32(buf, param->delay); - tlen += taosEncodeFixedI8(buf, param->nFuncIds); - for (int8_t i = 0; i < param->nFuncIds; ++i) { - tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); - } } break; default: @@ -522,13 +593,6 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { SRSmaParam *param = pReq->stbCfg.pRSmaParam; buf = taosDecodeBinaryTo(buf, (void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); buf = taosDecodeFixedI32(buf, ¶m->delay); - buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); - if (param->nFuncIds > 0) { - param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t)); - for (int8_t i = 0; i < param->nFuncIds; ++i) { - buf = taosDecodeFixedI32(buf, param->pFuncIds + i); - } - } buf = taosDecodeFixedI32(buf, ¶m->qmsg1Len); if (param->qmsg1Len > 0) { buf = taosDecodeString(buf, ¶m->qmsg1); @@ -562,15 +626,6 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { SRSmaParam *param = pReq->ntbCfg.pRSmaParam; buf = taosDecodeBinaryTo(buf, (void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); buf = taosDecodeFixedI32(buf, ¶m->delay); - buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); - if (param->nFuncIds > 0) { - param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t)); - for (int8_t i = 0; i < param->nFuncIds; ++i) { - buf = taosDecodeFixedI32(buf, param->pFuncIds + i); - } - } else { - param->pFuncIds = NULL; - } } else { pReq->ntbCfg.pRSmaParam = NULL; } @@ -633,7 +688,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1; - if (tEncodeI32(&encoder, pReq->aggregationMethod) < 0) return -1; if (tEncodeI32(&encoder, pReq->delay) < 0) return -1; if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; @@ -688,7 +742,6 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->aggregationMethod) < 0) return -1; if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; @@ -1568,13 +1621,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1; if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; - int32_t codeSize = 0; if (pReq->pCode != NULL) { - codeSize = strlen(pReq->pCode) + 1; - } - if (tEncodeI32(&encoder, codeSize) < 0) return -1; - if (pReq->pCode != NULL) { - if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1; + if (tEncodeBinary(&encoder, pReq->pCode, pReq->codeLen) < 0) return -1; } int32_t commentSize = 0; @@ -1608,10 +1656,8 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1; if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; - int32_t codeSize = 0; - if (tDecodeI32(&decoder, &codeSize) < 0) return -1; - if (codeSize > 0) { - pReq->pCode = taosMemoryCalloc(1, codeSize); + if (pReq->codeLen > 0) { + pReq->pCode = taosMemoryCalloc(1, pReq->codeLen); if (pReq->pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1734,7 +1780,7 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; if (pInfo->codeSize) { - if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + if (tEncodeBinary(&encoder, pInfo->pCode, pInfo->codeSize) < 0) return -1; } if (pInfo->commentSize) { if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; @@ -2091,10 +2137,15 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp if (tStartDecode(&decoder) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; - pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); - if (NULL == pRsp->addrsList) return -1; + if (NULL == pRsp->addrsList) { + pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); + if (NULL == pRsp->addrsList) return -1; + } + for (int32_t i = 0; i < num; ++i) { - if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1; + SQueryNodeAddr addr = {0}; + if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1; + taosArrayPush(pRsp->addrsList, &addr); } tEndDecode(&decoder); @@ -2747,11 +2798,11 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1; if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; - if (0 == astLen && tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1; tEndEncode(&encoder); @@ -2773,6 +2824,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1; @@ -2787,7 +2839,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } else { - if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1; } tEndDecode(&decoder); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index b8ace4bf50..43237e6e6c 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -24,6 +24,7 @@ static struct { bool printVersion; char envFile[PATH_MAX]; char apolloUrl[PATH_MAX]; + const char **envCmd; SArray *pArgs; // SConfigPair SDnode *pDnode; EDndNodeType ntype; @@ -56,6 +57,9 @@ static void dmSetSignalHandle() { } static int32_t dmParseArgs(int32_t argc, char const *argv[]) { + int32_t cmdEnvIndex = 0; + global.envCmd = taosMemoryMalloc(argc-1); + memset(global.envCmd, 0, argc-1); for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { if (i < argc - 1) { @@ -70,7 +74,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { } } else if (strcmp(argv[i], "-a") == 0) { tstrncpy(global.apolloUrl, argv[++i], PATH_MAX); - } else if (strcmp(argv[i], "-e") == 0) { + } else if (strcmp(argv[i], "-E") == 0) { tstrncpy(global.envFile, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-n") == 0) { global.ntype = atoi(argv[++i]); @@ -84,6 +88,9 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) { global.dumpConfig = true; } else if (strcmp(argv[i], "-V") == 0) { global.printVersion = true; + } else if (strcmp(argv[i], "-e") == 0) { + global.envCmd[cmdEnvIndex] = argv[++i]; + cmdEnvIndex++; } else { } } @@ -106,7 +113,7 @@ static void dmPrintVersion() { static void dmDumpCfg() { SConfig *pCfg = taosGetCfg(); - cfgDumpCfg(pCfg, 0, 1); + cfgDumpCfg(pCfg, 0, true); } static SDnodeOpt dmGetOpt() { @@ -129,7 +136,7 @@ static SDnodeOpt dmGetOpt() { static int32_t dmInitLog() { char logName[12] = {0}; snprintf(logName, sizeof(logName), "%slog", dmLogName(global.ntype)); - return taosCreateLog(logName, 1, configDir, global.envFile, global.apolloUrl, global.pArgs, 0); + return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); } static void dmSetProcInfo(int32_t argc, char **argv) { @@ -168,6 +175,10 @@ static int32_t dmRunDnode() { return code; } +static void taosCleanupArgs() { + if (global.envCmd != NULL) taosMemoryFree(global.envCmd); +} + int main(int argc, char const *argv[]) { if (!taosCheckSystemIsSmallEnd()) { printf("failed to start since on non-small-end machines\n"); @@ -176,26 +187,31 @@ int main(int argc, char const *argv[]) { if (dmParseArgs(argc, argv) != 0) { printf("failed to start since parse args error\n"); + taosCleanupArgs(); return -1; } if (global.generateGrant) { dmGenerateGrant(); + taosCleanupArgs(); return 0; } if (global.printVersion) { dmPrintVersion(); + taosCleanupArgs(); return 0; } if (dmInitLog() != 0) { - dError("failed to start since init log error"); + printf("failed to start since init log error"); + taosCleanupArgs(); return -1; } - if (taosInitCfg(configDir, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { + if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { dError("failed to start since read config error"); + taosCleanupArgs(); return -1; } @@ -203,9 +219,11 @@ int main(int argc, char const *argv[]) { dmDumpCfg(); taosCleanupCfg(); taosCloseLog(); + taosCleanupArgs(); return 0; } dmSetProcInfo(argc, (char **)argv); + taosCleanupArgs(); return dmRunDnode(); } diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index 90e28f3c5b..ca1b943fb2 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -17,7 +17,7 @@ #include "dmImp.h" static void dmUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { - if (pDnode->data.dnodeId == 0) { + if (pDnode->data.dnodeId == 0 || pDnode->data.clusterId == 0) { dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); taosWLockLatch(&pDnode->data.latch); pDnode->data.dnodeId = pCfg->dnodeId; @@ -57,6 +57,7 @@ void dmSendStatusReq(SDnode *pDnode) { req.dnodeVer = pDnode->data.dnodeVer; req.dnodeId = pDnode->data.dnodeId; req.clusterId = pDnode->data.clusterId; + if (req.clusterId == 0) req.dnodeId = 0; req.rebootTime = pDnode->data.rebootTime; req.updateTime = pDnode->data.updateTime; req.numOfCores = tsNumOfCores; @@ -145,10 +146,10 @@ int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMs dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); } else { dDebug("node:%s, has been created", pWrapper->name); + (void)dmOpenNode(pWrapper); pWrapper->required = true; pWrapper->deployed = true; pWrapper->procType = pDnode->ptype; - (void)dmOpenNode(pWrapper); } taosThreadMutexUnlock(&pDnode->mutex); @@ -170,13 +171,13 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg) dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); } else { dDebug("node:%s, has been dropped", pWrapper->name); + pWrapper->required = false; + pWrapper->deployed = false; } dmReleaseWrapper(pWrapper); if (code == 0) { - pWrapper->required = false; - pWrapper->deployed = false; dmCloseNode(pWrapper); taosRemoveDir(pWrapper->path); } @@ -310,6 +311,9 @@ static void dmWatchUdfd(void *args) { } static int32_t dmStartUdfd(SDnode *pDnode) { + char dnodeId[8] = {0}; + snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId); + uv_os_setenv("DNODE_ID", dnodeId); SUdfdData *pData = &pDnode->udfdData; if (pData->startCalled) { dInfo("dnode-mgmt start udfd already called"); @@ -319,8 +323,17 @@ static int32_t dmStartUdfd(SDnode *pDnode) { uv_barrier_init(&pData->barrier, 2); uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); uv_barrier_wait(&pData->barrier); - pData->needCleanUp = true; - return pData->spawnErr; + int32_t err = atomic_load_32(&pData->spawnErr); + if (err != 0) { + uv_barrier_destroy(&pData->barrier); + uv_async_send(&pData->stopAsync); + uv_thread_join(&pData->thread); + pData->needCleanUp = false; + dInfo("dnode-mgmt udfd cleaned up after spawn err"); + } else { + pData->needCleanUp = true; + } + return err; } static int32_t dmStopUdfd(SDnode *pDnode) { @@ -335,7 +348,7 @@ static int32_t dmStopUdfd(SDnode *pDnode) { uv_barrier_destroy(&pData->barrier); uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); - + dInfo("dnode-mgmt udfd cleaned up"); return 0; } @@ -370,9 +383,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "dnode-transport", "initialized"); -// if (dmStartUdfd(pDnode) != 0) { -// dError("failed to start udfd"); -// } + if (dmStartUdfd(pDnode) != 0) { + dError("failed to start udfd"); + } dInfo("dnode-mgmt is initialized"); return 0; diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index a574d802f9..fb8e5e7fb2 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -59,6 +59,10 @@ static inline int32_t dmBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) { pMsg->clientIp = connInfo.clientIp; pMsg->clientPort = connInfo.clientPort; memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); + if ((pRpc->msgType & 1u)) { + assert(pRpc->refId != 0); + } + // assert(pRpc->handle != NULL && pRpc->refId != 0 && pMsg->rpcMsg.refId != 0); return 0; } @@ -67,12 +71,15 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe SNodeMsg *pMsg = NULL; NodeMsgFp msgFp = NULL; uint16_t msgType = pRpc->msgType; + bool needRelease = false; if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) { dmSetMnodeEpSet(pWrapper->pDnode, pEpSet); } if (dmMarkWrapper(pWrapper) != 0) goto _OVER; + + needRelease = true; if ((msgFp = dmGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER; @@ -84,7 +91,7 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe dTrace("msg:%p, is created and put into child queue, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), pRpc->handle, pMsg->user); code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, pRpc->handle, - PROC_FUNC_REQ); + pRpc->refId, PROC_FUNC_REQ); } else { dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); ASSERT(1); @@ -107,7 +114,7 @@ _OVER: } } - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code, .refId = pRpc->refId}; tmsgSendRsp(&rsp); } dTrace("msg:%p, is freed", pMsg); @@ -115,7 +122,9 @@ _OVER: rpcFreeCont(pRpc->pCont); } - dmReleaseWrapper(pWrapper); + if (needRelease) { + dmReleaseWrapper(pWrapper); + } } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { @@ -131,10 +140,17 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } + if (msgType == TDMT_DND_NET_TEST) { + dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle); + dmProcessServerStatusReq(pDnode, pMsg); + return; + } + if (pDnode->status != DND_STAT_RUNNING) { dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); if (isReq) { - SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle}; + SRpcMsg rspMsg = { + .handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle, .refId = pMsg->refId}; rpcSendResponse(&rspMsg); } rpcFreeCont(pMsg->pCont); @@ -143,7 +159,8 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { if (isReq && pMsg->pCont == NULL) { dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); - SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_INVALID_MSG_LEN, .ahandle = pMsg->ahandle}; + SRpcMsg rspMsg = { + .handle = pMsg->handle, .code = TSDB_CODE_INVALID_MSG_LEN, .ahandle = pMsg->ahandle, .refId = pMsg->refId}; rpcSendResponse(&rspMsg); return; } @@ -151,7 +168,8 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { if (pWrapper == NULL) { dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); if (isReq) { - SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle}; + SRpcMsg rspMsg = { + .handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle, .refId = pMsg->refId}; rpcSendResponse(&rspMsg); } rpcFreeCont(pMsg->pCont); @@ -170,6 +188,9 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { } dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle); + if (isReq) { + assert(pMsg->refId != 0); + } dmProcessRpcMsg(pWrapper, pMsg, pEpSet); } @@ -317,7 +338,7 @@ static void dmConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t if (code != 0) { dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); if (pRpc->msgType & 1U) { - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno, .refId = pRpc->refId}; dmSendRsp(pWrapper, &rsp); } @@ -346,7 +367,7 @@ static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); break; case PROC_FUNC_RSP: - taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); + pMsg->refId = taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); dmSendRpcRsp(pWrapper->pDnode, pMsg); break; default: diff --git a/source/dnode/mgmt/implement/src/dmWorker.c b/source/dnode/mgmt/implement/src/dmWorker.c index 505efeb8c6..b19c2ab36b 100644 --- a/source/dnode/mgmt/implement/src/dmWorker.c +++ b/source/dnode/mgmt/implement/src/dmWorker.c @@ -105,7 +105,7 @@ void dmStopMonitorThread(SDnode *pDnode) { } static void dmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { - SDnode *pDnode = pInfo->ahandle; + SDnode * pDnode = pInfo->ahandle; SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; dTrace("msg:%p, will be processed in dnode-mgmt queue", pMsg); @@ -150,7 +150,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) { if (pRpc->msgType & 1u) { if (code != 0) code = terrno; - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; + SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code, .refId = pRpc->refId}; rpcSendResponse(&rsp); } diff --git a/source/dnode/mgmt/interface/inc/dmInt.h b/source/dnode/mgmt/interface/inc/dmInt.h index a2368f3173..63bfaf5ad2 100644 --- a/source/dnode/mgmt/interface/inc/dmInt.h +++ b/source/dnode/mgmt/interface/inc/dmInt.h @@ -37,6 +37,7 @@ void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc); void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg); +void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg); void dmGetMonitorSysInfo(SMonSysInfo *pInfo); // dmFile.c diff --git a/source/dnode/mgmt/interface/src/dmFile.c b/source/dnode/mgmt/interface/src/dmFile.c index e9117939d7..38acf169be 100644 --- a/source/dnode/mgmt/interface/src/dmFile.c +++ b/source/dnode/mgmt/interface/src/dmFile.c @@ -124,7 +124,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) { TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s", filepath, terrstr()); + dError("failed to open file:%s since %s", filepath, terrstr()); return NULL; } diff --git a/source/dnode/mgmt/interface/src/dmInt.c b/source/dnode/mgmt/interface/src/dmInt.c index 00abbd0199..2d15a7a008 100644 --- a/source/dnode/mgmt/interface/src/dmInt.c +++ b/source/dnode/mgmt/interface/src/dmInt.c @@ -171,13 +171,25 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { } } +void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) { + dDebug("net test req is received"); + SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0}; + rsp.pCont = rpcMallocCont(pRpc->contLen); + if (rsp.pCont == NULL) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.contLen = pRpc->contLen; + } + rpcSendResponse(&rsp); +} + void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) { dDebug("server status req is received"); SServerStatusRsp statusRsp = {0}; dmGetServerStatus(pDnode, &statusRsp); - SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle}; + SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle, .refId = pReq->refId}; int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); if (rspLen < 0) { rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c index 3481078d51..230fa23674 100644 --- a/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c +++ b/source/dnode/mgmt/mgmt_bnode/src/bmWorker.c @@ -17,7 +17,8 @@ #include "bmInt.h" static void bmSendErrorRsp(SNodeMsg *pMsg, int32_t code) { - SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + SRpcMsg rpcRsp = { + .handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code, .refId = pMsg->rpcMsg.refId}; tmsgSendRsp(&rpcRsp); dTrace("msg:%p, is freed", pMsg); @@ -38,6 +39,7 @@ static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) { static inline void bmSendRsp(SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, + .refId = pMsg->rpcMsg.refId, .code = code, .pCont = pMsg->pRsp, .contLen = pMsg->rspLen}; @@ -101,7 +103,7 @@ static void bmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SBnodeMgmt *pMgmt = pWrapper->pMgmt; + SBnodeMgmt * pMgmt = pWrapper->pMgmt; SMultiWorker *pWorker = &pMgmt->writeWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -110,7 +112,7 @@ int32_t bmProcessWriteMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t bmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SBnodeMgmt *pMgmt = pWrapper->pMgmt; + SBnodeMgmt * pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index b434cf102e..afe57e3d8f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -177,6 +177,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_MND_QNODE_LIST, mmProcessReadMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index f707426ab4..db69b62e58 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -161,9 +161,9 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) { SMnodeOpt option = {0}; if (!deployed) { dInfo("mnode start to deploy"); - if (pWrapper->procType == DND_PROC_CHILD) { + // if (pWrapper->procType == DND_PROC_CHILD) { pWrapper->pDnode->data.dnodeId = 1; - } + // } mmBuildOptionForDeploy(pMgmt, &option); } else { dInfo("mnode start to open"); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index e9c40fdd0f..1f27b314e2 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -19,6 +19,7 @@ static inline void mmSendRsp(SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, + .refId = pMsg->rpcMsg.refId, .code = code, .pCont = pMsg->pRsp, .contLen = pMsg->rspLen}; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 6b27af4fbd..da85ee64a8 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -19,6 +19,7 @@ static inline void qmSendRsp(SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, + .refId = pMsg->rpcMsg.refId, .code = code, .pCont = pMsg->pRsp, .contLen = pMsg->rspLen}; diff --git a/source/dnode/mgmt/mgmt_snode/src/smWorker.c b/source/dnode/mgmt/mgmt_snode/src/smWorker.c index cf343423b7..25872aec55 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smWorker.c +++ b/source/dnode/mgmt/mgmt_snode/src/smWorker.c @@ -19,6 +19,7 @@ static inline void smSendRsp(SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, + .refId = pMsg->rpcMsg.refId, .code = code, .pCont = pMsg->pRsp, .contLen = pMsg->rspLen}; @@ -149,7 +150,7 @@ static FORCE_INLINE int32_t smGetSWTypeFromMsg(SRpcMsg *pMsg) { } int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; + SSnodeMgmt * pMgmt = pWrapper->pMgmt; SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, 0); if (pWorker == NULL) { terrno = TSDB_CODE_INVALID_MSG; @@ -162,7 +163,7 @@ int32_t smProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; + SSnodeMgmt * pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -171,7 +172,7 @@ int32_t smProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; + SSnodeMgmt * pMgmt = pWrapper->pMgmt; int32_t index = smGetSWIdFromMsg(&pMsg->rpcMsg); SMultiWorker *pWorker = taosArrayGetP(pMgmt->uniqueWorkers, index); if (pWorker == NULL) { @@ -185,7 +186,7 @@ int32_t smProcessUniqueMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t smProcessSharedMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SSnodeMgmt *pMgmt = pWrapper->pMgmt; + SSnodeMgmt * pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->sharedWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 4cc1b8527c..7fc263c93c 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -202,6 +202,17 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return code; } + code = vnodeStart(pImpl); + if (code != 0) { + tFreeSCreateVnodeReq(&createReq); + dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr()); + vnodeClose(pImpl); + vnodeDestroy(path, pMgmt->pTfs); + terrno = code; + return code; + } + + code = vmWriteVnodesToFile(pMgmt); if (code != 0) { tFreeSCreateVnodeReq(&createReq); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index e784272581..3088c5dea4 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "libs/function/function.h" SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -73,12 +74,6 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { return -1; } - // sync integration - vnodeSyncSetQ(pImpl, NULL); - vnodeSyncSetRpc(pImpl, NULL); - int32_t ret = vnodeSyncStart(pImpl); - assert(ret == 0); - taosWLockLatch(&pMgmt->latch); int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); taosWUnLockLatch(&pMgmt->latch); @@ -152,6 +147,7 @@ static void *vmOpenVnodeFunc(void *param) { pThread->failed++; } else { vmOpenVnode(pMgmt, pCfg, pImpl); + //vnodeStart(pImpl); dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->opened++; } @@ -275,7 +271,7 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { pWrapper->pMgmt = NULL; // syncCleanUp(); - + udfcClose(); dInfo("vnode-mgmt is cleaned up"); } @@ -339,6 +335,10 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "vnode-vnodes", "initialized"); + if (udfcOpen() != 0) { + dError("failed to open udfc in dnode"); + } + code = 0; _OVER: @@ -359,10 +359,52 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) { return 0; } +static int32_t vmStart(SMgmtWrapper *pWrapper) { + dDebug("vnode-mgmt start to run"); + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + + taosRLockLatch(&pMgmt->latch); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + if (ppVnode == NULL || *ppVnode == NULL) continue; + + SVnodeObj *pVnode = *ppVnode; + vnodeStart(pVnode->pImpl); + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosRUnLockLatch(&pMgmt->latch); + return 0; +} + +static void vmStop(SMgmtWrapper *pWrapper) { +#if 0 + dDebug("vnode-mgmt start to stop"); + SVnodesMgmt *pMgmt = pWrapper->pMgmt; + taosRLockLatch(&pMgmt->latch); + + void *pIter = taosHashIterate(pMgmt->hash, NULL); + while (pIter) { + SVnodeObj **ppVnode = pIter; + if (ppVnode == NULL || *ppVnode == NULL) continue; + + SVnodeObj *pVnode = *ppVnode; + vnodeStop(pVnode->pImpl); + pIter = taosHashIterate(pMgmt->hash, pIter); + } + + taosRUnLockLatch(&pMgmt->latch); +#endif +} + void vmSetMgmtFp(SMgmtWrapper *pWrapper) { SMgmtFp mgmtFp = {0}; mgmtFp.openFp = vmInit; mgmtFp.closeFp = vmCleanup; + mgmtFp.startFp = vmStart; + mgmtFp.stopFp = vmStop; mgmtFp.requiredFp = vmRequire; vmInitMsgHandle(pWrapper); @@ -391,4 +433,4 @@ void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) { } taosRUnLockLatch(&pMgmt->latch); -} \ No newline at end of file +} diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 41f0c27692..7285503a73 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -23,6 +23,7 @@ static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, + .refId = pMsg->rpcMsg.refId, .code = code, .pCont = pMsg->pRsp, .contLen = pMsg->rspLen}; @@ -126,6 +127,7 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO rsp.code = 0; rsp.handle = pRpc->handle; rsp.ahandle = pRpc->ahandle; + rsp.refId = pRpc->refId; int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp); tmsgSendRsp(&rsp); @@ -134,13 +136,14 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO // sync integration response for (int i = 0; i < taosArrayGetSize(pArray); i++) { SNodeMsg *pMsg; - SRpcMsg *pRpc; + SRpcMsg * pRpc; pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); pRpc = &pMsg->rpcMsg; rsp.ahandle = pRpc->ahandle; rsp.handle = pRpc->handle; + rsp.refId = pRpc->refId; rsp.pCont = NULL; rsp.contLen = 0; @@ -172,11 +175,9 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg *pMsg = NULL; + SNodeMsg * pMsg = NULL; SRpcMsg rsp; - // static int64_t version = 0; - for (int32_t i = 0; i < numOfMsgs; ++i) { #if 1 // sync integration @@ -208,6 +209,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) { rsp.ahandle = pMsg->rpcMsg.ahandle; rsp.handle = pMsg->rpcMsg.handle; + rsp.refId = pMsg->rpcMsg.refId; tmsgSendRsp(&rsp); } #endif @@ -216,7 +218,7 @@ static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg *pMsg = NULL; + SNodeMsg * pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -229,7 +231,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { SVnodeObj *pVnode = pInfo->ahandle; - SNodeMsg *pMsg = NULL; + SNodeMsg * pMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pMsg); @@ -246,7 +248,7 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO } static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { - SRpcMsg *pRpc = &pMsg->rpcMsg; + SRpcMsg * pRpc = &pMsg->rpcMsg; SMsgHead *pHead = pRpc->pCont; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); @@ -315,7 +317,7 @@ int32_t vmProcessMergeMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; + SVnodesMgmt * pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->mgmtWorker; dTrace("msg:%p, will be written to vnode-mgmt queue, worker:%s", pMsg, pWorker->name); taosWriteQitem(pWorker->queue, pMsg); @@ -323,7 +325,7 @@ int32_t vmProcessMgmtMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { - SVnodesMgmt *pMgmt = pWrapper->pMgmt; + SVnodesMgmt * pMgmt = pWrapper->pMgmt; SSingleWorker *pWorker = &pMgmt->monitorWorker; dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); @@ -333,7 +335,7 @@ int32_t vmProcessMonitorMsg(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; - SMsgHead *pHead = pRpc->pCont; + SMsgHead * pHead = pRpc->pCont; SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) return -1; @@ -346,6 +348,7 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT } else { dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); pMsg->rpcMsg = *pRpc; + // if (pMsg->rpcMsg.handle != NULL) assert(pMsg->rpcMsg.refId != 0); switch (qtype) { case QUERY_QUEUE: dTrace("msg:%p, will be put into vnode-query queue", pMsg); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 59a00f0eaf..cc56db354e 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -135,6 +135,7 @@ typedef struct { int32_t failedTimes; void* rpcHandle; void* rpcAHandle; + int64_t rpcRefId; void* rpcRsp; int32_t rpcRspLen; SArray* redoLogs; @@ -349,7 +350,6 @@ typedef struct { int32_t version; int32_t nextColId; float xFilesFactor; - int32_t aggregationMethod; int32_t delay; int32_t ttl; int32_t numOfColumns; @@ -449,7 +449,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; SRWLatch lock; int32_t sqlLen; int32_t astLen; @@ -516,7 +515,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub // TODO put -1 into unassignVgs // SArray* unassignedVgs; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 78d54d273d..800a013c4f 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -237,7 +237,6 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { pSubNew->withTbName = pSub->withTbName; pSubNew->withSchema = pSub->withSchema; pSubNew->withTag = pSub->withTag; - pSubNew->withTagSchema = pSub->withTagSchema; pSubNew->vgNum = pSub->vgNum; pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); @@ -270,7 +269,6 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { tlen += taosEncodeFixedI8(buf, pSub->withTbName); tlen += taosEncodeFixedI8(buf, pSub->withSchema); tlen += taosEncodeFixedI8(buf, pSub->withTag); - tlen += taosEncodeFixedI8(buf, pSub->withTagSchema); void *pIter = NULL; int32_t sz = taosHashGetSize(pSub->consumerHash); @@ -297,7 +295,6 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { buf = taosDecodeFixedI8(buf, &pSub->withTbName); buf = taosDecodeFixedI8(buf, &pSub->withSchema); buf = taosDecodeFixedI8(buf, &pSub->withTag); - buf = taosDecodeFixedI8(buf, &pSub->withTagSchema); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 156d894a44..09d0587019 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -309,10 +309,10 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.pCode[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - goto _OVER; - } + if (createReq.codeLen <= 1) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + goto _OVER; + } if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 5dbbd27930..2b46fc9274 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -325,7 +325,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) { return -1; } - if (taosHashPut(hash, meta.tbName, strlen(meta.tbName) + 1, &meta, sizeof(meta))) { + if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -340,10 +340,10 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * return -1; } - STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName) + 1); + STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); if (NULL == pMeta) { mError("invalid information schema table name:%s", tbName); - terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; return -1; } diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index 737068a2dd..a0ecbe9ae4 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -128,7 +128,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); if (NULL == meta) { mError("invalid performance schema table name:%s", tbName); - terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; return -1; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 1c03ca30f4..cec5933ba3 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -451,8 +451,9 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { goto _OVER; } + void *pIter = NULL; while (1) { - void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj); + pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj); if (pIter == NULL) break; SQueryNodeAddr nodeAddr = {0}; @@ -472,7 +473,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { } int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); - void *pRsp = taosMemoryMalloc(rspLen); + void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 73583058f1..3dff65866c 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -476,33 +476,37 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { SSdb* pSdb = pMnode->pSdb; SVgObj* pVgroup = NULL; - SQueryPlan* pPlan = qStringToQueryPlan(pTopic->physicalPlan); - if (pPlan == NULL) { - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + SQueryPlan* pPlan = NULL; + SSubplan* plan = NULL; + if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { + pPlan = qStringToQueryPlan(pTopic->physicalPlan); + if (pPlan == NULL) { + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + + ASSERT(pSub->vgNum == -1); + + pSub->vgNum = 0; + + int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); + if (levelNum != 1) { + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + return -1; + } + + SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); + + int32_t opNum = LIST_LENGTH(inner->pNodeList); + if (opNum != 1) { + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; + return -1; + } + plan = nodesListGetNode(inner->pNodeList, 0); } - ASSERT(pSub->vgNum == -1); - - pSub->vgNum = 0; - - int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); - if (levelNum != 1) { - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; - return -1; - } - - SNodeListNode* inner = nodesListGetNode(pPlan->pSubplans, 0); - - int32_t opNum = LIST_LENGTH(inner->pNodeList); - if (opNum != 1) { - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; - return -1; - } - SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); - int64_t unexistKey = -1; SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(pEpInSub); @@ -519,38 +523,35 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib } pSub->vgNum++; - plan->execNode.nodeId = pVgroup->vgId; - plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup); SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); - pVgEp->epSet = plan->execNode.epSet; - pVgEp->vgId = plan->execNode.nodeId; - -#if 0 - SMqConsumerEp consumerEp = {0}; - consumerEp.status = 0; - consumerEp.consumerId = -1; - consumerEp.epSet = plan->execNode.epSet; - consumerEp.vgId = plan->execNode.nodeId; -#endif + pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); + pVgEp->vgId = pVgroup->vgId; + taosArrayPush(pEpInSub->vgs, &pVgEp); mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId); - int32_t msgLen; - if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) { - sdbRelease(pSdb, pVgroup); - qDestroyQueryPlan(pPlan); - terrno = TSDB_CODE_QRY_INVALID_INPUT; - return -1; + if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { + int32_t msgLen; + + plan->execNode.epSet = pVgEp->epSet; + plan->execNode.nodeId = pVgEp->vgId; + + if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) { + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + terrno = TSDB_CODE_QRY_INVALID_INPUT; + return -1; + } + } else { + pVgEp->qmsg = strdup(""); } - taosArrayPush(pEpInSub->vgs, &pVgEp); ASSERT(taosHashGetSize(pSub->consumerHash) == 1); /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/ } - ASSERT(pEpInSub->vgs->size > 0); pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(pEpInSub->vgs->size > 0); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1ab25f795f..94366a241d 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -100,6 +100,8 @@ static int32_t convertToRetrieveType(char* name, int32_t len) { type = TSDB_MGMT_TABLE_QUERIES; } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) { type = TSDB_MGMT_TABLE_VNODES; + } else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) { + type = TSDB_MGMT_TABLE_TOPICS; } else { // ASSERT(0); } @@ -187,11 +189,14 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { } if (retrieveReq.showId == 0) { - STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1); + STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { - terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; - mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); - return -1; + pMeta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); + if (pMeta == NULL) { + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; + mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); + return -1; + } } pShow = mndCreateShowObj(pMnode, &retrieveReq); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f304e3153d..6332ff8662 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -87,7 +87,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER) SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER) - SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER) @@ -175,7 +174,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int32_t xFilesFactor = 0; SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER) pStb->xFilesFactor = xFilesFactor / 10000.0f; - SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->delay, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER) @@ -404,7 +402,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt req.name = (char *)tNameGetTableName(&name); req.ttl = 0; req.keep = 0; - req.rollup = pStb->aggregationMethod > -1 ? 1 : 0; + req.rollup = pStb->pAst1 > 0 ? 1 : 0; req.type = TD_SUPER_TABLE; req.stbCfg.suid = pStb->uid; req.stbCfg.nCols = pStb->numOfColumns; @@ -433,29 +431,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt pRSmaParam->xFilesFactor = pStb->xFilesFactor; pRSmaParam->delay = pStb->delay; - pRSmaParam->nFuncIds = 1; // only 1 aggregation method supported currently - pRSmaParam->pFuncIds = (func_id_t *)taosMemoryCalloc(pRSmaParam->nFuncIds, sizeof(func_id_t)); - if (pRSmaParam->pFuncIds == NULL) { - taosMemoryFreeClear(req.stbCfg.pRSmaParam); - taosMemoryFreeClear(req.stbCfg.pSchema); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) { - *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod; - } if (pStb->ast1Len > 0) { if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pRSmaParam->pFuncIds); taosMemoryFreeClear(req.stbCfg.pRSmaParam); taosMemoryFreeClear(req.stbCfg.pSchema); return NULL; } } if (pStb->ast2Len > 0) { - int32_t qmsgLen2 = 0; if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pRSmaParam->pFuncIds); taosMemoryFreeClear(pRSmaParam->qmsg1); taosMemoryFreeClear(req.stbCfg.pRSmaParam); taosMemoryFreeClear(req.stbCfg.pSchema); @@ -470,7 +454,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt SMsgHead *pHead = taosMemoryMalloc(contLen); if (pHead == NULL) { if (pRSmaParam) { - taosMemoryFreeClear(pRSmaParam->pFuncIds); taosMemoryFreeClear(pRSmaParam->qmsg1); taosMemoryFreeClear(pRSmaParam->qmsg2); taosMemoryFreeClear(pRSmaParam); @@ -488,7 +471,6 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt *pContLen = contLen; if (pRSmaParam) { - taosMemoryFreeClear(pRSmaParam->pFuncIds); taosMemoryFreeClear(pRSmaParam->qmsg1); taosMemoryFreeClear(pRSmaParam->qmsg2); taosMemoryFreeClear(pRSmaParam); @@ -706,7 +688,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre stbObj.version = 1; stbObj.nextColId = 1; stbObj.xFilesFactor = pCreate->xFilesFactor; - stbObj.aggregationMethod = pCreate->aggregationMethod; stbObj.delay = pCreate->delay; stbObj.ttl = pCreate->ttl; stbObj.numOfColumns = pCreate->numOfColumns; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e37bd60e12..6a1994d7b8 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -35,11 +35,6 @@ #define MND_SUBSCRIBE_REBALANCE_CNT 3 -enum { - MQ_SUBSCRIBE_STATUS__ACTIVE = 1, - MQ_SUBSCRIBE_STATUS__DELETED, -}; - static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); @@ -89,7 +84,6 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, pSub->withTbName = pTopic->withTbName; pSub->withSchema = pTopic->withSchema; pSub->withTag = pTopic->withTag; - pSub->withTagSchema = pTopic->withTagSchema; ASSERT(taosHashGetSize(pSub->consumerHash) == 1); @@ -115,7 +109,6 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.withTbName = pSub->withTbName; req.withSchema = pSub->withSchema; req.withTag = pSub->withTag; - req.withTagSchema = pSub->withTagSchema; strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); @@ -514,9 +507,11 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { // TODO replace assert with error check ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); + // if add more consumer to balanced subscribe, // possibly no vg is changed /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ + ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); if (rebInput.pTopic) { @@ -673,177 +668,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -#if 0 -static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - char *msgStr = pMsg->rpcMsg.pCont; - SCMSubscribeReq subscribe; - tDeserializeSCMSubscribeReq(msgStr, &subscribe); - int64_t consumerId = subscribe.consumerId; - char *cgroup = subscribe.consumerGroup; - - SArray *newSub = subscribe.topicNames; - int32_t newTopicNum = subscribe.topicNum; - - taosArraySortString(newSub, taosArrayCompareString); - - SArray *oldSub = NULL; - int32_t oldTopicNum = 0; - bool createConsumer = false; - // create consumer if not exist - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); - if (pConsumer == NULL) { - // create consumer - pConsumer = mndCreateConsumer(consumerId, cgroup); - createConsumer = true; - } else { - pConsumer->epoch++; - oldSub = pConsumer->currentTopics; - } - pConsumer->currentTopics = newSub; - - if (oldSub != NULL) { - oldTopicNum = taosArrayGetSize(oldSub); - } - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); - if (pTrans == NULL) { - // TODO: free memory - return -1; - } - - int32_t i = 0, j = 0; - while (i < newTopicNum || j < oldTopicNum) { - char *newTopicName = NULL; - char *oldTopicName = NULL; - if (i >= newTopicNum) { - // encode unset topic msg to all vnodes related to that topic - oldTopicName = taosArrayGetP(oldSub, j); - j++; - } else if (j >= oldTopicNum) { - newTopicName = taosArrayGetP(newSub, i); - i++; - } else { - newTopicName = taosArrayGetP(newSub, i); - oldTopicName = taosArrayGetP(oldSub, j); - - int32_t comp = compareLenPrefixedStr(newTopicName, oldTopicName); - if (comp == 0) { - // do nothing - oldTopicName = newTopicName = NULL; - i++; - j++; - continue; - } else if (comp < 0) { - oldTopicName = NULL; - i++; - } else { - newTopicName = NULL; - j++; - } - } - - if (oldTopicName != NULL) { - ASSERT(newTopicName == NULL); - - // cancel subscribe of old topic - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, oldTopicName); - ASSERT(pSub); - int32_t csz = taosArrayGetSize(pSub->consumers); - for (int32_t ci = 0; ci < csz; ci++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, ci); - if (pSubConsumer->consumerId == consumerId) { - int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - for (int32_t vgi = 0; vgi < vgsz; vgi++) { - SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi); - mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp, oldTopicName); - taosArrayPush(pSub->unassignedVg, pConsumerEp); - } - taosArrayRemove(pSub->consumers, ci); - break; - } - } - char *oldTopicNameDup = strdup(oldTopicName); - taosArrayPush(pConsumer->recentRemovedTopics, &oldTopicNameDup); - atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__MODIFY); - /*pSub->status = MQ_SUBSCRIBE_STATUS__DELETED;*/ - } else if (newTopicName != NULL) { - ASSERT(oldTopicName == NULL); - - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); - if (pTopic == NULL) { - mError("topic being subscribed not exist: %s", newTopicName); - continue; - } - - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, newTopicName); - bool createSub = false; - if (pSub == NULL) { - mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup, - newTopicName); - pSub = mndCreateSubscription(pMnode, pTopic, cgroup); - createSub = true; - - mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg); - } - - SMqSubConsumer mqSubConsumer; - mqSubConsumer.consumerId = consumerId; - mqSubConsumer.vgInfo = taosArrayInit(0, sizeof(SMqConsumerEp)); - taosArrayPush(pSub->consumers, &mqSubConsumer); - - // if have un assigned vg, assign one to the consumer - if (taosArrayGetSize(pSub->unassignedVg) > 0) { - SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg); - pConsumerEp->oldConsumerId = pConsumerEp->consumerId; - pConsumerEp->consumerId = consumerId; - taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp); - if (pConsumerEp->oldConsumerId == -1) { - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, newTopicName, - pConsumerEp->consumerId); - mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); - } else { - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, newTopicName); - } - // to trigger rebalance at once, do not set status active - /*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/ - } - - SSdbRaw *pRaw = mndSubActionEncode(pSub); - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pRaw); - - if (!createSub) mndReleaseSubscribe(pMnode, pSub); - mndReleaseTopic(pMnode, pTopic); - } - } - - /*if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);*/ - - // persist consumerObj - SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); - sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pConsumerRaw); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("mq-subscribe-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer); - return -1; - } - - mndTransDrop(pTrans); - if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -} -#endif - static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } - -static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { - SSdb *pSdb = pMnode->pSdb; - sdbCancelFetch(pSdb, pIter); -} diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a4b98ba01a..7c4e51298f 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -82,7 +82,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER); - SDB_SET_INT8(pRaw, dataPos, pTopic->withTagSchema, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -146,7 +145,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER); - SDB_GET_INT8(pRaw, dataPos, &pTopic->withTagSchema, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); @@ -234,6 +232,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { sdbRelease(pSdb, pTopic); } +#if 0 static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { SName name = {0}; tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -243,6 +242,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { return mndAcquireDb(pMnode, db); } +#endif static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { int32_t contLen = sizeof(SDDropTopicReq); @@ -262,15 +262,11 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq } static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { - if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) { + if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) { terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; return -1; } - if ((pCreate->ast == NULL || pCreate->ast[0] == 0) && pCreate->subscribeDbName[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; - return -1; - } return 0; } @@ -286,10 +282,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq topicObj.version = 1; topicObj.sql = strdup(pCreate->sql); topicObj.sqlLen = strlen(pCreate->sql) + 1; - topicObj.ast = strdup(pCreate->ast); - topicObj.astLen = strlen(pCreate->ast) + 1; if (pCreate->ast && pCreate->ast[0]) { + topicObj.ast = strdup(pCreate->ast); + topicObj.astLen = strlen(pCreate->ast) + 1; topicObj.subType = TOPIC_SUB_TYPE__TABLE; topicObj.withTbName = 0; topicObj.withSchema = 0; @@ -318,6 +314,9 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq return -1; } } else { + topicObj.ast = strdup(""); + topicObj.astLen = 1; + topicObj.physicalPlan = strdup(""); topicObj.subType = TOPIC_SUB_TYPE__DB; topicObj.withTbName = 1; topicObj.withSchema = 1; @@ -386,7 +385,7 @@ static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq) { goto CREATE_TOPIC_OVER; } - pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name); + pDb = mndAcquireDb(pMnode, createTopicReq.subscribeDbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; goto CREATE_TOPIC_OVER; @@ -524,8 +523,11 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t cols = 0; char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN); - varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE])); + + SName n; + tNameFromString(&n, pTopic->name, T_NAME_ACCT|T_NAME_DB); + tNameGetDbName(&n, varDataVal(topicName)); + varDataSetLen(topicName, strlen(varDataVal(topicName))); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); @@ -539,7 +541,7 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); - taosMemoryFree(sql); +// taosMemoryFree(sql); numOfRows++; sdbRelease(pSdb, pTopic); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 59fe7d16b9..9e9a8b56c9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -193,9 +193,9 @@ TRANS_ENCODE_OVER: static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRow *pRow = NULL; - STrans *pTrans = NULL; - char *pData = NULL; + SSdbRow * pRow = NULL; + STrans * pTrans = NULL; + char * pData = NULL; int32_t dataLen = 0; int8_t sver = 0; int32_t redoLogNum = 0; @@ -456,7 +456,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { } static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); if (pTrans == NULL) { terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; @@ -484,6 +484,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S pTrans->createdTime = taosGetTimestampMs(); pTrans->rpcHandle = pReq->handle; pTrans->rpcAHandle = pReq->ahandle; + pTrans->rpcRefId = pReq->refId; pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); @@ -625,7 +626,7 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT if (mndIsBasicTrans(pNewTrans)) return 0; STrans *pTrans = NULL; - void *pIter = NULL; + void * pIter = NULL; int32_t code = 0; while (1) { @@ -703,6 +704,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { pNew->rpcHandle = pTrans->rpcHandle; pNew->rpcAHandle = pTrans->rpcAHandle; + pNew->rpcRefId = pTrans->rpcRefId; pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRspLen = pTrans->rpcRspLen; pTrans->rpcRsp = NULL; @@ -767,6 +769,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle, + .refId = pTrans->rpcRefId, .pCont = rpcCont, .contLen = pTrans->rpcRspLen}; tmsgSendRsp(&rspMsg); @@ -827,7 +830,7 @@ HANDLE_ACTION_RSP_OVER: } static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; int32_t arraySize = taosArrayGetSize(pArray); if (arraySize == 0) return 0; @@ -1202,11 +1205,11 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { } static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { - SMnode *pMnode = pReq->pNode; + SMnode * pMnode = pReq->pNode; SKillTransReq killReq = {0}; int32_t code = -1; - SUserObj *pUser = NULL; - STrans *pTrans = NULL; + SUserObj * pUser = NULL; + STrans * pTrans = NULL; if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -1246,7 +1249,7 @@ KILL_OVER: void mndTransPullup(SMnode *pMnode) { STrans *pTrans = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -1261,11 +1264,11 @@ void mndTransPullup(SMnode *pMnode) { static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; int32_t numOfRows = 0; STrans *pTrans = NULL; int32_t cols = 0; - char *pWrite; + char * pWrite; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_TRANS, pShow->pIter, (void **)&pTrans); diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index daf8dd431f..75caef2336 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -385,7 +385,11 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) { terrno = code; mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); } else if (code != 0) { - mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + if (terrno != TSDB_CODE_OPS_NOT_SUPPORT) { + mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + } else { + mTrace("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); + } } else { mTrace("msg:%p, is processed, app:%p", pMsg, ahandle); } diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 4db9411e87..0473fa375e 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -22,17 +22,16 @@ class MndTestFunc : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void SetCode(SCreateFuncReq* pReq, const char* pCode); + void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); }; Testbase MndTestFunc::test; -void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) { - int32_t len = strlen(pCode); - pReq->pCode = (char*)taosMemoryCalloc(1, len + 1); - strcpy(pReq->pCode, pCode); - pReq->codeLen = len; +void MndTestFunc::SetCode(SCreateFuncReq *pReq, const char *pCode, int32_t size) { + pReq->pCode = (char*)taosMemoryMalloc(size); + memcpy(pReq->pCode, pCode, size); + pReq->codeLen = size; } void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { @@ -79,7 +78,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, ""); + SetCode(&createReq, "", 1); SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -95,7 +94,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -111,7 +110,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; @@ -128,7 +127,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { for (int32_t i = 0; i < 3; ++i) { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; createReq.igExists = 0; @@ -253,7 +252,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { createReq.outputLen = 24; createReq.bufSize = 6; createReq.signature = 18; - SetCode(&createReq, "code2"); + SetCode(&createReq, "code2", 6); SetComment(&createReq, "comment2"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -439,3 +438,70 @@ TEST_F(MndTestFunc, 04_Drop_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); } + +TEST_F(MndTestFunc, 05_Actual_code) { + { + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "udf1"); + char code[300] = {0}; + for (int32_t i = 0; i < sizeof(code); ++i) { + code[i] = (i) % 20; + } + SetCode(&createReq, code, 300); + SetComment(&createReq, "comment1"); + createReq.bufSize = 8; + createReq.igExists = 0; + createReq.funcType = 1; + createReq.scriptType = 2; + createReq.outputType = TSDB_DATA_TYPE_SMALLINT; + createReq.outputLen = 12; + createReq.bufSize = 4; + createReq.signature = 5; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "udf1"); + + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + tFreeSRetrieveFuncReq(&retrieveReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 1); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); + + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); + + EXPECT_STREQ(pFuncInfo->name, "udf1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_STREQ("comment1", pFuncInfo->pComment); + for (int32_t i = 0; i < 300; ++i) { + EXPECT_EQ(pFuncInfo->pCode[i], (i) % 20); + } + tFreeSRetrieveFuncRsp(&retrieveRsp); + } + +} \ No newline at end of file diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 7b299f1f3c..907fddaec2 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -17,6 +17,7 @@ #include "qndInt.h" #include "query.h" #include "qworker.h" +//#include "tudf.h" SQnode *qndOpen(const SQnodeOpt *pOption) { SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); @@ -25,6 +26,8 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { return NULL; } + //udfcOpen(); + if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { taosMemoryFreeClear(pQnode); return NULL; @@ -37,13 +40,15 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); + //udfcClose(); + taosMemoryFree(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { - qTrace("message in query queue is processing"); + qTrace("message in qnode query queue is processing"); SReadHandle handle = {0}; switch (pMsg->msgType) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f4975c183e..9eab4d3376 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -61,6 +61,9 @@ int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName); +int32_t vnodeStart(SVnode *pVnode); +void vnodeStop(SVnode *pVnode); + int64_t vnodeGetSyncHandle(SVnode *pVnode); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); @@ -171,11 +174,6 @@ typedef struct { uint64_t uid; } STableKeyInfo; -// sync integration -void vnodeSyncSetQ(SVnode *pVnode, void *qHandle); -void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle); -int32_t vnodeSyncStart(SVnode *pVnode); - #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index fb875a46e0..7647929246 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -51,7 +51,7 @@ static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64() #define META_CHILD_TABLE TD_CHILD_TABLE #define META_NORMAL_TABLE TD_NORMAL_TABLE -int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg); +int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); int metaDropTable(SMeta* pMeta, tb_uid_t uid); int metaCommit(SMeta* pMeta); int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); @@ -74,7 +74,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); // SMetaDB int metaOpenDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta); -int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); +int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 6e004a89fc..347d28f1ea 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -159,7 +159,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; char* qmsg; STqPushHandle pushHandle; // SRWLatch lock; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a84776abfd..dddc170813 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -57,7 +57,6 @@ int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); void tsdbCleanupReadHandle(tsdbReaderT queryHandle); int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg); - typedef enum { TSDB_FILE_HEAD = 0, // .head TSDB_FILE_DATA, // .data diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h new file mode 100644 index 0000000000..e20d2989b9 --- /dev/null +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_VNODE_TSDB_SMA_H_ +#define _TD_VNODE_TSDB_SMA_H_ + +#include "os.h" +#include "thash.h" +#include "tmsg.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2); + +struct STbDdlH { + void *ahandle; + void *result; + __tb_ddl_fn_t fp; +}; + +typedef struct { + tb_uid_t suid; + SArray *tbUids; + SHashObj *uidHash; +} STbUidStore; + +static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) { + ASSERT(*pStore == NULL); + *pStore = taosMemoryCalloc(1, sizeof(STbUidStore)); + if (*pStore == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +void tsdbUidStoreDestory(STbUidStore *pStore); +void *tsdbUidStoreFree(STbUidStore *pStore); + +int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq); +int32_t tsdbFetchTbUidList(void *pTsdb, void **result, void *suid, void *uid); +int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pUidStore); +int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_VNODE_TSDB_SMA_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 0027424829..0f354401bf 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -103,6 +103,8 @@ struct SVnode { #define TD_VID(PVNODE) (PVNODE)->config.vgId +typedef struct STbDdlH STbDdlH; + // sma void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data); @@ -116,6 +118,8 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data); #include "vnodeSync.h" +#include "tsdbSma.h" + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 8389286596..d61d3346a2 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -250,7 +250,7 @@ void metaCloseDB(SMeta *pMeta) { } } -int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { +int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { tb_uid_t uid; SMetaDB *pMetaDb; void *pKey; @@ -349,6 +349,12 @@ int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg) { if (ret < 0) { return -1; } + // child table handle for rsma + if (pHandle && pHandle->fp) { + if (((*pHandle->fp)(pHandle->ahandle, &pHandle->result, &ctbIdxKey.suid, &uid)) < 0) { + return -1; + }; + } } else if (pTbCfg->type == META_NORMAL_TABLE) { pKey = &uid; kLen = sizeof(uid); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7f06ba8855..7f8ec3a656 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -15,7 +15,7 @@ #include "vnodeInt.h" -int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) { +int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) { // Validate the tbOptions // if (metaValidateTbCfg(pMeta, pTbCfg) < 0) { // // TODO: handle error @@ -24,7 +24,7 @@ int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg) { // TODO: add atomicity - if (metaSaveTableToDB(pMeta, pTbCfg) < 0) { + if (metaSaveTableToDB(pMeta, pTbCfg, pHandle) < 0) { // TODO: handle error return -1; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0b3feb5010..fdb43a761b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -31,6 +31,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; + #if 0 pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); @@ -401,10 +402,19 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch); } + SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); + if (pHeadWithCkSum == NULL) { + return -1; + } + + walSetReaderCapacity(pExec->pWalReader, 2048); + SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; + rsp.withSchema = pExec->withSchema; rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); + rsp.blockSchema = taosArrayInit(0, sizeof(void*)); while (1) { consumerEpoch = atomic_load_32(&pExec->epoch); @@ -414,6 +424,26 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { break; } + taosThreadMutexLock(&pExec->pWalReader->mutex); + + if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) { + vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchOffset); + taosThreadMutexUnlock(&pExec->pWalReader->mutex); + break; + } + + if (pHeadWithCkSum->head.msgType != TDMT_VND_SUBMIT) { + walSkipFetchBody(pExec->pWalReader, pHeadWithCkSum); + } else { + walFetchBody(pExec->pWalReader, &pHeadWithCkSum); + } + + SWalReadHead* pHead = &pHeadWithCkSum->head; + + taosThreadMutexUnlock(&pExec->pWalReader->mutex); + +#if 0 SWalReadHead* pHead; if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) { // TODO: no more log, set timer to wait blocking time @@ -443,14 +473,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; #endif - break; - } + break; + } +#endif vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; + // table subscribe if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { qTaskInfo_t task = pExec->task[workerId]; ASSERT(task); @@ -482,8 +514,15 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { ASSERT(actualLen <= dataStrLen); taosArrayPush(rsp.blockDataLen, &actualLen); taosArrayPush(rsp.blockData, &buf); + + if (pExec->withSchema) { + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper); + taosArrayPush(rsp.blockSchema, &pSW); + } + rsp.blockNum++; } + // db subscribe } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReadHandle* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pCont, 0); @@ -509,6 +548,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { ASSERT(actualLen <= dataStrLen); taosArrayPush(rsp.blockDataLen, &actualLen); taosArrayPush(rsp.blockData, &buf); + + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper); + taosArrayPush(rsp.blockSchema, &pSW); + rsp.blockNum++; } } else { @@ -555,6 +598,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO destroy taosArrayDestroy(rsp.blockData); taosArrayDestroy(rsp.blockDataLen); + taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + return 0; } @@ -789,7 +834,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pExec->withTbName = req.withTbName; pExec->withSchema = req.withSchema; pExec->withTag = req.withTag; - pExec->withTagSchema = req.withTagSchema; pExec->qmsg = req.qmsg; req.qmsg = NULL; @@ -797,12 +841,16 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); for (int32_t i = 0; i < 5; i++) { pExec->pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); - SReadHandle handle = { - .reader = pExec->pExecReader[i], - .meta = pTq->pVnode->pMeta, - }; - pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle); - ASSERT(pExec->task[i]); + if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { + SReadHandle handle = { + .reader = pExec->pExecReader[i], + .meta = pTq->pVnode->pMeta, + }; + pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle); + ASSERT(pExec->task[i]); + } else { + pExec->task[i] = NULL; + } } taosHashPut(pTq->execs, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec)); return 0; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index eb45577e0a..ef7e346867 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -65,7 +65,9 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { /*pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid);*/ /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ - ASSERT(pHandle->tbIdHash); + if (pHandle->tbIdHash == NULL) { + return true; + } void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); if (ret != NULL) { /*printf("retrieve one tb %ld\n", pHandle->pBlock->uid);*/ @@ -107,26 +109,15 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p *pNumOfRows = pHandle->pBlock->numOfRows; int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); - if (colNumNeed > pSchemaWrapper->nCols) { - colNumNeed = pSchemaWrapper->nCols; - } + if (colNumNeed == 0) { + *ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); + if (*ppCols == NULL) { + return -1; + } - *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - - int32_t colMeta = 0; - int32_t colNeed = 0; - while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - col_id_t colIdSchema = pColSchema->colId; - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed); - if (colIdSchema < colIdNeed) { - colMeta++; - } else if (colIdSchema > colIdNeed) { - colNeed++; - } else { + int32_t colMeta = 0; + while (colMeta < pSchemaWrapper->nCols) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; SColumnInfoData colInfo = {0}; colInfo.info.bytes = pColSchema->bytes; colInfo.info.colId = pColSchema->colId; @@ -137,7 +128,40 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p } taosArrayPush(*ppCols, &colInfo); colMeta++; - colNeed++; + } + } else { + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); + if (*ppCols == NULL) { + return -1; + } + + int32_t colMeta = 0; + int32_t colNeed = 0; + while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + col_id_t colIdSchema = pColSchema->colId; + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pHandle->pColIdList, colNeed); + if (colIdSchema < colIdNeed) { + colMeta++; + } else if (colIdSchema > colIdNeed) { + colNeed++; + } else { + SColumnInfoData colInfo = {0}; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = pColSchema->colId; + colInfo.info.type = pColSchema->type; + + if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + goto FAIL; + } + taosArrayPush(*ppCols, &colInfo); + colMeta++; + colNeed++; + } } } diff --git a/source/dnode/vnode/src/tsdb/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c index 2753579e9e..8691920c8f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMain.c +++ b/source/dnode/vnode/src/tsdb/tsdbMain.c @@ -81,8 +81,8 @@ static STsdb *tsdbNew(const char *path, SVnode *pVnode, const STsdbCfg *pTsdbCfg static void tsdbFree(STsdb *pTsdb) { if (pTsdb) { - // tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb)); - // tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb)); + tsdbFreeSmaEnv(REPO_TSMA_ENV(pTsdb)); + tsdbFreeSmaEnv(REPO_RSMA_ENV(pTsdb)); tsdbFreeFS(pTsdb->fs); taosMemoryFreeClear(pTsdb->path); taosMemoryFree(pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f70a4478b3..483475601b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3223,8 +3223,13 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa tsdbDebug("data block generated, uid:%" PRIu64 " numOfRows:%d, tsrange:%" PRId64 " - %" PRId64 " %s", uid, cur->rows, cur->win.skey, cur->win.ekey, pHandle->idStr); - // pDataBlockInfo->uid = uid; // block Id may be over write by assigning uid fro this data block. Do NOT assign - // the table uid + pDataBlockInfo->uid = uid; + +#if 0 + // for multi-group data query processing test purpose + pDataBlockInfo->groupId = uid; +#endif + pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); @@ -3631,6 +3636,8 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch tsdbError("%p failed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; + } else { + tsdbDebug("%p succeed to get stable, uid:%" PRIu64 ", TID:0x%" PRIx64 " QID:0x%" PRIx64, pMeta, uid, taskId, reqId); } if (pTbCfg->type != META_SUPER_TABLE) { diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 273b7447ff..9143e8ed12 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -33,6 +33,8 @@ static const char *TSDB_SMA_DNAME[] = { #define SMA_TEST_INDEX_NAME "smaTestIndexName" // TODO: just for test #define SMA_TEST_INDEX_UID 2000000001 // TODO: just for test + +typedef struct SRSmaInfo SRSmaInfo; typedef enum { SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma @@ -46,6 +48,7 @@ typedef struct SPoolMem { struct SSmaEnv { TdThreadRwlock lock; + int8_t type; TXN txn; SPoolMem *pPool; SDiskID did; @@ -55,6 +58,7 @@ struct SSmaEnv { }; #define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_TYPE(env) ((env)->type) #define SMA_ENV_DID(env) ((env)->did) #define SMA_ENV_ENV(env) ((env)->dbEnv) #define SMA_ENV_PATH(env) ((env)->path) @@ -91,16 +95,45 @@ typedef struct { * - TSDB_SMA_STAT_EXPIRED: 1) If sma calculation of history TS data is not finished; 2) Or if the TSDB is open, * without information about its previous state. * - TSDB_SMA_STAT_DROPPED: 1)sma dropped + * N.B. only applicable to tsma */ int8_t state; // ETsdbSmaStat SHashObj *expiredWindows; // key: skey of time window, value: N/A STSma *pSma; // cache schema } SSmaStatItem; +#define RSMA_MAX_LEVEL 2 +#define RSMA_TASK_INFO_HASH_SLOT 8 +struct SRSmaInfo { + void *taskInfo[RSMA_MAX_LEVEL]; // qTaskInfo_t +}; + struct SSmaStat { - SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem + union { + SHashObj *smaStatItems; // key: indexUid, value: SSmaStatItem for tsma + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + }; T_REF_DECLARE() }; +#define SMA_STAT_ITEMS(s) ((s)->smaStatItems) +#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) + +static FORCE_INLINE void tsdbFreeTaskHandle(qTaskInfo_t *taskHandle) { + // Note: free/kill may in RC + qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); + if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + qDestroyTask(otaskHandle); + } +} + +static FORCE_INLINE void *tsdbFreeRSmaInfo(SRSmaInfo *pInfo) { + for (int32_t i = 0; i < RSMA_MAX_LEVEL; ++i) { + if (pInfo->taskInfo[i]) { + tsdbFreeTaskHandle(pInfo->taskInfo[i]); + } + } + return NULL; +} // declaration of static functions @@ -108,11 +141,11 @@ struct SSmaStat { static int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t version); static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); -static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat); +static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem); -static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat); -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did); -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv); +static int32_t tsdbDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did); +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv); static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t indexUid, TSKEY skey); static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); @@ -139,6 +172,7 @@ static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg); static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); +static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -229,7 +263,7 @@ static void *poolMalloc(void *arg, size_t size) { SPoolMem *pMem; pMem = (SPoolMem *)tdbOsMalloc(sizeof(*pMem) + size); - if (pMem == NULL) { + if (!pMem) { assert(0); } @@ -317,15 +351,17 @@ static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]) { snprintf(dirName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, TSDB_SMA_DNAME[smaType]); } -static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) { +static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did) { SSmaEnv *pEnv = NULL; pEnv = (SSmaEnv *)taosMemoryCalloc(1, sizeof(SSmaEnv)); - if (pEnv == NULL) { + if (!pEnv) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + SMA_ENV_TYPE(pEnv) = smaType; + int code = taosThreadRwlockInit(&(pEnv->lock), NULL); if (code) { terrno = TAOS_SYSTEM_ERROR(code); @@ -334,15 +370,15 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) } ASSERT(path && (strlen(path) > 0)); - pEnv->path = strdup(path); - if (pEnv->path == NULL) { + SMA_ENV_PATH(pEnv) = strdup(path); + if (!SMA_ENV_PATH(pEnv)) { tsdbFreeSmaEnv(pEnv); return NULL; } - pEnv->did = did; + SMA_ENV_DID(pEnv) = did; - if (tsdbInitSmaStat(&pEnv->pStat) != TSDB_CODE_SUCCESS) { + if (tsdbInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { tsdbFreeSmaEnv(pEnv); return NULL; } @@ -354,7 +390,7 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) return NULL; } - if ((pEnv->pPool = openPool()) == NULL) { + if (!(pEnv->pPool = openPool())) { tsdbFreeSmaEnv(pEnv); return NULL; } @@ -362,14 +398,14 @@ static SSmaEnv *tsdbNewSmaEnv(const STsdb *pTsdb, const char *path, SDiskID did) return pEnv; } -static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaEnv **pEnv) { +static int32_t tsdbInitSmaEnv(STsdb *pTsdb, int8_t smaType, const char *path, SDiskID did, SSmaEnv **pEnv) { if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } - if (*pEnv == NULL) { - if ((*pEnv = tsdbNewSmaEnv(pTsdb, path, did)) == NULL) { + if (!(*pEnv)) { + if (!(*pEnv = tsdbNewSmaEnv(pTsdb, smaType, path, did))) { return TSDB_CODE_FAILED; } } @@ -385,7 +421,7 @@ static int32_t tsdbInitSmaEnv(STsdb *pTsdb, const char *path, SDiskID did, SSmaE */ void tsdbDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { - tsdbDestroySmaState(pSmaEnv->pStat); + tsdbDestroySmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); taosMemoryFreeClear(pSmaEnv->pStat); taosMemoryFreeClear(pSmaEnv->path); taosThreadRwlockDestroy(&(pSmaEnv->lock)); @@ -401,7 +437,7 @@ void *tsdbFreeSmaEnv(SSmaEnv *pSmaEnv) { } static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { - if (pStat == NULL) return 0; + if (!pStat) return 0; int ref = T_REF_INC(pStat); tsdbDebug("vgId:%d ref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref); @@ -409,17 +445,17 @@ static int32_t tsdbRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { } static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat) { - if (pStat == NULL) return 0; + if (!pStat) return 0; int ref = T_REF_DEC(pStat); tsdbDebug("vgId:%d unref sma stat:%p, val:%d", REPO_ID(pTsdb), pStat, ref); return 0; } -static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { +static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { ASSERT(pSmaStat != NULL); - if (*pSmaStat != NULL) { // no lock + if (*pSmaStat) { // no lock return TSDB_CODE_SUCCESS; } @@ -428,19 +464,31 @@ static int32_t tsdbInitSmaStat(SSmaStat **pSmaStat) { * 2. Currently, there is mutex lock when init SSmaEnv, thus no need add lock on SSmaStat, and please add lock if * tsdbInitSmaStat invoked in other multithread environment later. */ - if (*pSmaStat == NULL) { + if (!(*pSmaStat)) { *pSmaStat = (SSmaStat *)taosMemoryCalloc(1, sizeof(SSmaStat)); - if (*pSmaStat == NULL) { + if (!(*pSmaStat)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; } - (*pSmaStat)->smaStatItems = - taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (smaType == TSDB_SMA_TYPE_ROLLUP) { + SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - if ((*pSmaStat)->smaStatItems == NULL) { - taosMemoryFreeClear(*pSmaStat); - return TSDB_CODE_FAILED; + if (!SMA_STAT_INFO_HASH(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + SMA_STAT_ITEMS(*pSmaStat) = + taosHashInit(SMA_STATE_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + if (!SMA_STAT_ITEMS(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + } else { + ASSERT(0); } } return TSDB_CODE_SUCCESS; @@ -462,7 +510,7 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { } static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { - if (pSmaStatItem != NULL) { + if (pSmaStatItem) { tdDestroyTSma(pSmaStatItem->pSma); taosMemoryFreeClear(pSmaStatItem->pSma); taosHashCleanup(pSmaStatItem->expiredWindows); @@ -477,16 +525,28 @@ static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { * @param pSmaStat * @return int32_t */ -int32_t tsdbDestroySmaState(SSmaStat *pSmaStat) { +int32_t tsdbDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - void *item = taosHashIterate(pSmaStat->smaStatItems, NULL); - while (item != NULL) { - SSmaStatItem *pItem = *(SSmaStatItem **)item; - tsdbFreeSmaStatItem(pItem); - item = taosHashIterate(pSmaStat->smaStatItems, item); + if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + void *item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), NULL); + while (item) { + SSmaStatItem *pItem = *(SSmaStatItem **)item; + tsdbFreeSmaStatItem(pItem); + item = taosHashIterate(SMA_STAT_ITEMS(pSmaStat), item); + } + taosHashCleanup(SMA_STAT_ITEMS(pSmaStat)); + } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { + void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL); + while (infoHash) { + SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; + tsdbFreeRSmaInfo(pInfoHash); + infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash); + } + taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat)); + } else { + ASSERT(0); } - taosHashCleanup(pSmaStat->smaStatItems); } return TSDB_CODE_SUCCESS; } @@ -497,12 +557,12 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { // return if already init switch (smaType) { case TSDB_SMA_TYPE_TIME_RANGE: - if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_TSMA_ENV(pTsdb))) != NULL) { + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)))) { return TSDB_CODE_SUCCESS; } break; case TSDB_SMA_TYPE_ROLLUP: - if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_RSMA_ENV(pTsdb))) != NULL) { + if ((pEnv = (SSmaEnv *)atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)))) { return TSDB_CODE_SUCCESS; } break; @@ -515,7 +575,7 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { tsdbLockRepo(pTsdb); pEnv = (smaType == TSDB_SMA_TYPE_TIME_RANGE) ? atomic_load_ptr(&REPO_TSMA_ENV(pTsdb)) : atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)); - if (pEnv == NULL) { + if (!pEnv) { char rname[TSDB_FILENAME_LEN] = {0}; SDiskID did = {0}; @@ -531,7 +591,7 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { return TSDB_CODE_FAILED; } - if (tsdbInitSmaEnv(pTsdb, rname, did, &pEnv) != TSDB_CODE_SUCCESS) { + if (tsdbInitSmaEnv(pTsdb, smaType, rname, did, &pEnv) != TSDB_CODE_SUCCESS) { tsdbUnlockRepo(pTsdb); return TSDB_CODE_FAILED; } @@ -547,10 +607,10 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version) { SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); - if (pItem == NULL) { + if (!pItem) { // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later pItem = tsdbNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state - if (pItem == NULL) { + if (!pItem) { // Response to stream computing: OOM // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. return TSDB_CODE_FAILED; @@ -558,7 +618,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t // cache smaMeta STSma *pSma = metaGetSmaInfoByIndex(REPO_META(pTsdb), indexUid, true); - if (pSma == NULL) { + if (!pSma) { terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; taosHashCleanup(pItem->expiredWindows); taosMemoryFree(pItem); @@ -574,7 +634,7 @@ static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t taosMemoryFree(pItem); return TSDB_CODE_FAILED; } - } else if ((pItem = *(SSmaStatItem **)pItem) == NULL) { + } else if (!(pItem = *(SSmaStatItem **)pItem)) { terrno = TSDB_CODE_INVALID_PTR; return TSDB_CODE_FAILED; } @@ -634,7 +694,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); - TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL); + TASSERT(pEnv && pStat && pItemsHash); // basic procedure // TODO: optimization @@ -651,7 +711,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers while (true) { tGetSubmitMsgNext(&msgIter, &pBlock); - if (pBlock == NULL) break; + if (!pBlock) break; STSmaWrapper *pSW = NULL; STSma *pTSma = NULL; @@ -664,7 +724,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers while (true) { STSRow *row = tGetSubmitBlkNext(&blkIter); - if (row == NULL) { + if (!row) { tdFreeTSmaWrapper(pSW); break; } @@ -672,10 +732,10 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers if (pSW) { pSW = tdFreeTSmaWrapper(pSW); } - if ((pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) { + if (!(pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid))) { break; } - if ((pSW->number) <= 0 || (pSW->tSma == NULL)) { + if ((pSW->number) <= 0 || !pSW->tSma) { pSW = tdFreeTSmaWrapper(pSW); break; } @@ -721,10 +781,10 @@ static int32_t tsdbResetExpiredWindow(STsdb *pTsdb, SSmaStat *pStat, int64_t ind tsdbRefSmaStat(pTsdb, pStat); - if (pStat && pStat->smaStatItems) { - pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); } - if ((pItem != NULL) && ((pItem = *(SSmaStatItem **)pItem) != NULL)) { + if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { // pItem resides in hash buffer all the time unless drop sma index // TODO: multithread protect if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { @@ -934,7 +994,7 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; - ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); + ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); pSmaH->dFile.fid = fid; char tSmaFile[TSDB_FILENAME_LEN] = {0}; @@ -1004,6 +1064,8 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char STsdbCfg *pCfg = REPO_CFG(pTsdb); const SArray *pDataBlocks = (const SArray *)msg; + // TODO: destroy SSDataBlocks(msg) + // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus // the sma data would arrive ahead of the update-expired-window msg. if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { @@ -1011,7 +1073,7 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char return TSDB_CODE_FAILED; } - if (pDataBlocks == NULL) { + if (!pDataBlocks) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); return terrno; @@ -1029,11 +1091,11 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char tsdbRefSmaStat(pTsdb, pStat); - if (pStat && pStat->smaStatItems) { - pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); } - if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tsdbSmaStatIsDropped(pItem)) { terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; @@ -1061,9 +1123,8 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); - // key: skey + groupId - char smaKey[SMA_KEY_LEN] = {0}; - char dataBuf[512] = {0}; + char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId + char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? void *pDataBuf = NULL; int32_t sz = taosArrayGetSize(pDataBlocks); for (int32_t i = 0; i < sz; ++i) { @@ -1228,7 +1289,7 @@ static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { tsdbDebug("vgId:%d drop tSma local cache for %" PRIi64, REPO_ID(pTsdb), indexUid); SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem != NULL) || ((pItem = *(SSmaStatItem **)pItem) != NULL)) { + if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { if (tsdbSmaStatIsDropped(pItem)) { tsdbDebug("vgId:%d tSma stat is already dropped for %" PRIi64, REPO_ID(pTsdb), indexUid); return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode @@ -1284,19 +1345,13 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) { SSmaEnv *pEnv = atomic_load_ptr(&REPO_RSMA_ENV(pTsdb)); int64_t indexUid = SMA_TEST_INDEX_UID; - if (pEnv == NULL) { + if (!pEnv) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); return terrno; } - if (pEnv == NULL) { - terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert rSma data failed since pTSmaEnv is NULL", REPO_ID(pTsdb)); - return terrno; - } - - if (pDataBlocks == NULL) { + if (!pDataBlocks) { terrno = TSDB_CODE_INVALID_PTR; tsdbWarn("vgId:%d insert rSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); return terrno; @@ -1313,11 +1368,11 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) { tsdbRefSmaStat(pTsdb, pStat); - if (pStat && pStat->smaStatItems) { - pItem = taosHashGet(pStat->smaStatItems, &indexUid, sizeof(indexUid)); + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); } - if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL) || tsdbSmaStatIsDropped(pItem)) { + if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tsdbSmaStatIsDropped(pItem)) { terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_FAILED; @@ -1438,7 +1493,7 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { ++pReadH->smaFsIter.iter; } - if (pReadH->pDFile != NULL) { + if (pReadH->pDFile) { tsdbDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); return true; } @@ -1471,7 +1526,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, tsdbRefSmaStat(pTsdb, pStat); SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); - if ((pItem == NULL) || ((pItem = *(SSmaStatItem **)pItem) == NULL)) { + if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if // it's NULL. tsdbUnRefSmaStat(pTsdb, pStat); @@ -1484,7 +1539,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, int32_t nQueryWin = taosArrayGetSize(pQuerySKey); for (int32_t n = 0; n < nQueryWin; ++n) { TSKEY skey = taosArrayGet(pQuerySKey, n); - if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY)) != NULL) { + if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { // TODO: mark this window as expired. } } @@ -1500,7 +1555,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, return TSDB_CODE_FAILED; } - if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY)) != NULL) { + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { // TODO: mark this window as expired. tsdbDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, REPO_ID(pTsdb), querySKey, indexUid); @@ -1510,7 +1565,6 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, } STSma *pTSma = pItem->pSma; - #endif STSmaReadH tReadH = {0}; @@ -1535,7 +1589,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, void *result = NULL; int32_t valueSize = 0; - if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) { + if (!(result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); tsdbCloseDBF(&tReadH.dFile); @@ -1579,7 +1633,7 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { SSmaCfg vCreateSmaReq = {0}; - if (tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq) == NULL) { + if (!tDeserializeSVCreateTSmaReq(pMsg, &vCreateSmaReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; tsdbWarn("vgId:%d TDMT_VND_CREATE_SMA received but deserialize failed since %s", REPO_ID(pTsdb), terrstr(terrno)); return -1; @@ -1605,7 +1659,7 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { SVDropTSmaReq vDropSmaReq = {0}; - if (tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq) == NULL) { + if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -1633,6 +1687,395 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { return TSDB_CODE_SUCCESS; } +/** + * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. + * + * @param pTsdb + * @param pMeta + * @param pReq + * @return int32_t + */ +int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq) { + SRSmaParam *param = pReq->stbCfg.pRSmaParam; + + if (!param) { + tsdbDebug("vgId:%d return directly since no rollup for stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, + pReq->stbCfg.suid); + return TSDB_CODE_SUCCESS; + } + + if ((param->qmsg1Len == 0) && (param->qmsg2Len == 0)) { + tsdbWarn("vgId:%d no qmsg1/qmsg2 for rollup stable %s %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->stbCfg.suid); + return TSDB_CODE_SUCCESS; + } + + if (tsdbCheckAndInitSmaEnv(pTsdb, TSDB_SMA_TYPE_ROLLUP) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->stbCfg.suid, sizeof(tb_uid_t)); + if (pRSmaInfo) { + tsdbWarn("vgId:%d rsma info already exists for stb: %s, %" PRIi64, REPO_ID(pTsdb), pReq->name, pReq->stbCfg.suid); + return TSDB_CODE_SUCCESS; + } + + pRSmaInfo = (SRSmaInfo *)taosMemoryCalloc(1, sizeof(SRSmaInfo)); + if (!pRSmaInfo) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta); + if (!pReadHandle) { + taosMemoryFree(pRSmaInfo); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + SReadHandle handle = { + .reader = pReadHandle, + .meta = pMeta, + }; + + if (param->qmsg1) { + pRSmaInfo->taskInfo[0] = qCreateStreamExecTaskInfo(param->qmsg1, &handle); + if (!pRSmaInfo->taskInfo[0]) { + taosMemoryFree(pRSmaInfo); + taosMemoryFree(pReadHandle); + return TSDB_CODE_FAILED; + } + } + + if (param->qmsg2) { + pRSmaInfo->taskInfo[1] = qCreateStreamExecTaskInfo(param->qmsg2, &handle); + if (!pRSmaInfo->taskInfo[1]) { + taosMemoryFree(pRSmaInfo); + taosMemoryFree(pReadHandle); + return TSDB_CODE_FAILED; + } + } + + if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->stbCfg.suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != + TSDB_CODE_SUCCESS) { + return TSDB_CODE_FAILED; + } else { + tsdbDebug("vgId:%d register rsma info succeed for suid:%" PRIi64, REPO_ID(pTsdb), pReq->stbCfg.suid); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief store suid/[uids], prefer to use array and then hash + * + * @param pStore + * @param suid + * @param uid + * @return int32_t + */ +int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid) { + // prefer to store suid/uids in array + if ((suid == pStore->suid) || (pStore->suid == 0)) { + if (pStore->suid == 0) { + pStore->suid = suid; + } + if (uid) { + if (!pStore->tbUids) { + if (!(pStore->tbUids = taosArrayInit(1, sizeof(tb_uid_t)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + if (!taosArrayPush(pStore->tbUids, &uid)) { + return TSDB_CODE_FAILED; + } + } + } else { + // store other suid/uids in hash when multiple stable/table included in 1 batch of request + if (!pStore->uidHash) { + pStore->uidHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + if (!pStore->uidHash) { + return TSDB_CODE_FAILED; + } + } + if (uid) { + SArray *uidArray = taosHashGet(pStore->uidHash, &suid, sizeof(tb_uid_t)); + if (uidArray && ((uidArray = *(SArray **)uidArray))) { + taosArrayPush(uidArray, &uid); + } else { + SArray *pUidArray = taosArrayInit(1, sizeof(tb_uid_t)); + if (!pUidArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + if (!taosArrayPush(pUidArray, &uid)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), &pUidArray, sizeof(pUidArray)) != 0) { + return TSDB_CODE_FAILED; + } + } + } else { + if (taosHashPut(pStore->uidHash, &suid, sizeof(suid), NULL, 0) != 0) { + return TSDB_CODE_FAILED; + } + } + } + return TSDB_CODE_SUCCESS; +} + +void tsdbUidStoreDestory(STbUidStore *pStore) { + if (pStore) { + if (pStore->uidHash) { + if (pStore->tbUids) { + void *pIter = taosHashIterate(pStore->uidHash, NULL); + while (pIter) { + SArray *arr = *(SArray **)pIter; + taosArrayDestroy(arr); + pIter = taosHashIterate(pStore->uidHash, pIter); + } + } + taosHashCleanup(pStore->uidHash); + } + taosArrayDestroy(pStore->tbUids); + } +} + +void *tsdbUidStoreFree(STbUidStore *pStore) { + tsdbUidStoreDestory(pStore); + taosMemoryFree(pStore); + return NULL; +} + +/** + * @brief fetch suid/uids when create child tables of rollup SMA + * + * @param pTsdb + * @param ppStore + * @param suid + * @param uid + * @return int32_t + */ +int32_t tsdbFetchTbUidList(void *pTsdb, void **ppStore, void *suid, void *uid) { + SSmaEnv *pEnv = REPO_RSMA_ENV((STsdb *)pTsdb); + + // only applicable to rollup SMA ctables + if (!pEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SHashObj *infoHash = NULL; + if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + return TSDB_CODE_FAILED; + } + + // info cached when create rsma stable and return directly for non-rsma ctables + if (!taosHashGet(infoHash, suid, sizeof(tb_uid_t))) { + return TSDB_CODE_SUCCESS; + } + + if (!(*ppStore)) { + if (tsdbUidStoreInit((STbUidStore **)ppStore) != 0) { + return TSDB_CODE_FAILED; + } + } + + if (tsdbUidStorePut(*ppStore, *(tb_uid_t *)suid, (tb_uid_t *)uid) != 0) { + *ppStore = tsdbUidStoreFree(*ppStore); + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t tsdbUpdateTbUidListImpl(STsdb *pTsdb, tb_uid_t *suid, SArray *tbUids) { + SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + if (!suid || !tbUids) { + terrno = TSDB_CODE_INVALID_PTR; + tsdbError("vgId:%d failed to get rsma info for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + tsdbError("vgId:%d failed to get rsma info for uid:%" PRIi64, REPO_ID(pTsdb), *suid); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + return TSDB_CODE_FAILED; + } + + if (pRSmaInfo->taskInfo[0] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[0], tbUids, true) != 0)) { + tsdbError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + tsdbDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, REPO_ID(pTsdb), + pRSmaInfo->taskInfo[0], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } + + if (pRSmaInfo->taskInfo[1] && (qUpdateQualifiedTableId(pRSmaInfo->taskInfo[1], tbUids, true) != 0)) { + tsdbError("vgId:%d update tbUidList failed for uid:%" PRIi64 " since %s", REPO_ID(pTsdb), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + tsdbDebug("vgId:%d update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, REPO_ID(pTsdb), + pRSmaInfo->taskInfo[1], *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { + if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { + tsdbDebug("vgId:%d no need to update tbUids since empty uidStore", REPO_ID(pTsdb)); + tsdbUidStoreFree(pStore); + return TSDB_CODE_SUCCESS; + } + + if (tsdbUpdateTbUidListImpl(pTsdb, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) { + tsdbUidStoreFree(pStore); + return TSDB_CODE_FAILED; + } + + void *pIter = taosHashIterate(pStore->uidHash, NULL); + while (pIter) { + tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + SArray *pTbUids = *(SArray **)pIter; + + if (tsdbUpdateTbUidListImpl(pTsdb, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) { + taosHashCancelIterate(pStore->uidHash, pIter); + tsdbUidStoreFree(pStore); + return TSDB_CODE_FAILED; + } + + pIter = taosHashIterate(pStore->uidHash, pIter); + } + + tsdbUidStoreFree(pStore); + + return TSDB_CODE_SUCCESS; +} + +static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { + ASSERT(pMsg != NULL); + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SSubmitBlkIter blkIter = {0}; + STSRow *row = NULL; + + terrno = TSDB_CODE_SUCCESS; + // pMsg->length = htonl(pMsg->length); + // pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1; + while (true) { + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + + if (!pBlock) break; + tsdbUidStorePut(pStore, pBlock->suid, NULL); + } + + if (terrno != TSDB_CODE_SUCCESS) return -1; + return 0; +} + +int32_t tsdbExecuteRSma(STsdb *pTsdb, SMeta *pMeta, const void *pMsg, int32_t inputType, tb_uid_t *suid) { + SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); + if (!pEnv) { + // only applicable when rsma env exists + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaInfo *pRSmaInfo = NULL; + + pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + + if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + tsdbDebug("vgId:%d no rsma info for suid:%" PRIu64, REPO_ID(pTsdb), *suid); + return TSDB_CODE_SUCCESS; + } + + SArray *pResult = NULL; + + pResult = taosArrayInit(0, sizeof(SSDataBlock)); + if (!pResult) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (pRSmaInfo->taskInfo[0]) { + qSetStreamInput(pRSmaInfo->taskInfo[0], pMsg, inputType); + while (1) { + SSDataBlock *output; + uint64_t ts; + if (qExecTask(pRSmaInfo->taskInfo[0], &output, &ts) < 0) { + ASSERT(false); + } + if (!output) { + break; + } + taosArrayPush(pResult, output); + } + blockDebugShowData(pResult); + } + + // if (pRSmaInfo->taskInfo[1]) { + // qSetStreamInput(pRSmaInfo->taskInfo[1], pMsg, inputType); + // while (1) { + // SSDataBlock *output; + // uint64_t ts; + // if (qExecTask(pRSmaInfo->taskInfo[1], &output, &ts) < 0) { + // ASSERT(false); + // } + // if (!output) { + // break; + // } + // taosArrayPush(pResult, output); + // } + // blockDebugShowData(pResult); + // } + } + + return TSDB_CODE_SUCCESS; +} + +int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType) { + SSmaEnv *pEnv = REPO_RSMA_ENV(pTsdb); + if (!pEnv) { + // only applicable when rsma env exists + return TSDB_CODE_SUCCESS; + } + + if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + STbUidStore uidStore = {0}; + tsdbFetchSubmitReqSuids(pMsg, &uidStore); + + if (uidStore.suid != 0) { + tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, &uidStore.suid); + + void *pIter = taosHashIterate(uidStore.uidHash, NULL); + while (pIter) { + tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + tsdbExecuteRSma(pTsdb, pMeta, pMsg, inputType, pTbSuid); + pIter = taosHashIterate(uidStore.uidHash, pIter); + } + + tsdbUidStoreDestory(&uidStore); + } + } + return TSDB_CODE_SUCCESS; +} + #if 0 /** * @brief Get the start TS key of the last data block of one interval/sliding. @@ -1675,6 +2118,7 @@ int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg) { if ((code = tsdbInsertTSmaDataImpl(pTsdb, indexUid, msg)) < 0) { tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } + // TODO: destroy SSDataBlocks(msg) return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 2b10afa5ed..11136bc7ba 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -14,6 +14,7 @@ */ #include "vnodeInt.h" +#include "vnodeSync.h" int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) { SVnodeInfo info = {0}; @@ -171,6 +172,16 @@ void vnodeClose(SVnode *pVnode) { } } +// start the sync timer after the queue is ready +int32_t vnodeStart(SVnode *pVnode) { + vnodeSyncSetQ(pVnode, NULL); + vnodeSyncSetRpc(pVnode, NULL); + vnodeSyncStart(pVnode); + return 0; +} + +void vnodeStop(SVnode *pVnode) {} + int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 3747e1dbc7..6280542ffe 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -22,21 +22,21 @@ int vnodeQueryOpen(SVnode *pVnode) { void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); } int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { - STbCfg *pTbCfg = NULL; - STbCfg *pStbCfg = NULL; + STbCfg * pTbCfg = NULL; + STbCfg * pStbCfg = NULL; tb_uid_t uid; int32_t nCols; int32_t nTagCols; SSchemaWrapper *pSW = NULL; - STableMetaRsp *pTbMetaMsg = NULL; + STableMetaRsp * pTbMetaMsg = NULL; STableMetaRsp metaRsp = {0}; - SSchema *pTagSchema; + SSchema * pTagSchema; SRpcMsg rpcMsg; int msgLen = 0; int32_t code = 0; char tableFName[TSDB_TABLE_FNAME_LEN]; int32_t rspLen = 0; - void *pRsp = NULL; + void * pRsp = NULL; STableInfoReq infoReq = {0}; if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { @@ -142,6 +142,7 @@ _exit: rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; + rpcMsg.refId = pMsg->refId; rpcMsg.pCont = pRsp; rpcMsg.contLen = rspLen; rpcMsg.code = code; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index aa90a04478..e907158897 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -83,6 +83,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg case TDMT_VND_SUBMIT: pRsp->msgType = TDMT_VND_SUBMIT_RSP; vnodeProcessSubmitReq(pVnode, ptr, pRsp); + tsdbTriggerRSma(pVnode->pTsdb, pVnode->pMeta, ptr, STREAM_DATA_TYPE_SUBMIT_BLOCK); break; case TDMT_VND_MQ_VG_CHANGE: if (tqProcessVgChangeReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), @@ -101,7 +102,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg } } break; case TDMT_VND_CREATE_SMA: { // timeRangeSMA - if (tsdbCreateTSma(pVnode->pTsdb, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) { // TODO } @@ -135,7 +135,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { - vTrace("message in query queue is processing"); + vTrace("message in vnode query queue is processing"); SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config}; switch (pMsg->msgType) { @@ -277,24 +277,16 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) { SVCreateTbReq vCreateTbReq = {0}; tDeserializeSVCreateTbReq(pReq, &vCreateTbReq); - if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq)) < 0) { + if (metaCreateTable(pVnode->pMeta, &(vCreateTbReq), NULL) < 0) { // TODO return -1; } - // TODO: remove the debug log - SRSmaParam *param = vCreateTbReq.stbCfg.pRSmaParam; - if (param) { - printf("qmsg1 len = %d, body = %s\n", param->qmsg1 ? (int32_t)strlen(param->qmsg1) : 0, - param->qmsg1 ? param->qmsg1 : ""); - printf("qmsg1 len = %d, body = %s\n", param->qmsg2 ? (int32_t)strlen(param->qmsg2) : 0, - param->qmsg2 ? param->qmsg2 : ""); - } + tsdbRegisterRSma(pVnode->pTsdb, pVnode->pMeta, &vCreateTbReq); taosMemoryFree(vCreateTbReq.stbCfg.pSchema); taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); if (vCreateTbReq.stbCfg.pRSmaParam) { - taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds); taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1); taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2); taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); @@ -309,6 +301,13 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR SVCreateTbBatchRsp vCreateTbBatchRsp = {0}; tDeserializeSVCreateTbBatchReq(pReq, &vCreateTbBatchReq); int reqNum = taosArrayGetSize(vCreateTbBatchReq.pArray); + + STbDdlH ddlHandle = { + .ahandle = pVnode->pTsdb, + .result = NULL, + .fp = tsdbFetchTbUidList, + }; + for (int i = 0; i < reqNum; i++) { SVCreateTbReq *pCreateTbReq = taosArrayGet(vCreateTbBatchReq.pArray, i); @@ -324,7 +323,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR taosArrayPush(vCreateTbBatchRsp.rspList, &rsp); } - if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { + if (metaCreateTable(pVnode->pMeta, pCreateTbReq, &ddlHandle) < 0) { // TODO: handle error vError("vgId:%d, failed to create table: %s", TD_VID(pVnode), pCreateTbReq->name); } @@ -334,7 +333,6 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR taosMemoryFree(pCreateTbReq->stbCfg.pSchema); taosMemoryFree(pCreateTbReq->stbCfg.pTagSchema); if (pCreateTbReq->stbCfg.pRSmaParam) { - taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam->pFuncIds); taosMemoryFree(pCreateTbReq->stbCfg.pRSmaParam); } } else if (pCreateTbReq->type == TD_CHILD_TABLE) { @@ -342,12 +340,13 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR } else { taosMemoryFree(pCreateTbReq->ntbCfg.pSchema); if (pCreateTbReq->ntbCfg.pRSmaParam) { - taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam->pFuncIds); taosMemoryFree(pCreateTbReq->ntbCfg.pRSmaParam); } } } + tsdbUpdateTbUidList(pVnode->pTsdb, ddlHandle.result); + vTrace("vgId:%d process create %" PRIzu " tables", TD_VID(pVnode), taosArrayGetSize(vCreateTbBatchReq.pArray)); taosArrayDestroy(vCreateTbBatchReq.pArray); if (vCreateTbBatchRsp.rspList) { @@ -371,7 +370,6 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq) { taosMemoryFree(vAlterTbReq.stbCfg.pSchema); taosMemoryFree(vAlterTbReq.stbCfg.pTagSchema); if (vAlterTbReq.stbCfg.pRSmaParam) { - taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam->pFuncIds); taosMemoryFree(vAlterTbReq.stbCfg.pRSmaParam); } taosMemoryFree(vAlterTbReq.name); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 546e925106..5f8bf3c7b5 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -113,7 +113,7 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex); syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg); - SVnode *pVnode = (SVnode *)(pFsm->data); + SVnode * pVnode = (SVnode *)(pFsm->data); SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta); SRpcMsg applyMsg; syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg); @@ -133,6 +133,7 @@ void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cb if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) { applyMsg.handle = saveRpcMsg.handle; applyMsg.ahandle = saveRpcMsg.ahandle; + applyMsg.refId = saveRpcMsg.refId; } else { applyMsg.handle = NULL; applyMsg.ahandle = NULL; diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index b0217a0462..ab617cb186 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -407,7 +407,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { } } - EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS); + // EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS); EXPECT_EQ(tsdbUpdateSmaWindow(pTsdb, pMsg, 0), 0); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 410527c9e6..272c370ca3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -494,7 +494,7 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray **out) { +int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) { char *msg = NULL; int32_t msgLen = 0; @@ -526,7 +526,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt CTG_ERR_RET(code); } - ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(*out)); + ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); return TSDB_CODE_SUCCESS; } @@ -2778,7 +2778,8 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } if (pReq->qNodeRequired) { - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, &pRsp->pEpSetList)); + pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList)); } CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -2807,7 +2808,7 @@ int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, &pQnodeList)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList)); _return: diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 4853bb4eb3..d17a5a59a4 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -304,8 +304,8 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t memcpy(row.buf, tbuf, len); row.level = level; - row.len = len; - ctx->dataSize += len; + row.len = len; + ctx->dataSize += row.len; if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); @@ -344,11 +344,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); - if (pTagScanNode->reverse) { - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); - } EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -361,10 +356,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen)); if (tlen) { @@ -388,11 +379,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); - if (pTblScanNode->scan.reverse) { - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); - } EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -405,10 +391,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -434,11 +416,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); - if (pSTblScanNode->scan.reverse) { - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); - } EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -451,10 +428,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pSTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); @@ -783,7 +756,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } int32_t colNum = 1; - int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); @@ -793,29 +766,38 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - *(int32_t *)rsp->data = htonl(pCtx->dataSize); + // payload length + *(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; - int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); + // group id + *(uint64_t*)(rsp->data + sizeof(int32_t)) = 0; + + // column length + int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); + + // varchar column offset segment + int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t)); + + // varchar data real payload char *data = (char *)(offset + rowNum); - int32_t tOffset = 0; - + + char* start = data; for (int32_t i = 0; i < rowNum; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); - *offset = tOffset; - tOffset += row->len; + offset[i] = data - start; - memcpy(data, row->buf, row->len); - - ++offset; + varDataCopy(data, row->buf); + ASSERT(varDataTLen(row->buf) == row->len); data += row->len; } - *pRsp = rsp; + *colLength = htonl(data - start); + rsp->compLen = htonl(rspSize); + *pRsp = rsp; return TSDB_CODE_SUCCESS; } - int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { int32_t code = 0; SNodeListNode *plans = NULL; @@ -922,9 +904,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0)); - QRY_ERR_RET(qExplainAppendPlanRows(pCtx)); - QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp)); return TSDB_CODE_SUCCESS; @@ -994,13 +974,10 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { SExplainCtx *pCtx = NULL; QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx)); - QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp)); _return: - qExplainFreeCtx(pCtx); - QRY_RET(code); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 834d37927a..f0cb1c1107 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -40,8 +40,6 @@ #define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) -#define curTimeWindowIndex(_winres) ((_winres)->curIndex) - typedef struct SGroupResInfo { int32_t totalGroup; int32_t currentGroup; @@ -68,11 +66,16 @@ typedef struct SResultRowPosition { int32_t offset; } SResultRowPosition; +typedef struct SResKeyPos { + SResultRowPosition pos; + uint64_t groupId; + char key[]; +} SResKeyPos; + typedef struct SResultRowInfo { SResultRowPosition *pPosition; int32_t size; // number of result set int32_t capacity; // max capacity -// int32_t curPos; // current active result row index of pResult list SResultRowPosition cur; } SResultRowInfo; @@ -135,7 +138,7 @@ typedef struct { int32_t colId; } SStddevInterResult; -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c6d209e706..363824f2e2 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -347,10 +347,11 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; + SColumnInfo *pCols; + SSDataBlock *pRes; int32_t totalTables; int32_t curPos; + void *pReader; } STagScanInfo; typedef struct SStreamBlockScanInfo { @@ -376,13 +377,11 @@ typedef struct SSysTableScanInfo { SEpSet epSet; tsem_t ready; - int32_t accountId; - bool showRewrite; - SNode* pCondition; // db_name filter condition, to discard data that are not in current database - void* pCur; // cursor for iterate the local table meta store. - SArray* scanCols; // SArray scan column id list - -// int32_t type; // show type, TODO remove it + int32_t accountId; + bool showRewrite; + SNode* pCondition; // db_name filter condition, to discard data that are not in current database + void* pCur; // cursor for iterate the local table meta store. + SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; int32_t capacity; @@ -400,8 +399,8 @@ typedef struct SOptrBasicInfo { // TODO move the resultrowsiz together with SOptrBasicInfo:rowCellInfoOffset typedef struct SAggSupporter { SHashObj* pResultRowHashTable; // quick locate the window object for each result - SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not - SArray* pResultRowArrayList; // The array list that contains the Result rows +// SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not +// SArray* pResultRowArrayList; // The array list that contains the Result rows char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row @@ -628,7 +627,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime, +SOperatorInfo* createTableScanOperatorInfo(void* pReaderHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double ratio, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, @@ -668,12 +667,12 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTagScanOperatorInfo(SReaderHandle* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput); #endif void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index e897bc8892..a5ce01dba2 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -64,10 +64,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { } // data format: -// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... -// | | sizeof(int32_t) * numOfCols | actual size | | actual size | | -// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... +// | | (4 bytes) |(8 bytes) | sizeof(int32_t) * numOfCols | actual size | | actual size | | +// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 98b72cf4c2..c3fa777779 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -186,12 +186,50 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->index = 0; } -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) { +static int32_t resultrowCompar1(const void* p1, const void* p2) { + SResKeyPos* pp1 = *(SResKeyPos**) p1; + SResKeyPos* pp2 = *(SResKeyPos**) p2; + + if (pp1->groupId == pp2->groupId) { + int64_t pts1 = *(int64_t*) pp1->key; + int64_t pts2 = *(int64_t*) pp2->key; + + if (pts1 == pts2) { + return 0; + } else { + return pts1 < pts2? -1:1; + } + } else { + return pp1->groupId < pp2->groupId? -1:1; + } +} + +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult) { if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } - pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pPosition, pResultInfo->size, sizeof(SResultRowPosition)); + // extract the result rows information from the hash map + void* pData = NULL; + pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); + + size_t keyLen = 0; + while((pData = taosHashIterate(pHashmap, pData)) != NULL) { + void* key = taosHashGetKey(pData, &keyLen); + + SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition)); + + p->groupId = *(uint64_t*) key; + p->pos = *(SResultRowPosition*) pData; + memcpy(p->key, key + sizeof(uint64_t), keyLen - sizeof(uint64_t)); + + taosArrayPush(pGroupResInfo->pRows, &p); + } + + if (sortGroupResult) { + qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, resultrowCompar1); + } + pGroupResInfo->index = 0; assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4a0206f55e..434591c4c3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -18,8 +18,8 @@ #include "functionMgt.h" #include "os.h" #include "querynodes.h" -#include "tname.h" #include "tfill.h" +#include "tname.h" #include "tdatablock.h" #include "tglobal.h" @@ -32,8 +32,8 @@ #include "tcompare.h" #include "tcompression.h" #include "thash.h" -#include "vnode.h" #include "ttypes.h" +#include "vnode.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -59,7 +59,6 @@ typedef enum SResultTsInterpType { RESULT_ROW_END_INTERP = 2, } SResultTsInterpType; - #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -230,13 +229,14 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { void operatorDummyCloseFn(void* param, int32_t numOfCols) {} -static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx); +static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, int32_t orderType, int32_t* rowCellOffset, + SqlFunctionCtx* pCtx); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); -static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); -static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo); +static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, + SExecTaskInfo* pTaskInfo); SArray* getOrderCheckColumns(STaskAttr* pQuery); @@ -268,11 +268,10 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = numOfCols; pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); pBlock->info.blockId = pNode->dataBlockId; - pBlock->info.rowSize = pNode->totalRowSize; // todo ?? + pBlock->info.rowSize = pNode->totalRowSize; // todo ?? for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {{0}}; @@ -281,7 +280,7 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { continue; } - idata.info.type = pDescNode->dataType.type; + idata.info.type = pDescNode->dataType.type; idata.info.bytes = pDescNode->dataType.bytes; idata.info.scale = pDescNode->dataType.scale; idata.info.slotId = pDescNode->slotId; @@ -294,31 +293,13 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { taosArrayPush(pBlock->pDataBlock, &idata); } + pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); return pBlock; } -static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - return true; - // bool hasTags = false; - // int32_t numOfSelectivity = 0; - // - // for (int32_t i = 0; i < numOfOutput; ++i) { - // int32_t functId = pCtx[i].functionId; - // if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) { - // hasTags = true; - // continue; - // } - // - // if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) { - // numOfSelectivity++; - // } - // } - // - // return (numOfSelectivity > 0 && hasTags); -} - static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) { - if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || + pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { return false; } @@ -343,11 +324,16 @@ static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); } - if (newCapacity == pResultRowInfo->capacity) { + if (newCapacity <= pResultRowInfo->capacity) { newCapacity += 4; } - pResultRowInfo->pPosition = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); + char* p = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); + if (p == NULL) { + longjmp(env, TSDB_CODE_OUT_OF_MEMORY); + } + + pResultRowInfo->pPosition = (SResultRowPosition*)p; int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; memset(&pResultRowInfo->pPosition[pResultRowInfo->capacity], 0, sizeof(SResultRowPosition) * inc); @@ -372,7 +358,7 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR if (pResultRowInfo->size == 0) { existed = false; } else if (pResultRowInfo->size == 1) { -// existed = (pResultRowInfo->pResult[0] == (*p1)); + // existed = (pResultRowInfo->pResult[0] == (*p1)); } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid, pResultRowInfo); int64_t* index = @@ -431,82 +417,63 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return pResultRow; } -static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid, +/** + * the struct of key in hash table + * +----------+---------------+ + * | group id | key data | + * | 8 bytes | actual length | + * +----------+---------------+ + */ +static SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid, char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { - bool existInCurrentResusltRowInfo = false; SET_RES_WINDOW_KEY(pSup->keyBuf, pData, bytes, groupId); - SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + SResultRowPosition* p1 = + (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + + SResultRow* pResult = NULL; // in case of repeat scan/reverse scan, no new time window added. if (isIntervalQuery) { - if (!masterscan) { // the *p1 may be NULL in case of sliding+offset exists. - if (p1 != NULL) { - return getResultRowByPos(pResultBuf, p1); - } else { - return NULL; - } - } - - if (p1 != NULL) { - if (pResultRowInfo->size == 0) { - existInCurrentResusltRowInfo = false; // this time window created by other timestamp that does not belongs to current table. - } else if (pResultRowInfo->size == 1) { - SResultRowPosition* p = &pResultRowInfo->pPosition[0]; - existInCurrentResusltRowInfo = (p->pageId == p1->pageId && p->offset == p1->offset); - } else { // check if current pResultRowInfo contains the existInCurrentResusltRowInfo pResultRow - SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, uid, pResultRowInfo); - int64_t* index = taosHashGet(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); - if (index != NULL) { - // TODO check the scan order for current opened time window - existInCurrentResusltRowInfo = true; - } else { - existInCurrentResusltRowInfo = false; - } - } - } - } else { - // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the pResultRowInfo object. - if (p1 != NULL) { - return getResultRowByPos(pResultBuf, p1); - } - } - - SResultRow* pResult = NULL; - if (!existInCurrentResusltRowInfo) { - // 1. close current opened time window - if (pResultRowInfo->cur.pageId != -1) { // todo extract function - SResultRowPosition pos = pResultRowInfo->cur; - SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)pPage + pos.offset); - closeResultRow(pRow); - releaseBufPage(pResultBuf, pPage); - } - - prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); - if (p1 == NULL) { - pResult = getNewResultRow_rv(pResultBuf, groupId, pSup->resultRowSize); - initResultRow(pResult); - - // add a new result set for a new group - SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; - taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, sizeof(SResultRowPosition)); - SResultRowCell cell = {.groupId = groupId, .pos = pos}; - taosArrayPush(pSup->pResultRowArrayList, &cell); - } else { + if (masterscan && p1 != NULL) { // the *p1 may be NULL in case of sliding+offset exists. pResult = getResultRowByPos(pResultBuf, p1); } - - // 2. set the new time window to be the new active time window - pResultRowInfo->pPosition[pResultRowInfo->size++] = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - SET_RES_EXT_WINDOW_KEY(pSup->keyBuf, pData, bytes, uid, pResultRowInfo); - taosHashPut(pSup->pResultRowListSet, pSup->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &pResultRowInfo->cur, POINTER_BYTES); } else { - pResult = getResultRowByPos(pResultBuf, p1); + // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the + // pResultRowInfo object. + if (p1 != NULL) { + pResult = getResultRowByPos(pResultBuf, p1); + } } + // 1. close current opened time window + if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId && + pResult->offset != pResultRowInfo->cur.offset))) { + // todo extract function + SResultRowPosition pos = pResultRowInfo->cur; + SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)pPage + pos.offset); + closeResultRow(pRow); + releaseBufPage(pResultBuf, pPage); + } + + // allocate a new buffer page + prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); + if (pResult == NULL) { + pResult = getNewResultRow_rv(pResultBuf, groupId, pSup->resultRowSize); + initResultRow(pResult); + + // add a new result set for a new group + SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, sizeof(SResultRowPosition)); + } + + // 2. set the new time window to be the new active time window + pResultRowInfo->pPosition[pResultRowInfo->size++] = + (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + // too many time window in query if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); @@ -515,7 +482,8 @@ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultR return pResult; } -static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, bool ascQuery) { +static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, TSKEY ts, STimeWindow* w, + bool ascQuery) { if (ascQuery) { getAlignQueryTimeWindow(pInterval, precision, ts, w); } else { @@ -535,8 +503,8 @@ static void getInitialStartTimeWindow(SInterval* pInterval, int32_t precision, T } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SDiskbasedBuf * pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, - int32_t precision, STimeWindow* win) { +static STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, + SInterval* pInterval, int32_t precision, STimeWindow* win) { STimeWindow w = {0}; if (pResultRowInfo->cur.pageId == -1) { // the first window, from the previous stored value @@ -653,14 +621,15 @@ static bool chkWindowOutputBufByKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo return chkResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char*)&win->skey, TSDB_KEYSIZE, masterscan, groupId); } -static void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset); +static void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, + int32_t* rowCellInfoOffset); static int32_t setResultOutputBufByKey_rv(SResultRowInfo* pResultRowInfo, int64_t id, STimeWindow* win, bool masterscan, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset, SAggSupporter* pAggSup, SExecTaskInfo* pTaskInfo) { assert(win->skey <= win->ekey); - SResultRow* pResultRow = doSetResultOutBufByKey_rv(pAggSup->pResultBuf, pResultRowInfo, id, (char*)&win->skey, + SResultRow* pResultRow = doSetResultOutBufByKey(pAggSup->pResultBuf, pResultRowInfo, id, (char*)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId, pTaskInfo, true, pAggSup); if (pResultRow == NULL) { @@ -721,7 +690,8 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se return forwardStep; } -static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, bool timeWindowInterpo) { +static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey, bool ascQuery, + bool timeWindowInterpo) { int64_t skey = TSKEY_INITIAL_VAL; #if 0 int32_t i = 0; @@ -778,7 +748,7 @@ static void doUpdateResultRowIndex(SResultRowInfo* pResultRowInfo, TSKEY lastKey #endif } // -//static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, +// static void updateResultRowInfoActiveIndex(SResultRowInfo* pResultRowInfo, const STimeWindow* pWin, TSKEY lastKey, // bool ascQuery, bool interp) { // if ((lastKey > pWin->ekey && ascQuery) || (lastKey < pWin->ekey && (!ascQuery))) { // closeAllResultRows(pResultRowInfo); @@ -844,22 +814,22 @@ static void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQuer static void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, bool includeEndpoint) { int64_t* ts = (int64_t*)pColData->pData; - int32_t delta = includeEndpoint? 1:0; + int32_t delta = includeEndpoint ? 1 : 0; int64_t duration = pWin->ekey - pWin->skey + delta; - ts[2] = duration; // set the duration - ts[3] = pWin->skey; // window start key - ts[4] = pWin->ekey + delta; // window end key + ts[2] = duration; // set the duration + ts[3] = pWin->skey; // window start key + ts[4] = pWin->ekey + delta; // window end key } -void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, - int32_t numOfTotal, int32_t numOfOutput, int32_t order) { +void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, + int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { pCtx[k].startTs = pWin->skey; // keep it temporarily - bool hasAgg = pCtx[k].input.colDataAggIsSet; - int32_t numOfRows = pCtx[k].input.numOfRows; + bool hasAgg = pCtx[k].input.colDataAggIsSet; + int32_t numOfRows = pCtx[k].input.numOfRows; int32_t startOffset = pCtx[k].input.startRowIndex; int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1); @@ -878,12 +848,12 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* if (fmIsWindowPseudoColumnFunc(pCtx[k].functionId)) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); - char* p = GET_ROWCELL_INTERBUF(pEntryInfo); + char* p = GET_ROWCELL_INTERBUF(pEntryInfo); SColumnInfoData idata = {0}; - idata.info.type = TSDB_DATA_TYPE_BIGINT; + idata.info.type = TSDB_DATA_TYPE_BIGINT; idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - idata.pData = p; + idata.pData = p; SScalarParam out = {.columnData = &idata}; SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; @@ -975,7 +945,7 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, } static FORCE_INLINE TSKEY reviseWindowEkey(STaskAttr* pQueryAttr, STimeWindow* pWindow) { - TSKEY ekey = -1; + TSKEY ekey = -1; int32_t order = TSDB_ORDER_ASC; if (order == TSDB_ORDER_ASC) { ekey = pWindow->ekey; @@ -1027,9 +997,11 @@ static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols, int32_t rows, return ts; } -static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol); +static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + bool createDummyCol); -static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order) { +static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, + int32_t order) { for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { pCtx[i].order = order; pCtx[i].size = pBlock->info.rows; @@ -1037,7 +1009,8 @@ static void doSetInputDataBlockInfo(SOperatorInfo* pOperator, SqlFunctionCtx* pC } } -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol) { +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + bool createDummyCol) { if (pBlock->pBlockAgg != NULL) { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } else { @@ -1045,7 +1018,8 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo } } -static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, int32_t paramIndex, int32_t numOfRows) { +static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, + int32_t paramIndex, int32_t numOfRows) { SColumnInfoData* pColInfo = NULL; if (pInput->pData[paramIndex] == NULL) { pColInfo = taosMemoryCalloc(1, sizeof(SColumnInfoData)); @@ -1054,10 +1028,12 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } // Set the correct column info (data type and bytes) - pColInfo->info.type = type; - pColInfo->info.bytes = tDataTypes[type].bytes; + pColInfo->info.type = type; + pColInfo->info.bytes = tDataTypes[type].bytes; pInput->pData[paramIndex] = pColInfo; + } else { + pColInfo = pInput->pData[paramIndex]; } ASSERT(!IS_VAR_DATA_TYPE(type)); @@ -1065,12 +1041,12 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { int64_t v = pFuncParam->param.i; - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { colDataAppendInt64(pColInfo, i, &v); } } else if (type == TSDB_DATA_TYPE_DOUBLE) { double v = pFuncParam->param.d; - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { colDataAppendDouble(pColInfo, i, &v); } } @@ -1078,33 +1054,40 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc return TSDB_CODE_SUCCESS; } -static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, bool createDummyCol) { +static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + bool createDummyCol) { int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].pSrcBlock = pBlock; pCtx[i].currentStage = MAIN_SCAN; SInputColumnInfoData* pInput = &pCtx[i].input; pInput->uid = pBlock->info.uid; + pInput->colDataAggIsSet = false; SExprInfo* pOneExpr = &pOperator->pExpr[i]; for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { - SFunctParam *pFuncParam = &pOneExpr->base.pParam[j]; + SFunctParam* pFuncParam = &pOneExpr->base.pParam[j]; if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; - pInput->pData[j] = taosArrayGet(pBlock->pDataBlock, slotId); + pInput->pData[j] = taosArrayGet(pBlock->pDataBlock, slotId); pInput->totalRows = pBlock->info.rows; pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; - pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column + pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { // todo avoid case: top(k, 12), 12 is the value parameter. // sum(11), 11 is also the value parameter. if (createDummyCol && pOneExpr->base.numOfParams == 1) { + pInput->totalRows = pBlock->info.rows; + pInput->numOfRows = pBlock->info.rows; + pInput->startRowIndex = 0; + code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1166,23 +1149,25 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunction } static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) { - size_t num = (pPseudoList != NULL)? taosArrayGetSize(pPseudoList):0; + size_t num = (pPseudoList != NULL) ? taosArrayGetSize(pPseudoList) : 0; for (int32_t i = 0; i < num; ++i) { pCtx[i].pOutput = taosArrayGet(pResult->pDataBlock, i); } } -void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList) { +void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, + int32_t numOfOutput, SArray* pPseudoList) { setPseudoOutputColInfo(pResult, pCtx, pPseudoList); pResult->info.groupId = pSrcBlock->info.groupId; - // if the source equals to the destination, it is to create a new column as the result of scalar function or some operators. + // if the source equals to the destination, it is to create a new column as the result of scalar function or some + // operators. bool createNewColModel = (pResult == pSrcBlock); int32_t numOfRows = 0; for (int32_t k = 0; k < numOfOutput; ++k) { - int32_t outputSlotId = pExpr[k].base.resSchema.slotId; + int32_t outputSlotId = pExpr[k].base.resSchema.slotId; SqlFunctionCtx* pfCtx = &pCtx[k]; if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query @@ -1197,9 +1182,11 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) { SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId); - int32_t offset = createNewColModel? 0: pResult->info.rows; + int32_t offset = createNewColModel ? 0 : pResult->info.rows; for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i + offset, taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].param.nType), TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType); + colDataAppend(pColInfoData, i + offset, + taosVariantGet(&pExpr[k].base.pParam[0].param, pExpr[k].base.pParam[0].param.nType), + TSDB_DATA_TYPE_NULL == pExpr[k].base.pParam[0].param.nType); } numOfRows = pSrcBlock->info.rows; @@ -1208,12 +1195,12 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* taosArrayPush(pBlockList, &pSrcBlock); SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId); - SColumnInfoData idata = {.info = pResColData->info}; + SColumnInfoData idata = {.info = pResColData->info}; SScalarParam dest = {.columnData = &idata}; scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest); - int32_t startOffset = createNewColModel? 0:pResult->info.rows; + int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); numOfRows = dest.numOfRows; @@ -1227,11 +1214,11 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* // todo set the correct timestamp column pfCtx->input.pPTS = taosArrayGet(pSrcBlock->pDataBlock, 1); - SResultRowEntryInfo *pResInfo = GET_RES_INFO(&pCtx[k]); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[k]); pfCtx->fpSet.init(&pCtx[k], pResInfo); pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId); - pfCtx->offset = createNewColModel? 0:pResult->info.rows; // set the start offset + pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset // set the timestamp(_rowts) output buffer if (taosArrayGetSize(pPseudoList) > 0) { @@ -1245,12 +1232,12 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* taosArrayPush(pBlockList, &pSrcBlock); SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId); - SColumnInfoData idata = {.info = pResColData->info}; + SColumnInfoData idata = {.info = pResColData->info}; SScalarParam dest = {.columnData = &idata}; scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest); - int32_t startOffset = createNewColModel? 0:pResult->info.rows; + int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; colDataMergeCol(pResColData, startOffset, &idata, dest.numOfRows); numOfRows = dest.numOfRows; @@ -1268,7 +1255,7 @@ void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SExprInfo* pExpr = pOperator->pExpr; + SExprInfo* pExpr = pOperator->pExpr; SqlFunctionCtx* pCtx = pInfo->pCtx; @@ -1287,7 +1274,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, double v1 = 0, v2 = 0, v = 0; if (prevRowIndex == -1) { -// GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); + // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); } else { GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); } @@ -1304,7 +1291,7 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { if (prevRowIndex == -1) { -// pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; + // pCtx[k].start.ptr = (char*)pRuntimeEnv->prevRow[index]; } else { pCtx[k].start.ptr = (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes; } @@ -1331,10 +1318,11 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, STimeWindow* win) { + int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, + STimeWindow* win) { bool ascQuery = true; TSKEY curTs = tsCols[pos]; - TSKEY lastTs = 0;//*(TSKEY*)pRuntimeEnv->prevRow[0]; + TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. // start exactly from this point, no need to do interpolation @@ -1349,10 +1337,11 @@ static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlF return true; } - int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); + doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, + RESULT_ROW_START_INTERP); return true; } @@ -1366,7 +1355,8 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun TSKEY key = order ? win->ekey : win->skey; // not ended in current data block, do not invoke interpolation - if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { + if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || + (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); return false; } @@ -1409,7 +1399,7 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc if (!done) { // it is not interpolated, now start to generated the interpolated value int32_t startRowIndex = startPos; bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, - pBlock->pDataBlock, tsCols, win); + pBlock->pDataBlock, tsCols, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } @@ -1438,7 +1428,8 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc } } -static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, int32_t tableGroupId) { +static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pSDataBlock, + int32_t tableGroupId) { STableIntervalOperatorInfo* pInfo = (STableIntervalOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -1446,38 +1437,41 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe SArray* pUpdated = NULL; if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - pUpdated = taosArrayInit(4, sizeof(SResultRowPosition)); + pUpdated = taosArrayInit(4, POINTER_BYTES); } int32_t step = 1; bool ascScan = true; -// int32_t prevIndex = pResultRowInfo->curPos; + // int32_t prevIndex = pResultRowInfo->curPos; TSKEY* tsCols = NULL; if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; -// assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] == -// pSDataBlock->info.window.ekey); } - int32_t startPos = ascScan? 0 : (pSDataBlock->info.rows - 1); + int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols, pSDataBlock->info.rows, ascScan); - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, + pInfo->interval.precision, &pInfo->win); bool masterScan = true; SResultRow* pResult = NULL; int32_t ret = setResultOutputBufByKey_rv(pResultRowInfo, pSDataBlock->info.uid, &win, masterScan, &pResult, - tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, - &pInfo->aggSup, pTaskInfo); + tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, + &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*) pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); } @@ -1487,7 +1481,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // prev time window not interpolation yet. -// int32_t curIndex = pResultRowInfo->curPos; + // int32_t curIndex = pResultRowInfo->curPos; #if 0 if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { @@ -1527,10 +1521,12 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe #endif // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, pInfo->order, false); + doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, + pInfo->order, false); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); STimeWindow nextWin = win; while (1) { @@ -1549,7 +1545,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*) pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); } @@ -1562,7 +1562,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe pInfo->order, false); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } if (pInfo->timeWindowInterpo) { @@ -1575,15 +1576,15 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts) { - pRowSup->win.ekey = ts; - pRowSup->prevTs = ts; + pRowSup->win.ekey = ts; + pRowSup->prevTs = ts; pRowSup->numOfRows += 1; } static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsList, int32_t rowIndex) { pRowSup->startRowIndex = rowIndex; - pRowSup->numOfRows = 0; - pRowSup->win.skey = tsList[rowIndex]; + pRowSup->numOfRows = 0; + pRowSup->win.skey = tsList[rowIndex]; } // todo handle multiple tables cases. @@ -1627,14 +1628,16 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator pRowSup->win.ekey = pRowSup->win.skey; int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + &pResult, gid, pInfo->binfo.pCtx, numOfOutput, + pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } // pInfo->numOfRows data belong to the current session window updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); // here we start a new session window doKeepNewWindowStartInfo(pRowSup, tsList, j); @@ -1644,25 +1647,27 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator SResultRow* pResult = NULL; pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, &pResult, - gid, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, + &pResult, gid, pInfo->binfo.pCtx, numOfOutput, + pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { if (IS_VAR_DATA_TYPE(type)) { // todo disable this -// if (pResultRow->key == NULL) { -// pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); -// varDataCopy(pResultRow->key, pData); -// } else { -// ASSERT(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); -// } + // if (pResultRow->key == NULL) { + // pResultRow->key = taosMemoryMalloc(varDataTLen(pData)); + // varDataCopy(pResultRow->key, pData); + // } else { + // ASSERT(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); + // } } else { int64_t v = -1; GET_TYPED_DATA(v, int64_t, type, pData); @@ -1673,11 +1678,12 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { } int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes, - int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup) { + int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, + SAggSupporter* pAggSup) { SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = binfo->pCtx; - SResultRow* pResultRow = doSetResultOutBufByKey_rv(pBuf, pResultRowInfo, groupId, (char*)pData, bytes, true, groupId, + SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, groupId, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup); assert(pResultRow != NULL); @@ -1717,7 +1723,8 @@ static bool functionNeedToExecute(SqlFunctionCtx* pCtx) { return true; } -static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, int32_t paramIndex, int32_t numOfRows) { +static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, + int32_t paramIndex, int32_t numOfRows) { if (pInput->pData[paramIndex] == NULL) { pInput->pData[paramIndex] = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pInput->pData[paramIndex] == NULL) { @@ -1725,8 +1732,8 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF } // Set the correct column info (data type and bytes) - pInput->pData[paramIndex]->info.type = type; - pInput->pData[paramIndex]->info.bytes = tDataTypes[type].bytes; + pInput->pData[paramIndex]->info.type = type; + pInput->pData[paramIndex]->info.bytes = tDataTypes[type].bytes; } SColumnDataAgg* da = NULL; @@ -1744,21 +1751,21 @@ static int32_t doCreateConstantValColumnAggInfo(SInputColumnInfoData* pInput, SF if (type == TSDB_DATA_TYPE_BIGINT) { int64_t v = pFuncParam->param.i; - *da = (SColumnDataAgg) {.numOfNull = 0, .min = v, .max = v, .maxIndex = 0, .minIndex = 0, .sum = v * numOfRows}; + *da = (SColumnDataAgg){.numOfNull = 0, .min = v, .max = v, .maxIndex = 0, .minIndex = 0, .sum = v * numOfRows}; } else if (type == TSDB_DATA_TYPE_DOUBLE) { double v = pFuncParam->param.d; - *da = (SColumnDataAgg) {.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; + *da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; - *(double*) &da->min = v; - *(double*) &da->max = v; - *(double*) &da->sum = v * numOfRows; + *(double*)&da->min = v; + *(double*)&da->max = v; + *(double*)&da->sum = v * numOfRows; } else if (type == TSDB_DATA_TYPE_BOOL) { // todo validate this data type bool v = pFuncParam->param.i; - *da = (SColumnDataAgg) {.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; - *(bool*) &da->min = 0; - *(bool*) &da->max = v; - *(bool*) &da->sum = v * numOfRows; + *da = (SColumnDataAgg){.numOfNull = 0, .maxIndex = 0, .minIndex = 0}; + *(bool*)&da->min = 0; + *(bool*)&da->max = v; + *(bool*)&da->sum = v * numOfRows; } else if (type == TSDB_DATA_TYPE_TIMESTAMP) { // do nothing } else { @@ -1806,10 +1813,6 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* // set the output buffer for the selectivity + tag query static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { - return TSDB_CODE_SUCCESS; - } - int32_t num = 0; int16_t tagLen = 0; @@ -1836,9 +1839,8 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } if (p != NULL) { - p->subsidiaryRes.pCtx = pTagCtx; - p->subsidiaryRes.numOfCols = num; - p->subsidiaryRes.bufLen = tagLen; + p->subsidiaries.pCtx = pTagCtx; + p->subsidiaries.num = num; } else { taosMemoryFreeClear(pTagCtx); } @@ -1865,6 +1867,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SqlFunctionCtx* pCtx = &pFuncCtx[i]; pCtx->functionId = -1; + pCtx->curBufPage = -1; + pCtx->pExpr = pExpr; + if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) { SFuncExecEnv env = {0}; pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; @@ -1892,45 +1897,46 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->pTsOutput = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; pCtx->resDataInfo.type = pFunct->resSchema.type; - pCtx->order = TSDB_ORDER_ASC; + pCtx->order = TSDB_ORDER_ASC; pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; + pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; pCtx->param = pFunct->pParam; -// for (int32_t j = 0; j < pCtx->numOfParams; ++j) { -// // set the order information for top/bottom query -// int32_t functionId = pCtx->functionId; -// if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { -// int32_t f = getExprFunctionId(&pExpr[0]); -// assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); -// -// // pCtx->param[2].i = pQueryAttr->order.order; -// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; -// // pCtx->param[3].i = functionId; -// // pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; -// -// // pCtx->param[1].i = pQueryAttr->order.col.info.colId; -// } else if (functionId == FUNCTION_INTERP) { -// // pCtx->param[2].i = (int8_t)pQueryAttr->fillType; -// // if (pQueryAttr->fillVal != NULL) { -// // if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { -// // pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; -// // } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value -// // if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { -// // taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); -// // } -// // } -// // } -// } else if (functionId == FUNCTION_TWA) { -// // pCtx->param[1].i = pQueryAttr->window.skey; -// // pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; -// // pCtx->param[2].i = pQueryAttr->window.ekey; -// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; -// } else if (functionId == FUNCTION_ARITHM) { -// // pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); -// } -// } + // for (int32_t j = 0; j < pCtx->numOfParams; ++j) { + // // set the order information for top/bottom query + // int32_t functionId = pCtx->functionId; + // if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { + // int32_t f = getExprFunctionId(&pExpr[0]); + // assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); + // + // // pCtx->param[2].i = pQueryAttr->order.order; + // // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + // // pCtx->param[3].i = functionId; + // // pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; + // + // // pCtx->param[1].i = pQueryAttr->order.col.info.colId; + // } else if (functionId == FUNCTION_INTERP) { + // // pCtx->param[2].i = (int8_t)pQueryAttr->fillType; + // // if (pQueryAttr->fillVal != NULL) { + // // if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { + // // pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; + // // } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value + // // if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { + // // taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], + // pCtx->inputBytes, pCtx->inputType); + // // } + // // } + // // } + // } else if (functionId == FUNCTION_TWA) { + // // pCtx->param[1].i = pQueryAttr->window.skey; + // // pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; + // // pCtx->param[2].i = pQueryAttr->window.ekey; + // // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; + // } else if (functionId == FUNCTION_ARITHM) { + // // pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); + // } + // } } for (int32_t i = 1; i < numOfOutput; ++i) { @@ -1953,7 +1959,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx); + taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx); } taosMemoryFreeClear(pCtx); @@ -2056,25 +2062,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { return status; } -static void doUpdateLastKey(STaskAttr* pQueryAttr) { - STimeWindow* win = &pQueryAttr->window; - - size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); - for (int32_t i = 0; i < num; ++i) { - SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t len = taosArrayGetSize(p1); - for (int32_t j = 0; j < len; ++j) { - // STableKeyInfo* pInfo = taosArrayGet(p1, j); - // - // // update the new lastkey if it is equalled to the value of the old skey - // if (pInfo->lastKey == win->ekey) { - // pInfo->lastKey = win->skey; - // } - } - } -} - // static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) { // STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // @@ -2627,7 +2614,6 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void* pTable, SqlFunctionCtx* pCt setCtxTagForJoin(pRuntimeEnv, &pCtx[0], pExprInfo, pTable); } #endif - } void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) { @@ -2672,12 +2658,12 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) // pTableQueryInfo->cur.vgroupIndex = -1; // set the index to be the end slot of result rows array -// SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; -// if (pResultRowInfo->size > 0) { -// pResultRowInfo->curPos = pResultRowInfo->size - 1; -// } else { -// pResultRowInfo->curPos = -1; -// } + // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; + // if (pResultRowInfo->size > 0) { + // pResultRowInfo->curPos = pResultRowInfo->size - 1; + // } else { + // pResultRowInfo->curPos = -1; + // } } void initResultRow(SResultRow* pResultRow) { @@ -2703,7 +2689,7 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t int64_t tid = 0; int64_t groupId = 0; - SResultRow* pRow = doSetResultOutBufByKey_rv(pSup->pResultBuf, pResultRowInfo, tid, (char*)&tid, sizeof(tid), true, + SResultRow* pRow = doSetResultOutBufByKey(pSup->pResultBuf, pResultRowInfo, tid, (char*)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { @@ -2753,7 +2739,7 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOf if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { -// if (i > 0) pBInfo->pCtx[i].pTsOutput = pBInfo->pCtx[i - 1].pOutput; + // if (i > 0) pBInfo->pCtx[i].pTsOutput = pBInfo->pCtx[i - 1].pOutput; } } } @@ -2791,7 +2777,8 @@ void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || fmIsScalarFunc(pCtx[j].functionId)) { + if (isRowEntryInitialized(pResInfo) || fmIsPseudoColumnFunc(pCtx[j].functionId) || pCtx[j].functionId == -1 || + fmIsScalarFunc(pCtx[j].functionId)) { continue; } @@ -2809,6 +2796,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } } +// todo merged with the build group result. void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < pResultRowInfo->size; ++i) { @@ -2818,9 +2806,9 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); // TODO ignore the close status anyway. -// if (!isResultRowClosed(pRow)) { -// continue; -// } + // if (!isResultRowClosed(pRow)) { + // continue; + // } for (int32_t j = 0; j < numOfOutput; ++j) { pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); @@ -2831,7 +2819,7 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } if (pCtx[j].fpSet.process) { // TODO set the dummy function, to avoid the check for null ptr. -// pCtx[j].fpSet.finalize(&pCtx[j]); + // pCtx[j].fpSet.finalize(&pCtx[j]); } if (pRow->numOfRows < pResInfo->numOfRes) { @@ -2843,40 +2831,36 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } } +// todo merged with the build group result. void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, int32_t* rowCellInfoOffset) { size_t num = taosArrayGetSize(pUpdateList); for (int32_t i = 0; i < num; ++i) { - SResultRowPosition* pPos = taosArrayGet(pUpdateList, i); - - SFilePage* bufPage = getBufPage(pBuf, pPos->pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); + SResKeyPos * pPos = taosArrayGetP(pUpdateList, i); + SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); +// for (int32_t j = 0; j < numOfOutput; ++j) { pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); - +// struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; - if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - continue; - } - - if (pCtx[j].fpSet.process) { // TODO set the dummy function. -// pCtx[j].fpSet.finalize(&pCtx[j]); - pResInfo->initialized = true; - } - +// if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { +// continue; +// } +// +// if (pCtx[j].fpSet.process) { // TODO set the dummy function. +//// pCtx[j].fpSet.finalize(&pCtx[j]); +// pResInfo->initialized = true; +// } +// if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; } } releaseBufPage(pBuf, bufPage); - /* - * set the number of output results for group by normal columns, the number of output rows usually is 1 except - * the top and bottom query - */ - // buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); } } @@ -2887,10 +2871,10 @@ STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow // set more initial size of interval/groupby query // if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) { int32_t initialSize = 128; -// int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); -// if (code != TSDB_CODE_SUCCESS) { -// return NULL; -// } + // int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize); + // if (code != TSDB_CODE_SUCCESS) { + // return NULL; + // } // } else { // in other aggregate query, do not initialize the windowResInfo // } @@ -2903,10 +2887,11 @@ void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { } // taosVariantDestroy(&pTableQueryInfo->tag); -// cleanupResultRowInfo(&pTableQueryInfo->resInfo); + // cleanupResultRowInfo(&pTableQueryInfo->resInfo); } -void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { +void setResultRowOutputBufInitCtx_rv(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, + int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); @@ -2982,7 +2967,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { blockDataUpdateTsWindow(pBlock); } -void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo) { +void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, + SExecTaskInfo* pTaskInfo) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; @@ -2991,7 +2977,7 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; SResultRow* pResultRow = - doSetResultOutBufByKey_rv(pAggInfo->aggSup.pResultBuf, pResultRowInfo, uid, (char*)&groupId, sizeof(groupId), + doSetResultOutBufByKey(pAggInfo->aggSup.pResultBuf, pResultRowInfo, uid, (char*)&groupId, sizeof(groupId), true, groupId, pTaskInfo, false, &pAggInfo->aggSup); assert(pResultRow != NULL); @@ -3000,7 +2986,8 @@ void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, u * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, groupId, pAggInfo->binfo.pRes->info.rowSize); + int32_t ret = + addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, groupId, pAggInfo->binfo.pRes->info.rowSize); if (ret != TSDB_CODE_SUCCESS) { return; } @@ -3020,33 +3007,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } -void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - SExprBasicInfo* pExpr = &pExprInfo->base; - // if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && - // (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) && - // (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) { - // assert(pExpr->numOfParams == 1); - // - // int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; - // SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - // - // doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); - // - // int16_t tagType = pCtx[0].tag.nType; - // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { - // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", - // GET_TASKID(pRuntimeEnv), - //// pExprInfo->base.param[0].i, pCtx[0].tag.pz); - // } else { - // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, - // GET_TASKID(pRuntimeEnv), - //// pExprInfo->base.param[0].i, pCtx[0].tag.i); - // } - // } -} - /* * There are two cases to handle: * @@ -3057,13 +3017,13 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprI * is a previous result generated or not. */ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { -// SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; -// if (pResultRowInfo->curPos != -1) { -// return; -// } + // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; + // if (pResultRowInfo->curPos != -1) { + // return; + // } -// pTableQueryInfo->win.skey = key; -// STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; + // pTableQueryInfo->win.skey = key; + // STimeWindow win = {.skey = key, .ekey = pQRange->ekey}; /** * In handling the both ascending and descending order super table query, we need to find the first qualified @@ -3071,10 +3031,10 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional * operations involve. */ -// STimeWindow w = TSWINDOW_INITIALIZER; -// -// TSKEY sk = TMIN(win.skey, win.ekey); -// TSKEY ek = TMAX(win.skey, win.ekey); + // STimeWindow w = TSWINDOW_INITIALIZER; + // + // TSKEY sk = TMIN(win.skey, win.ekey); + // TSKEY ek = TMAX(win.skey, win.ekey); // getAlignQueryTimeWindow(pQueryAttr, win.skey, sk, ek, &w); // if (pResultRowInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -3089,7 +3049,6 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin } /** - * copyToOutputBuf support copy data in ascending/descending order * For interval query of both super table and table, copy the data in ascending order, since the output results are * ordered in SWindowResutl already. While handling the group by query for both table and super table, * all group result are completed already. @@ -3098,7 +3057,7 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin * @param result */ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, - int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { + int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -3117,10 +3076,10 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased } for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i); - SFilePage* page = getBufPage(pBuf, pPos->pageId); + SResKeyPos *pPos = taosArrayGetP(pGroupResInfo->pRows, i); + SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)page + pPos->offset); + SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; continue; @@ -3139,7 +3098,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); if (pCtx[j].fpSet.process) { - pCtx[j].fpSet.finalize(&pCtx[j], pBlock, slotId); + pCtx[j].fpSet.finalize(&pCtx[j], pBlock); } else { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); @@ -3161,8 +3120,8 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased return 0; } -void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, - int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { +void doBuildResultDatablock(SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, + SDiskbasedBuf* pBuf, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); blockDataCleanup(pBlock); @@ -3198,7 +3157,6 @@ static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutpu } } #endif - } static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { @@ -3225,9 +3183,9 @@ void publishOperatorProfEvent(SOperatorInfo* pOperator, EQueryProfEventType even event.eventType = eventType; event.eventTime = taosGetTimestampUs(); event.operatorType = pOperator->operatorType; -// if (pQInfo->summary.queryProfEvents) { -// taosArrayPush(pQInfo->summary.queryProfEvents, &event); -// } + // if (pQInfo->summary.queryProfEvents) { + // taosArrayPush(pQInfo->summary.queryProfEvents, &event); + // } } void publishQueryAbortEvent(SExecTaskInfo* pTaskInfo, int32_t code) { @@ -3274,15 +3232,15 @@ static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* ev } void calculateOperatorProfResults(void) { -// if (pQInfo->summary.queryProfEvents == NULL) { -// // qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId); -// return; -// } -// -// if (pQInfo->summary.operatorProfResults == NULL) { -// // qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId); -// return; -// } + // if (pQInfo->summary.queryProfEvents == NULL) { + // // qDebug("QInfo:0x%"PRIx64" query prof events array is null", pQInfo->qId); + // return; + // } + // + // if (pQInfo->summary.operatorProfResults == NULL) { + // // qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId); + // return; + // } SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem)); if (opStack == NULL) { @@ -3680,8 +3638,8 @@ int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->compLen = htonl(pRsp->compLen); - pRsp->useconds = htobe64(pRsp->useconds); + pRsp->compLen = htonl(pRsp->compLen); + pRsp->useconds = htobe64(pRsp->useconds); } else { pSourceDataInfo->code = code; } @@ -3760,13 +3718,20 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf // TODO if only one or two columns required, how to extract data? int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { + int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, + SArray* pColList) { blockDataEnsureCapacity(pRes, numOfRows); if (pColList == NULL) { // data from other sources - int32_t* colLen = (int32_t*)pData; - char* pStart = pData + sizeof(int32_t) * numOfOutput; + int32_t dataLen = *(int32_t*) pData; + pData += sizeof(int32_t); + pRes->info.groupId = *(uint64_t*) pData; + pData += sizeof(uint64_t); + + int32_t* colLen = (int32_t*)pData; + + char* pStart = pData + sizeof(int32_t) * numOfOutput; for (int32_t i = 0; i < numOfOutput; ++i) { colLen[i] = htonl(colLen[i]); ASSERT(colLen[i] > 0); @@ -3786,6 +3751,9 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } memcpy(pColInfoData->pData, pStart, colLen[i]); + //TODO setting this flag to true temporarily so aggregate function on stable will + //examine NULL value for non-primary key column + pColInfoData->hasNull = true; pStart += colLen[i]; } } else { // extract data according to pColList @@ -3796,7 +3764,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI pStart += sizeof(int32_t); SSysTableSchema* pSchema = (SSysTableSchema*)pStart; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SSysTableSchema* p = (SSysTableSchema*)pStart; p->colId = htons(p->colId); @@ -3805,7 +3773,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } SSDataBlock block = {.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)), .info.numOfCols = numOfCols}; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {0}; idata.info.type = pSchema[i].type; idata.info.bytes = pSchema[i].bytes; @@ -3819,7 +3787,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI blockDataEnsureCapacity(&block, numOfRows); - int32_t* colLen = (int32_t*) pStart; + int32_t dataLen = *(int32_t*) pStart; + uint64_t groupId = *(uint64_t*) (pStart + sizeof(int32_t)); + pStart += sizeof(int32_t) + sizeof(uint64_t); + + int32_t* colLen = (int32_t*) (pStart); pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { @@ -3861,6 +3833,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } pRes->info.rows = numOfRows; + blockDataUpdateTsWindow(pRes); int64_t el = taosGetTimestampUs() - startTs; @@ -3942,7 +3915,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx pLoadInfo->totalRows, pLoadInfo->totalSize, i + 1, totalSources); pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED; } else { - qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 + ", totalBytes:%" PRIu64, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pLoadInfo->totalRows, pLoadInfo->totalSize); } @@ -4037,12 +4011,12 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); tsem_wait(&pExchangeInfo->ready); - SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); if (pDataInfo->code != TSDB_CODE_SUCCESS) { - qError("%s vgId:%d, taskID:0x%" PRIx64 " error happens, code:%s", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, tstrerror(pDataInfo->code)); + qError("%s vgId:%d, taskID:0x%" PRIx64 " error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId, + pSource->taskId, tstrerror(pDataInfo->code)); pOperator->pTaskInfo->code = pDataInfo->code; return NULL; } @@ -4050,7 +4024,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRsp = pDataInfo->pRsp; SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo; if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%" PRIx64 " %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next", + qDebug("%s vgId:%d, taskID:0x%" PRIx64 " %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 + " try next", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, pDataInfo->totalRows, pLoadInfo->totalRows); @@ -4190,21 +4165,21 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock goto _error; } - pInfo->pResult = pBlock; + pInfo->pResult = pBlock; pInfo->seqLoadData = true; tsem_init(&pInfo->ready, 0, 0); - pOperator->name = "ExchangeOperator"; + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = pBlock->info.numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = prepareLoadRemoteData; // assign a dummy function. - pOperator->getNextFn = doLoadRemoteData; - pOperator->closeFn = destroyExchangeOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = pBlock->info.numOfCols; + pOperator->pTaskInfo = pTaskInfo; + pOperator->_openFn = prepareLoadRemoteData; // assign a dummy function. + pOperator->getNextFn = doLoadRemoteData; + pOperator->closeFn = destroyExchangeOperatorInfo; #if 1 { // todo refactor @@ -4242,7 +4217,8 @@ _error: return NULL; } -static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); +static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, + const char* pKey); static void cleanupAggSup(SAggSupporter* pAggSup); static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { @@ -4273,18 +4249,6 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { // } } -static SExprInfo* exprArrayDup(SArray* pExprList) { - size_t numOfOutput = taosArrayGetSize(pExprList); - - SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo)); - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprList, i); - assignExprInfo(&p[i], pExpr); - } - - return p; -} - // TODO merge aggregate super table static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { @@ -4395,7 +4359,7 @@ static void doFinalizeResultImpl(SqlFunctionCtx* pCtx, int32_t numOfExpr) { // SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); // doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); // } else { -// pCtx[j].fpSet.finalize(&pCtx[j]); + // pCtx[j].fpSet.finalize(&pCtx[j]); } } @@ -4509,8 +4473,8 @@ static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator, bool* newgroup) { } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, NULL, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->binfo.pRes, "GET_TASKID(pTaskInfo)"); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, NULL, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, + numOfBufPage, pInfo->binfo.pRes, "GET_TASKID(pTaskInfo)"); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); @@ -4594,7 +4558,7 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t goto _error; } - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; int32_t code = doInitAggInfoSup(&pInfo->aggSup, pInfo->binfo.pCtx, num, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4657,8 +4621,8 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) { } int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT, pInfo->bufPageSize, numOfBufPage, - pInfo->pDataBlock, pTaskInfo->id.str); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, pInfo->inputSlotMap, SORT_SINGLESOURCE_SORT, + pInfo->bufPageSize, numOfBufPage, pInfo->pDataBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock); @@ -4676,10 +4640,11 @@ static SSDataBlock* doSort(SOperatorInfo* pOperator, bool* newgroup) { return getSortedBlockData(pInfo->pSortHandle, pInfo->pDataBlock, pInfo->numOfRowsInRes); } -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, SArray* pIndexMap, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pResBlock, SArray* pSortInfo, + SArray* pIndexMap, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - int32_t rowSize = pResBlock->info.rowSize; + int32_t rowSize = pResBlock->info.rowSize; if (pInfo == NULL || pOperator == NULL || rowSize > 100 * 1024 * 1024) { taosMemoryFreeClear(pInfo); @@ -4688,7 +4653,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR return NULL; } - pInfo->bufPageSize = rowSize < 1024 ? 1024*2 : rowSize*2; // there are headers, so pageSize = rowSize + header + pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; // there are headers, so pageSize = rowSize + header pInfo->sortBufSize = pInfo->bufPageSize * 16; // TODO dynamic set the available sort buffer pInfo->numOfRowsInRes = 1024; @@ -4696,20 +4661,20 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pInfo->pSortInfo = pSortInfo; pInfo->inputSlotMap = pIndexMap; - pOperator->name = "SortOperator"; + pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doSort; - pOperator->closeFn = destroyOrderOperatorInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->getNextFn = doSort; + pOperator->closeFn = destroyOrderOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pInfo); taosMemoryFree(pOperator); @@ -4727,7 +4692,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SAggOperatorInfo* pAggInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pAggInfo->binfo; + SOptrBasicInfo* pInfo = &pAggInfo->binfo; int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -4747,7 +4712,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated before apply the group aggregation. if (pAggInfo->pScalarExprInfo != NULL) { - projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, pAggInfo->numOfScalarExpr, NULL); + projectApplyFunctions(pAggInfo->pScalarExprInfo, pBlock, pBlock, pAggInfo->pScalarCtx, pAggInfo->numOfScalarExpr, + NULL); } // the pDataBlock are always the same one, no need to call this again @@ -4755,7 +4721,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order, true); doAggregateImpl(pOperator, 0, pInfo->pCtx); -#if 0 // test for encode/decode result info +#if 0 // test for encode/decode result info if(pOperator->encodeResultRow){ char *result = NULL; int32_t length = 0; @@ -4775,7 +4741,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfOutput, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pAggInfo->groupResInfo, &pAggInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -4795,7 +4761,8 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->pRes, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, pInfo->rowCellInfoOffset, pInfo->pCtx); + doBuildResultDatablock(pInfo->pRes, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf, + pInfo->rowCellInfoOffset, pInfo->pCtx); if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -4804,7 +4771,8 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) return (blockDataGetNumOfRows(pInfo->pRes) != 0) ? pInfo->pRes : NULL; } -void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasicInfo *pInfo, char **result, int32_t *length) { +void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char** result, + int32_t* length) { int32_t size = taosHashGetSize(pSup->pResultRowHashTable); size_t keyLen = sizeof(uint64_t) * 2; // estimate the key length int32_t totalSize = sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); @@ -4817,17 +4785,17 @@ void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasi // prepare memory SResultRowPosition* pos = &pInfo->resultRowInfo.cur; - void* pPage = getBufPage(pSup->pResultBuf, pos->pageId); - SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset); + void* pPage = getBufPage(pSup->pResultBuf, pos->pageId); + SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset); setBufPageDirty(pPage, true); releaseBufPage(pSup->pResultBuf, pPage); - void* pIter = taosHashIterate(pSup->pResultRowHashTable, NULL); + void* pIter = taosHashIterate(pSup->pResultRowHashTable, NULL); while (pIter) { - void* key = taosHashGetKey(pIter, &keyLen); + void* key = taosHashGetKey(pIter, &keyLen); SResultRowPosition* p1 = (SResultRowPosition*)pIter; - pPage = (SFilePage*) getBufPage(pSup->pResultBuf, p1->pageId); + pPage = (SFilePage*)getBufPage(pSup->pResultBuf, p1->pageId); pRow = (SResultRow*)((char*)pPage + p1->offset); setBufPageDirty(pPage, true); releaseBufPage(pSup->pResultBuf, pPage); @@ -4866,7 +4834,8 @@ void aggEncodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasi return; } -bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasicInfo *pInfo, char* result, int32_t length) { +bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasicInfo* pInfo, char* result, + int32_t length) { if (!result || length <= 0) { return false; } @@ -4904,9 +4873,10 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter *pSup, SOptrBasi initResultRow(resultRow); prepareResultListBuffer(&pInfo->resultRowInfo, pOperator->pTaskInfo->env); -// pInfo->resultRowInfo.cur = pInfo->resultRowInfo.size; - pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = (SResultRowPosition) {.pageId = resultRow->pageId, .offset = resultRow->offset}; - pInfo->resultRowInfo.cur = (SResultRowPosition) {.pageId = resultRow->pageId, .offset = resultRow->offset}; + // pInfo->resultRowInfo.cur = pInfo->resultRowInfo.size; +// pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = +// (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; + pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; } if (offset != length) { @@ -4925,7 +4895,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) if (pOperator->status == OP_EXEC_DONE) { return NULL; } - + #if 0 if (pProjectInfo->existDataBlock) { // TODO refactor SSDataBlock* pBlock = pProjectInfo->existDataBlock; @@ -4986,7 +4956,8 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) setInputDataBlock(pOperator, pInfo->pCtx, pBlock, TSDB_ORDER_ASC, false); blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows); - projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput, pProjectInfo->pPseudoColInfo); + projectApplyFunctions(pOperator->pExpr, pInfo->pRes, pBlock, pInfo->pCtx, pOperator->numOfOutput, + pProjectInfo->pPseudoColInfo); if (pProjectInfo->curSOffset > 0) { if (pProjectInfo->groupId == 0) { // it is the first group @@ -5034,7 +5005,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) break; } } - + if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pInfo->pRes->info.rows >= pProjectInfo->limit.limit) { pInfo->pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); } @@ -5050,12 +5021,12 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableIntervalOperatorInfo* pInfo = pOperator->info; int32_t order = TSDB_ORDER_ASC; // STimeWindow win = {0}; - bool newgroup = false; + bool newgroup = false; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -5073,9 +5044,9 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); -#if 0 // test for encode/decode result info +#if 0 // test for encode/decode result info if(pOperator->encodeResultRow){ char *result = NULL; int32_t length = 0; @@ -5095,7 +5066,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -5119,7 +5090,8 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro } blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBlock, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -5129,16 +5101,17 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro } } -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { STableIntervalOperatorInfo* pInfo = pOperator->info; - int32_t order = TSDB_ORDER_ASC; + int32_t order = TSDB_ORDER_ASC; if (pOperator->status == OP_EXEC_DONE) { return NULL; } if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -5160,8 +5133,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup break; } - // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the caller. - // Note that all the time window are not close till now. + // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // caller. Note that all the time window are not close till now. // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again @@ -5169,11 +5142,13 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); } - finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); + finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, + pInfo->binfo.rowCellInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); ASSERT(pInfo->binfo.pRes->info.rows > 0); pOperator->status = OP_RES_TO_RETURN; @@ -5181,7 +5156,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; } -static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { +static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { if (pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -5196,8 +5171,8 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { return pSliceInfo->binfo.pRes; } - int32_t order = TSDB_ORDER_ASC; -// STimeWindow win = pQueryAttr->window; + int32_t order = TSDB_ORDER_ASC; + // STimeWindow win = pQueryAttr->window; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -5211,16 +5186,16 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { // setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSliceInfo->binfo.pCtx, pBlock, order, true); -// hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); + // hashAllIntervalAgg(pOperator, &pSliceInfo->binfo.resultRowInfo, pBlock, 0); } // restore the value pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pSliceInfo->binfo.resultRowInfo); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); -// finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); + // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); - initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); +// initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { @@ -5264,19 +5239,19 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); -// hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pBlock->info.groupId); + // hashIntervalAgg(pOperator, &pTableQueryInfo->resInfo, pBlock, pBlock->info.groupId); } closeAllResultRows(&pInfo->binfo.resultRowInfo); finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); +// initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); OPTR_SET_OPENED(pOperator); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, - pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pInfo->binfo.pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -5286,11 +5261,11 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup } static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SOptrBasicInfo* pBInfo = &pInfo->binfo; SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); - int64_t gid = pBlock->info.groupId; + int64_t gid = pBlock->info.groupId; bool masterScan = true; int32_t numOfOutput = pOperator->numOfOutput; @@ -5303,7 +5278,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SWindowRowsSup* pRowSup = &pInfo->winSup; pRowSup->numOfRows = 0; - + for (int32_t j = 0; j < pBlock->info.rows; ++j) { if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pBlock->pBlockAgg)) { continue; @@ -5330,13 +5305,15 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI pRowSup->win.ekey = pRowSup->win.skey; int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &window, masterScan, - &pResult, gid, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + &pResult, gid, pInfo->binfo.pCtx, numOfOutput, + pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false); - doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); // here we start a new session window doKeepNewWindowStartInfo(pRowSup, tsList, j); @@ -5346,14 +5323,16 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SResultRow* pResult = NULL; pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, &pResult, - gid, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setResultOutputBufByKey_rv(&pInfo->binfo.resultRowInfo, pBlock->info.uid, &pRowSup->win, masterScan, + &pResult, gid, pInfo->binfo.pCtx, numOfOutput, + pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false); - doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, + pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); } static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { @@ -5362,11 +5341,12 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { } SStateWindowOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5375,8 +5355,8 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { return pBInfo->pRes; } - int32_t order = TSDB_ORDER_ASC; - STimeWindow win = pTaskInfo->window; + int32_t order = TSDB_ORDER_ASC; + STimeWindow win = pTaskInfo->window; SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -5394,11 +5374,13 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5415,7 +5397,8 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); return NULL; @@ -5443,11 +5426,13 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) // restore the value pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); + finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, + pBInfo->rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); - doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); + doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, + pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5596,16 +5581,17 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { taosMemoryFreeClear(pOperator); } -int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey) { +int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, + const char* pKey) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); - pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); + pAggSup->resultRowSize = getResultRowSize(pCtx, numOfOutput); + pAggSup->keyBuf = taosMemoryCalloc(1, keyBufSize + POINTER_BYTES + sizeof(int64_t)); pAggSup->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); - pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); - pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); +// pAggSup->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); +// pAggSup->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); - if (pAggSup->keyBuf == NULL || pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL || + if (pAggSup->keyBuf == NULL /*|| pAggSup->pResultRowArrayList == NULL || pAggSup->pResultRowListSet == NULL*/ || pAggSup->pResultRowHashTable == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -5621,8 +5607,8 @@ int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t n static void cleanupAggSup(SAggSupporter* pAggSup) { taosMemoryFreeClear(pAggSup->keyBuf); taosHashCleanup(pAggSup->pResultRowHashTable); - taosHashCleanup(pAggSup->pResultRowListSet); - taosArrayDestroy(pAggSup->pResultRowArrayList); +// taosHashCleanup(pAggSup->pResultRowListSet); +// taosArrayDestroy(pAggSup->pResultRowArrayList); destroyDiskbasedBuf(pAggSup->pResultBuf); } @@ -5632,6 +5618,11 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf pBasicInfo->pRes = pResultBlock; doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey); + + for(int32_t i = 0; i < numOfCols; ++i) { + pBasicInfo->pCtx[i].pBuf = pAggSup->pResultBuf; + } + return TSDB_CODE_SUCCESS; } @@ -5657,9 +5648,9 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf STableKeyInfo* pk = taosArrayGet(pa, j); STableQueryInfo* pTQueryInfo = &pTableQueryInfo[index++]; -// pTQueryInfo->uid = pk->uid; + // pTQueryInfo->uid = pk->uid; pTQueryInfo->lastKey = pk->lastKey; -// pTQueryInfo->groupIndex = i; + // pTQueryInfo->groupIndex = i; } } @@ -5670,7 +5661,8 @@ static STableQueryInfo* initTableQueryInfo(const STableGroupInfo* pTableGroupInf SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, - int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { + int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo, + const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5678,10 +5670,11 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* } int32_t numOfRows = 1; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, numOfRows); - int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResultBlock, keyBufSize, pTaskInfo->id.str); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResultBlock, keyBufSize, pTaskInfo->id.str); pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); if (code != TSDB_CODE_SUCCESS || pInfo->pTableQueryInfo == NULL) { goto _error; @@ -5700,17 +5693,17 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->pScalarCtx = createSqlFunctionCtx(pScalarExprInfo, numOfCols, &pInfo->rowCellInfoOffset); } - pOperator->name = "TableAggregate"; + pOperator->name = "TableAggregate"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_AGG; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = doOpenAggregateOptr; - pOperator->getNextFn = getAggregateResult; - pOperator->closeFn = destroyAggOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->pTaskInfo = pTaskInfo; + pOperator->_openFn = doOpenAggregateOptr; + pOperator->getNextFn = getAggregateResult; + pOperator->closeFn = destroyAggOperatorInfo; pOperator->encodeResultRow = aggEncodeResultRow; pOperator->decodeResultRow = aggDecodeResultRow; @@ -5778,11 +5771,6 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { - STagScanInfo* pInfo = (STagScanInfo*)param; - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); @@ -5804,7 +5792,7 @@ void destroyExchangeOperatorInfo(void* param, int32_t numOfOutput) { static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) { SArray* pList = taosArrayInit(4, sizeof(int32_t)); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { if (fmIsPseudoColumnFunc(pCtx[i].functionId)) { taosArrayPush(pList, &i); } @@ -5814,39 +5802,40 @@ static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) } SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t num, - SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SLimit* pLimit, SLimit* pSlimit, + SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->limit = *pLimit; - pInfo->slimit = *pSlimit; - pInfo->curOffset = pLimit->offset; + pInfo->limit = *pLimit; + pInfo->slimit = *pSlimit; + pInfo->curOffset = pLimit->offset; pInfo->curSOffset = pSlimit->offset; pInfo->binfo.pRes = pResBlock; int32_t numOfCols = num; int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, numOfRows); initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); setFunctionResultOutput(&pInfo->binfo, &pInfo->aggSup, MAIN_SCAN, pTaskInfo); pInfo->pPseudoColInfo = setRowTsColumnOutputInfo(pInfo->binfo.pCtx, numOfCols); - pOperator->name = "ProjectOperator"; + pOperator->name = "ProjectOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PROJECT; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = num; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doProjectOperation; - pOperator->closeFn = destroyProjectOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = num; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doProjectOperation; + pOperator->closeFn = destroyProjectOperatorInfo; pOperator->pTaskInfo = pTaskInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -5863,7 +5852,8 @@ _error: SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { + STimeWindowAggSupp* pTwAggSupp, const STableGroupInfo* pTableGroupInfo, + SExecTaskInfo* pTaskInfo) { STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -5872,16 +5862,18 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->order = TSDB_ORDER_ASC; pInfo->interval = *pInterval; +// pInfo->execModel = OPTR_EXEC_MODEL_STREAM; pInfo->execModel = pTaskInfo->execModel; pInfo->win = pTaskInfo->window; pInfo->twAggSup = *pTwAggSupp; pInfo->primaryTsIndex = primaryTsSlotId; int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, numOfRows); - int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); @@ -5891,18 +5883,18 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - pOperator->name = "TimeIntervalAggOperator"; + pOperator->name = "TimeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->_openFn = doOpenIntervalAgg; - pOperator->getNextFn = doBuildIntervalResult; - pOperator->getStreamResFn= doStreamIntervalAgg; - pOperator->closeFn = destroyIntervalOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->_openFn = doOpenIntervalAgg; + pOperator->getNextFn = doBuildIntervalResult; + pOperator->getStreamResFn = doStreamIntervalAgg; + pOperator->closeFn = destroyIntervalOperatorInfo; pOperator->encodeResultRow = aggEncodeResultRow; pOperator->decodeResultRow = aggDecodeResultRow; @@ -5921,40 +5913,42 @@ _error: return NULL; } -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pOperator == NULL || pInfo == NULL) { goto _error; } initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - pOperator->name = "TimeSliceOperator"; + pOperator->name = "TimeSliceOperator"; // pOperator->operatorType = OP_AllTimeWindow; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doAllIntervalAgg; - pOperator->closeFn = destroyBasicOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->getNextFn = doAllIntervalAgg; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; } -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSup, - SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, + SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSup, + SExecTaskInfo* pTaskInfo) { SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -5969,30 +5963,31 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pInfo->twAggSup = *pTwAggSup; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - pOperator->name = "StateWindowOperator"; + pOperator->name = "StateWindowOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pInfo; - pOperator->getNextFn = doStateWindowAgg; - pOperator->closeFn = destroyStateWindowOperatorInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->getNextFn = doStateWindowAgg; + pOperator->closeFn = destroyStateWindowOperatorInfo; pOperator->encodeResultRow = aggEncodeResultRow; pOperator->decodeResultRow = aggDecodeResultRow; int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_SUCCESS; return NULL; } SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp* pTwAggSupp, + SExecTaskInfo* pTaskInfo) { SSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSessionAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -6000,10 +5995,11 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo } int32_t numOfRows = 4096; - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, numOfRows); - int32_t code = initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -6012,22 +6008,22 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); - pInfo->gap = gap; - pInfo->binfo.pRes = pResBlock; - pInfo->winSup.prevTs = INT64_MIN; - pInfo->reptScan = false; - pOperator->name = "SessionWindowAggOperator"; + pInfo->gap = gap; + pInfo->binfo.pRes = pResBlock; + pInfo->winSup.prevTs = INT64_MIN; + pInfo->reptScan = false; + pOperator->name = "SessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW; pOperator->blockingOptr = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->getNextFn = doSessionWindowAgg; - pOperator->closeFn = destroySWindowOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->getNextFn = doSessionWindowAgg; + pOperator->closeFn = destroySWindowOperatorInfo; pOperator->encodeResultRow = aggEncodeResultRow; pOperator->decodeResultRow = aggDecodeResultRow; - pOperator->pTaskInfo = pTaskInfo; + pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6075,12 +6071,24 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp int32_t type = TSDB_FILL_NONE; switch (fillType) { - case FILL_MODE_PREV: type = TSDB_FILL_PREV;break; - case FILL_MODE_NONE: type = TSDB_FILL_NONE;break; - case FILL_MODE_NULL: type = TSDB_FILL_NULL;break; - case FILL_MODE_NEXT: type = TSDB_FILL_NEXT;break; - case FILL_MODE_VALUE: type = TSDB_FILL_SET_VALUE;break; - case FILL_MODE_LINEAR: type = TSDB_FILL_LINEAR;break; + case FILL_MODE_PREV: + type = TSDB_FILL_PREV; + break; + case FILL_MODE_NONE: + type = TSDB_FILL_NONE; + break; + case FILL_MODE_NULL: + type = TSDB_FILL_NULL; + break; + case FILL_MODE_NEXT: + type = TSDB_FILL_NEXT; + break; + case FILL_MODE_VALUE: + type = TSDB_FILL_SET_VALUE; + break; + case FILL_MODE_LINEAR: + type = TSDB_FILL_LINEAR; + break; default: type = TSDB_FILL_NONE; } @@ -6094,16 +6102,16 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp goto _error; } - pOperator->name = "FillOperator"; + pOperator->name = "FillOperator"; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; + pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Fill; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doFill; - pOperator->pTaskInfo = pTaskInfo; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->_openFn = operatorDummyOpenFn; + pOperator->getNextFn = doFill; + pOperator->pTaskInfo = pTaskInfo; pOperator->closeFn = destroySFillOperatorInfo; @@ -6116,158 +6124,6 @@ _error: return NULL; } -static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { -#if 0 - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); - if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->pExpr[0]; - int32_t rsize = pExprInfo->base.resSchema.bytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resSchema.bytes; - int16_t type = pExprInfo->base.resSchema.type; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo - - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - - STableQueryInfo* item = taosArrayGetP(pa, i); - - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { - continue; - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resSchema.type; - bytes = pExprInfo[j].base.resSchema.bytes; - - if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); - } - - dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; - doSetTagValueToResultBuf(dst, data, type, bytes); - } - - count += 1; - } - - if (pInfo->curPos >= pInfo->totalTables) { - pOperator->status = OP_EXEC_DONE; - } - - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } - - if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); - } - - pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; - -#endif - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - assert(numOfGroup == 0 || numOfGroup == 1); - - pInfo->curPos = 0; - - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "SeqTableTagScan"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->getNextFn = doTagScan; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->closeFn = destroyTagScanOperatorInfo; - - return pOperator; -} - static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { int32_t j = 0; @@ -6308,10 +6164,10 @@ bool validateExprColumnInfo(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, int32_t scale, int32_t precision, const char* name) { SResSchema s = {0}; - s.scale = scale; - s.type = type; - s.bytes = bytes; - s.slotId = slotId; + s.scale = scale; + s.type = type; + s.bytes = bytes; + s.slotId = slotId; s.precision = precision; strncpy(s.name, name, tListLen(s.name)); @@ -6325,11 +6181,11 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, SDataType* pType) return NULL; } - pCol->slotId = slotId; - pCol->bytes = pType->bytes; - pCol->type = pType->type; - pCol->scale = pType->scale; - pCol->precision = pType->precision; + pCol->slotId = slotId; + pCol->bytes = pType->bytes; + pCol->type = pType->type; + pCol->scale = pType->scale; + pCol->precision = pType->precision; pCol->dataBlockId = blockId; return pCol; @@ -6369,7 +6225,8 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->base.numOfParams = 1; SDataType* pType = &pColNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pColNode->colName); + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pColNode->colName); pExp->base.pParam[0].pCol = createColumn(pColNode->dataBlockId, pColNode->slotId, pType); pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; } else if (type == QUERY_NODE_VALUE) { @@ -6380,7 +6237,8 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->base.numOfParams = 1; SDataType* pType = &pValNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pValNode->node.aliasName); + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pValNode->node.aliasName); pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; valueNodeToVariant(pValNode, &pExp->base.pParam[0].param); } else if (type == QUERY_NODE_FUNCTION) { @@ -6388,11 +6246,13 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; SDataType* pType = &pFuncNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pFuncNode->node.aliasName); pExp->pExpr->_function.functionId = pFuncNode->funcId; pExp->pExpr->_function.pFunctNode = pFuncNode; - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, tListLen(pExp->pExpr->_function.functionName)); + strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, + tListLen(pExp->pExpr->_function.functionName)); int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); @@ -6402,7 +6262,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* for (int32_t j = 0; j < numOfParam; ++j) { SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); if (p1->type == QUERY_NODE_COLUMN) { - SColumnNode* pcn = (SColumnNode*) p1; + SColumnNode* pcn = (SColumnNode*)p1; pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; pExp->base.pParam[j].pCol = createColumn(pcn->dataBlockId, pcn->slotId, &pcn->node.resType); @@ -6420,7 +6280,8 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* pExp->base.numOfParams = 1; SDataType* pType = &pNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pNode->node.aliasName); + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pNode->node.aliasName); pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; } else { ASSERT(0); @@ -6436,7 +6297,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; - pTaskInfo->execModel = model; + pTaskInfo->execModel = model; char* p = taosMemoryCalloc(1, 128); snprintf(p, 128, "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, queryId); @@ -6465,25 +6326,30 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { - SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; - STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode* ) pPhyNode; + SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); + if (pDataReader == NULL && terrno != 0) { + return NULL; + } - SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); + SArray* pColList = + extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); SInterval interval = { - .interval = pTableScanNode->interval, - .sliding = pTableScanNode->sliding, + .interval = pTableScanNode->interval, + .sliding = pTableScanNode->sliding, .intervalUnit = pTableScanNode->intervalUnit, - .slidingUnit = pTableScanNode->slidingUnit, - .offset = pTableScanNode->offset, + .slidingUnit = pTableScanNode->slidingUnit, + .offset = pTableScanNode->offset, }; - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pTableScanNode->dataRequired, - pScanPhyNode->count, pScanPhyNode->reverse, pColList, pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); + return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC, + numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList, + pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); @@ -6497,8 +6363,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); int32_t numOfCols = 0; - SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pColList, tableIdList, pTaskInfo); + SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -6521,26 +6387,29 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo size_t size = LIST_LENGTH(pPhyNode->pChildren); SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); - for(int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + if (ops[i] == NULL) { + return NULL; + } } SOperatorInfo* pOptr = NULL; if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { - SProjectPhysiNode* pProjPhyNode = (SProjectPhysiNode*) pPhyNode; - SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &num); + SProjectPhysiNode* pProjPhyNode = (SProjectPhysiNode*)pPhyNode; + SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SLimit limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset}; - SLimit slimit = {.limit = pProjPhyNode->slimit, .offset = pProjPhyNode->soffset}; + SLimit limit = {.limit = pProjPhyNode->limit, .offset = pProjPhyNode->offset}; + SLimit slimit = {.limit = pProjPhyNode->slimit, .offset = pProjPhyNode->soffset}; pOptr = createProjectOperatorInfo(ops[0], pExprInfo, num, pResBlock, &limit, &slimit, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_AGG == type) { SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - int32_t numOfScalarExpr = 0; + int32_t numOfScalarExpr = 0; SExprInfo* pScalarExprInfo = NULL; if (pAggNode->pExprs != NULL) { pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); @@ -6548,9 +6417,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pAggNode->pGroupKeys != NULL) { SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); - pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pTaskInfo, NULL); + pOptr = createGroupOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pAggNode->node.pConditions, + pScalarExprInfo, numOfScalarExpr, pTaskInfo, NULL); } else { - pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr, pTaskInfo, pTableGroupInfo); + pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pScalarExprInfo, numOfScalarExpr, + pTaskInfo, pTableGroupInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -6558,22 +6429,23 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - SInterval interval = { - .interval = pIntervalPhyNode->interval, - .sliding = pIntervalPhyNode->sliding, - .intervalUnit = pIntervalPhyNode->intervalUnit, - .slidingUnit = pIntervalPhyNode->slidingUnit, - .offset = pIntervalPhyNode->offset, - .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision - }; + SInterval interval = {.interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; - STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType}; + STimeWindowAggSupp as = {.waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType}; - int32_t primaryTsSlotId = ((SColumnNode*) pIntervalPhyNode->window.pTspk)->slotId; - pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, pTableGroupInfo, pTaskInfo); + int32_t primaryTsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, primaryTsSlotId, &as, + pTableGroupInfo, pTaskInfo); if (pIntervalPhyNode->pFill != NULL) { - pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, NULL, false, pTaskInfo); + pOptr = createFillOperatorInfo(pOptr, pExprInfo, num, &interval, pResBlock, pIntervalPhyNode->pFill->mode, NULL, + false, pTaskInfo); } } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { @@ -6586,29 +6458,30 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW == type) { SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType}; + STimeWindowAggSupp as = {.waterMark = pSessionNode->window.watermark, + .calTrigger = pSessionNode->window.triggerType}; SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); pOptr = createSessionAggOperatorInfo(ops[0], pExprInfo, num, pResBlock, pSessionNode->gap, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { - SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*) pPhyNode; - SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SPartitionPhysiNode* pPartNode = (SPartitionPhysiNode*)pPhyNode; + SArray* pColList = extractPartitionColInfo(pPartNode->pPartitionKeys); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &num); pOptr = createPartitionOperatorInfo(ops[0], pExprInfo, num, pResBlock, pColList, pTaskInfo, NULL); } else if (QUERY_NODE_PHYSICAL_PLAN_STATE_WINDOW == type) { - SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*) pPhyNode; + SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; STimeWindowAggSupp as = {.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType}; - SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); + SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); pOptr = createStatewindowOperatorInfo(ops[0], pExprInfo, num, pResBlock, &as, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_JOIN == type) { - SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*) pPhyNode; - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SJoinPhysiNode* pJoinNode = (SJoinPhysiNode*)pPhyNode; + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &num); pOptr = createJoinOperatorInfo(ops, size, pExprInfo, num, pResBlock, pJoinNode->pOnConditions, pTaskInfo); @@ -6624,7 +6497,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa void* readHandle, uint64_t queryId, uint64_t taskId) { STsdbQueryCond cond = {.loadExternalRows = false}; - cond.order = pTableScanNode->scan.order; + cond.order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); if (cond.colList == NULL) { @@ -6644,7 +6517,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa continue; } - cond.colList[j].type = pColNode->node.resType.type; + cond.colList[j].type = pColNode->node.resType.type; cond.colList[j].bytes = pColNode->node.resType.bytes; cond.colList[j].colId = pColNode->colId; j += 1; @@ -6691,9 +6564,9 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { // todo extract method SColumn c = {0}; c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; c.precision = pColNode->node.resType.precision; c.scale = pColNode->node.resType.scale; @@ -6717,9 +6590,9 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { // todo extract method SColumn c = {0}; c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; c.precision = pColNode->node.resType.precision; c.scale = pColNode->node.resType.scale; @@ -6738,7 +6611,6 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { } for (int32_t i = 0; i < numOfCols; ++i) { - SOrderByExprNode* pSortKey = (SOrderByExprNode*)nodesListGetNode(pNodeList, i); SBlockOrderInfo bi = {0}; bi.order = (pSortKey->order == ORDER_ASC) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; @@ -6751,14 +6623,14 @@ SArray* createSortInfo(SNodeList* pNodeList, SNodeList* pNodeListTarget) { STargetNode* pTarget = (STargetNode*)nodesListGetNode(pNodeListTarget, j); SColumnNode* pColNodeT = (SColumnNode*)pTarget->pExpr; - if(pColNode->slotId == pColNodeT->slotId){ // to find slotId in PhysiSort OutputDataBlockDesc + if (pColNode->slotId == pColNodeT->slotId) { // to find slotId in PhysiSort OutputDataBlockDesc bi.slotId = pTarget->slotId; found = true; break; } } - if(!found){ + if (!found) { qError("sort slot id does not found"); } taosArrayPush(pList, &bi); @@ -6809,7 +6681,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod for (int32_t i = 0; i < num; ++i) { SSlotDescNode* pNode = (SSlotDescNode*)nodesListGetNode(pOutputNodeList->pSlots, i); // todo: add reserve flag check - if (pNode->slotId >= numOfCols) { // it is a column reserved for the arithmetic expression calculation + if (pNode->slotId >= numOfCols) { // it is a column reserved for the arithmetic expression calculation (*numOfOutputCols) += 1; continue; } @@ -6825,7 +6697,8 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod return pList; } -int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { +int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, + uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { code = tsdbQuerySTableByTagCond(metaHandle, tableUid, 0, NULL, 0, 0, NULL, pGroupInfo, NULL, 0, queryId, taskId); @@ -6876,7 +6749,8 @@ _error: return NULL; } -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, EOPTR_EXEC_MODEL model) { +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, + EOPTR_EXEC_MODEL model) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; @@ -6886,8 +6760,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } - STableGroupInfo group = {0}; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo); if (NULL == (*pTaskInfo)->pRoot) { code = terrno; goto _complete; @@ -6938,16 +6811,16 @@ void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo) { const float THRESHOLD_RATIO = 0.85f; -// if (isProjQuery(pQueryAttr)) { -// int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; -// if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { -// numOfRes = MIN_ROWS_FOR_PRJ_QUERY; -// } -// -// pResultInfo->capacity = numOfRes; -// } else { // in case of non-prj query, a smaller output buffer will be used. -// pResultInfo->capacity = DEFAULT_MIN_ROWS; -// } + // if (isProjQuery(pQueryAttr)) { + // int32_t numOfRes = DEFAULT_RESULT_MSG_SIZE / pQueryAttr->resultRowSize; + // if (numOfRes < MIN_ROWS_FOR_PRJ_QUERY) { + // numOfRes = MIN_ROWS_FOR_PRJ_QUERY; + // } + // + // pResultInfo->capacity = numOfRes; + // } else { // in case of non-prj query, a smaller output buffer will be used. + // pResultInfo->capacity = DEFAULT_MIN_ROWS; + // } pResultInfo->threshold = (int32_t)(pResultInfo->capacity * THRESHOLD_RATIO); pResultInfo->totalRows = 0; @@ -7064,10 +6937,11 @@ void releaseQueryBuf(size_t numOfTables) { atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); } -int32_t getOperatorExplainExecInfo(SOperatorInfo *operatorInfo, SExplainExecInfo **pRes, int32_t *capacity, int32_t *resNum) { +int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo** pRes, int32_t* capacity, + int32_t* resNum) { if (*resNum >= *capacity) { *capacity += 10; - + *pRes = taosMemoryRealloc(*pRes, (*capacity) * sizeof(SExplainExecInfo)); if (NULL == *pRes) { qError("malloc %d failed", (*capacity) * (int32_t)sizeof(SExplainExecInfo)); @@ -7086,9 +6960,9 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo *operatorInfo, SExplainExecInfo return code; } } - + ++(*resNum); - + int32_t code = 0; for (int32_t i = 0; i < operatorInfo->numOfDownstream; ++i) { code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pRes, capacity, resNum); @@ -7103,7 +6977,7 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo *operatorInfo, SExplainExecInfo static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) { SJoinOperatorInfo* pJoinInfo = pOperator->info; -// SOptrBasicInfo* pInfo = &pJoinInfo->binfo; + // SOptrBasicInfo* pInfo = &pJoinInfo->binfo; SSDataBlock* pRes = pJoinInfo->pRes; blockDataCleanup(pRes); @@ -7141,54 +7015,54 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) } SColumnInfoData* pLeftCol = taosArrayGet(pJoinInfo->pLeft->pDataBlock, pJoinInfo->leftCol.slotId); - char* pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos); + char* pLeftVal = colDataGetData(pLeftCol, pJoinInfo->leftPos); SColumnInfoData* pRightCol = taosArrayGet(pJoinInfo->pRight->pDataBlock, pJoinInfo->rightCol.slotId); - char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); + char* pRightVal = colDataGetData(pRightCol, pJoinInfo->rightPos); // only the timestamp match support for ordinary table - ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - if (*(int64_t*) pLeftVal == *(int64_t*) pRightVal) { - for(int32_t i = 0; i < pOperator->numOfOutput; ++i) { - SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i); + ASSERT(pLeftCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + if (*(int64_t*)pLeftVal == *(int64_t*)pRightVal) { + for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, i); - SExprInfo* pExprInfo = &pOperator->pExpr[i]; + SExprInfo* pExprInfo = &pOperator->pExpr[i]; - int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; - int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; + int32_t blockId = pExprInfo->base.pParam[0].pCol->dataBlockId; + int32_t slotId = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = NULL; - if (pJoinInfo->pLeft->info.blockId == blockId) { - pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); - } else { - pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); - } - - if (colDataIsNull_s(pSrc, pJoinInfo->leftPos)) { - colDataAppendNULL(pDst, nrows); - } else { - char* p = colDataGetData(pSrc, pJoinInfo->leftPos); - colDataAppend(pDst, nrows, p, false); - } + SColumnInfoData* pSrc = NULL; + if (pJoinInfo->pLeft->info.blockId == blockId) { + pSrc = taosArrayGet(pJoinInfo->pLeft->pDataBlock, slotId); + } else { + pSrc = taosArrayGet(pJoinInfo->pRight->pDataBlock, slotId); } - pJoinInfo->leftPos += 1; - pJoinInfo->rightPos += 1; - - nrows += 1; - } else if (*(int64_t*) pLeftVal < *(int64_t*) pRightVal) { - pJoinInfo->leftPos += 1; - - if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { - continue; - } - } else if (*(int64_t*) pLeftVal > *(int64_t*) pRightVal) { - pJoinInfo->rightPos += 1; - if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { - continue; + if (colDataIsNull_s(pSrc, pJoinInfo->leftPos)) { + colDataAppendNULL(pDst, nrows); + } else { + char* p = colDataGetData(pSrc, pJoinInfo->leftPos); + colDataAppend(pDst, nrows, p, false); } } + pJoinInfo->leftPos += 1; + pJoinInfo->rightPos += 1; + + nrows += 1; + } else if (*(int64_t*)pLeftVal < *(int64_t*)pRightVal) { + pJoinInfo->leftPos += 1; + + if (pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { + continue; + } + } else if (*(int64_t*)pLeftVal > *(int64_t*)pRightVal) { + pJoinInfo->rightPos += 1; + if (pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { + continue; + } + } + // the pDataBlock are always the same one, no need to call this again pRes->info.rows = nrows; if (pRes->info.rows >= pOperator->resultInfo.threshold) { @@ -7199,9 +7073,11 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) return (pRes->info.rows > 0) ? pRes : NULL; } -SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, + int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, + SExecTaskInfo* pTaskInfo) { SJoinOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SJoinOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pOperator == NULL || pInfo == NULL) { goto _error; } @@ -7209,27 +7085,27 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf pOperator->resultInfo.capacity = 4096; pOperator->resultInfo.threshold = 4096 * 0.75; -// initResultRowInf -// o(&pInfo->binfo.resultRowInfo, 8); - pInfo->pRes = pResBlock; + // initResultRowInf + // o(&pInfo->binfo.resultRowInfo, 8); + pInfo->pRes = pResBlock; - pOperator->name = "JoinOperator"; + pOperator->name = "JoinOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_JOIN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->numOfOutput = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doMergeJoin; - pOperator->closeFn = destroyBasicOperatorInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->getNextFn = doMergeJoin; + pOperator->closeFn = destroyBasicOperatorInfo; int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); return pOperator; - _error: +_error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; return NULL; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 4793600710..52bdbea8a5 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -308,7 +308,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou // } blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3a9742d48a..a70e402871 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -271,9 +271,7 @@ static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCt static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { STableScanInfo* pTableScanInfo = pOperator->info; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; - + SSDataBlock* pBlock = pTableScanInfo->pResBlock; *newgroup = false; while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { @@ -284,18 +282,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { pTableScanInfo->numOfBlocks += 1; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); - // todo opt - // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - // STableQueryInfo** pTableQueryInfo = - // (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); - // if (pTableQueryInfo == NULL) { - // break; - // } - // - // doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); - // } - - // this function never returns error? uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); @@ -308,6 +294,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { continue; } + // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator. + pBlock->info.blockId = 0; return pBlock; } @@ -405,11 +393,11 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; pOperator->cost.openCost = ++cost; @@ -824,12 +812,12 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { int32_t tableNameSlotId = 1; SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); - char* name = NULL; + char* tb = NULL; int32_t numOfRows = 0; char n[TSDB_TABLE_NAME_LEN] = {0}; - while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { - STR_TO_VARSTR(n, name); + while ((tb = metaTbCursorNext(pInfo->pCur)) != NULL) { + STR_TO_VARSTR(n, tb); colDataAppend(pTableNameCol, numOfRows, n, false); numOfRows += 1; if (numOfRows >= pInfo->capacity) { @@ -992,3 +980,167 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB return pOperator; } + +static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { +#if 0 + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; + + STagScanInfo *pInfo = pOperator->info; + SSDataBlock *pRes = pInfo->pRes; + *newgroup = false; + + int32_t count = 0; + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + + int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); + if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id + assert(pQueryAttr->numOfOutput == 1); + + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + int32_t rsize = pExprInfo->base.resSchema.bytes; + + count = 0; + + int16_t bytes = pExprInfo->base.resSchema.bytes; + int16_t type = pExprInfo->base.resSchema.type; + + for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { + if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { + bytes = pQueryAttr->tagColList[i].bytes; + type = pQueryAttr->tagColList[i].type; + break; + } + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + STableQueryInfo *item = taosArrayGetP(pa, i); + + char *output = pColInfo->pData + count * rsize; + varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); + + output = varDataVal(output); + STableId* id = TSDB_TABLEID(item->pTable); + + *(int16_t *)output = 0; + output += sizeof(int16_t); + + *(int64_t *)output = id->uid; // memory align problem, todo serialize + output += sizeof(id->uid); + + *(int32_t *)output = id->tid; + output += sizeof(id->tid); + + *(int32_t *)output = pQueryAttr->vgId; + output += sizeof(pQueryAttr->vgId); + + char* data = NULL; + if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); + } + + doSetTagValueToResultBuf(output, data, type, bytes); + count += 1; + } + + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); + } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + *(int64_t*)pColInfo->pData = pInfo->totalTables; + count = 1; + + pOperator->status = OP_EXEC_DONE; + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); + } else { // return only the tags|table name etc. + SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo + + count = 0; + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + + STableQueryInfo* item = taosArrayGetP(pa, i); + + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; + for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { + continue; + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); + type = pExprInfo[j].base.resSchema.type; + bytes = pExprInfo[j].base.resSchema.bytes; + + if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); + } + + dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + doSetTagValueToResultBuf(dst, data, type, bytes); + } + + count += 1; + } + + if (pInfo->curPos >= pInfo->totalTables) { + pOperator->status = OP_EXEC_DONE; + } + + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); + } + + if (pOperator->status == OP_EXEC_DONE) { + setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); + } + + pRes->info.rows = count; + return (pRes->info.rows == 0)? NULL:pInfo->pRes; + +#endif + return TSDB_CODE_SUCCESS; +} + +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { + STagScanInfo* pInfo = (STagScanInfo*)param; + pInfo->pRes = blockDataDestroy(pInfo->pRes); +} + +SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { + STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->pReader = pReaderHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->getNextFn = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + pOperator->closeFn = destroyTagScanOperatorInfo; + + return pOperator; + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index db92740fff..0ad3730b5a 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -24,7 +24,7 @@ extern "C" { #include "functionMgt.h" bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); -int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); @@ -40,15 +40,20 @@ bool getMinmaxFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t minFunction(SqlFunctionCtx* pCtx); int32_t maxFunction(SqlFunctionCtx *pCtx); +bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t avgFunction(SqlFunctionCtx* pCtx); +int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunction(SqlFunctionCtx* pCtx); -int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx *pCtx); -int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); @@ -60,7 +65,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); -int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); #ifdef __cplusplus } diff --git a/source/libs/function/inc/functionMgtInt.h b/source/libs/function/inc/functionMgtInt.h index 39287f08ee..af4c8a7bdb 100644 --- a/source/libs/function/inc/functionMgtInt.h +++ b/source/libs/function/inc/functionMgtInt.h @@ -41,7 +41,9 @@ extern "C" { #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0) -#define FUNC_UDF_ID_START_OFFSET_VAL 5000 +#define FUNC_UDF_ID_START 5000 +#define FUNC_AGGREGATE_UDF_ID 5001 +#define FUNC_SCALAR_UDF_ID 5002 extern const int funcMgtUdfNum; diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 5f4f96c4cc..b72905b872 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -29,29 +29,20 @@ extern "C" { #define UDF_LISTEN_PIPE_NAME_LEN 32 #define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." +#define UDF_DNODE_ID_ENV_NAME "DNODE_ID" //====================================================================================== //begin API to taosd and qworker enum { UDFC_CODE_STOPPING = -1, - UDFC_CODE_PIPE_READ_ERR = -3, + UDFC_CODE_PIPE_READ_ERR = -2, + UDFC_CODE_CONNECT_PIPE_ERR = -3, + UDFC_CODE_LOAD_UDF_FAILURE = -4, + UDFC_CODE_INVALID_STATE = -5 }; -typedef void *UdfcHandle; -typedef void *UdfcFuncHandle; -/** - * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf - * @return error code - */ -int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle); - -/** - * destroy udfd proxy - * @return error code - */ -int32_t udfcClose(UdfcHandle proxyhandle); /** @@ -60,7 +51,7 @@ int32_t udfcClose(UdfcHandle proxyhandle); * @param handle, out * @return error code */ -int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle); +int32_t setupUdf(char udfName[], UdfcFuncHandle *handle); typedef struct SUdfColumnMeta { int16_t type; diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 2c16afbd0d..4e7178f7fd 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -39,7 +39,6 @@ enum { typedef struct SUdfSetupRequest { char udfName[TSDB_FUNC_NAME_LEN]; - SEpSet epSet; } SUdfSetupRequest; typedef struct SUdfSetupResponse { @@ -112,6 +111,7 @@ void freeUdfDataDataBlock(SUdfDataBlock *block); int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock); int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block); +int32_t getUdfdPipeName(char* pipeName, int32_t size); #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 3da5dc3289..70087ee46b 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -104,7 +104,7 @@ static int32_t translateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -479,6 +479,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = stddevFunction, .finalizeFunc = stddevFinalize }, + { + .name = "avg", + .type = FUNCTION_TYPE_AVG, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getAvgFuncEnv, + .initFunc = avgFunctionSetup, + .processFunc = avgFunction, + .finalizeFunc = avgFinalize + }, { .name = "percentile", .type = FUNCTION_TYPE_PERCENTILE, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3dc2e62e92..0b9765ef15 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,12 +14,63 @@ */ #include "builtinsimpl.h" -#include +#include "function.h" #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" #include "tpercentile.h" +typedef struct SSumRes { + union { + int64_t isum; + uint64_t usum; + double dsum; + }; +} SSumRes; + +typedef struct SAvgRes { + double result; + SSumRes sum; + int64_t count; +} SAvgRes; + +typedef struct STopBotResItem { + SVariant v; + uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data + struct { + int32_t pageId; + int32_t offset; + } tuplePos; // tuple data of this chosen row +} STopBotResItem; + +typedef struct STopBotRes { + STopBotResItem *pItems; +} STopBotRes; + +typedef struct SStddevRes { + double result; + int64_t count; + union {double quadraticDSum; int64_t quadraticISum;}; + union {double dsum; int64_t isum;}; +} SStddevRes; + +typedef struct SPercentileInfo { + double result; + tMemBucket *pMemBucket; + int32_t stage; + double minval; + double maxval; + int64_t numOfElems; +} SPercentileInfo; + +typedef struct SDiffInfo { + bool hasPrev; + bool includeNull; + bool ignoreNegative; + bool firstOutput; + union { int64_t i64; double d64;} prev; +} SDiffInfo; + #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -28,13 +79,38 @@ (_info)->numOfRes = (res); \ } while (0) -typedef struct SSumRes { - union { - int64_t isum; - uint64_t usum; - double dsum; - }; -} SSumRes; +#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) + +#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0); + +#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_SUBSID_RES(ctx, _ts); \ + (num) += 1; \ + } \ + } while (0) + +#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ + do { \ + _t *d = (_t *)((_col)->pData); \ + for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + } \ + TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \ + UPDATE_DATA(ctx, val, d[i], num, sign, ts); \ + } \ + } while (0) + bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { @@ -49,7 +125,8 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { return true; } -int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -135,7 +212,7 @@ int32_t sumFunction(SqlFunctionCtx *pCtx) { int32_t type = pInput->pData[0]->info.type; SSumRes* pSumRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - + if (pInput->colDataAggIsSet) { numOfElem = pInput->numOfRows - pAgg->numOfNull; ASSERT(numOfElem >= 0); @@ -190,6 +267,145 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } +bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(double); + return true; +} + +bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo); + memset(pRes, 0, sizeof(SAvgRes)); + return true; +} + +int32_t avgFunction(SqlFunctionCtx* pCtx) { + int32_t numOfElem = 0; + + // Only the pre-computing information loaded and actual data does not loaded + SInputColumnInfoData* pInput = &pCtx->input; + int32_t type = pInput->pData[0]->info.type; + + SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + // computing based on the true data block + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + switch (type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* plist = (int8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; + } + + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* plist = (int16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_INT: { + int32_t* plist = (int32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; + } + + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + int64_t* plist = (int64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.isum += plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + float* plist = (float*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.dsum += plist[i]; + } + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double* plist = (double*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem += 1; + pAvgRes->count += 1; + pAvgRes->sum.dsum += plist[i]; + } + break; + } + + default: + break; + } + + // data in the check operation are all null, not output + SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1); + return TSDB_CODE_SUCCESS; +} + +int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SInputColumnInfoData* pInput = &pCtx->input; + int32_t type = pInput->pData[0]->info.type; + SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (IS_INTEGER_TYPE(type)) { + pAvgRes->result = pAvgRes->sum.isum / ((double) pAvgRes->count); + } else { + pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count); + } + + return functionFinalize(pCtx, pBlock); +} + EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){ return FUNC_DATA_REQUIRED_STATIS_LOAD; } @@ -303,16 +519,16 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { } \ } while (0); -#define DO_UPDATE_SUBSID_RES(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - __ctx->fpSet.process(__ctx); \ - } \ +#define DO_UPDATE_SUBSID_RES(ctx, ts) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \ + SqlFunctionCtx* __ctx = (ctx)->subsidiaries.pCtx[_i]; \ + if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ + __ctx->tag.i = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + __ctx->fpSet.process(__ctx); \ + } \ } while (0) #define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ @@ -378,8 +594,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { *(int64_t*) buf = val; - for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { - SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -395,8 +611,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { *(uint64_t*) buf = val; - for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { - SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -479,13 +695,6 @@ int32_t maxFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -typedef struct SStddevRes { - double result; - int64_t count; - union {double quadraticDSum; int64_t quadraticISum;}; - union {double dsum; int64_t isum;}; -} SStddevRes; - bool getStddevFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SStddevRes); return true; @@ -588,8 +797,8 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { numOfElem += 1; pStddevRes->count += 1; - pStddevRes->isum += plist[i]; - pStddevRes->quadraticISum += plist[i] * plist[i]; + pStddevRes->dsum += plist[i]; + pStddevRes->quadraticDSum += plist[i] * plist[i]; } break; } @@ -603,8 +812,8 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { numOfElem += 1; pStddevRes->count += 1; - pStddevRes->isum += plist[i]; - pStddevRes->quadraticISum += plist[i] * plist[i]; + pStddevRes->dsum += plist[i]; + pStddevRes->quadraticDSum += plist[i] * plist[i]; } break; } @@ -618,21 +827,21 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + SInputColumnInfoData* pInput = &pCtx->input; + int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - double avg = pStddevRes->isum / ((double) pStddevRes->count); - pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg); - return functionFinalize(pCtx, pBlock, slotId); -} + double avg; + if (IS_INTEGER_TYPE(type)) { + avg = pStddevRes->isum / ((double) pStddevRes->count); + pStddevRes->result = sqrt(pStddevRes->quadraticISum/((double)pStddevRes->count) - avg*avg); + } else { + avg = pStddevRes->dsum / ((double) pStddevRes->count); + pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg); + } -typedef struct SPercentileInfo { - double result; - tMemBucket *pMemBucket; - int32_t stage; - double minval; - double maxval; - int64_t numOfElems; -} SPercentileInfo; + return functionFinalize(pCtx, pBlock); +} bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SPercentileInfo); @@ -744,7 +953,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; @@ -757,7 +966,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t sl } tMemBucketDestroy(pMemBucket); - return functionFinalize(pCtx, pBlock, slotId); + return functionFinalize(pCtx, pBlock); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -928,14 +1137,6 @@ int32_t lastFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -typedef struct SDiffInfo { - bool hasPrev; - bool includeNull; - bool ignoreNegative; - bool firstOutput; - union { int64_t i64; double d64;} prev; -} SDiffInfo; - bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDiffInfo); return true; @@ -1168,21 +1369,6 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) { } } -typedef struct STopBotResItem { - SVariant v; - uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data - struct { - int32_t pageId; - int32_t offset; - } tuplePos; // tuple data of this chosen row -} STopBotResItem; - -typedef struct STopBotRes { - int32_t pageId; -// int32_t num; - STopBotResItem *pItems; -} STopBotRes; - bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * sizeof(STopBotResItem); @@ -1197,15 +1383,16 @@ static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { return pRes; } -static void doAddIntoResult(STopBotRes* pRes, int32_t maxSize, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, +static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo); +static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); +static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); + int32_t topFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotRes *pRes = getTopBotOutputInfo(pCtx); - // if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { // buildTopBotStruct(pRes, pCtx); // } @@ -1225,7 +1412,7 @@ int32_t topFunction(SqlFunctionCtx *pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pRes, pCtx->param[1].param.i, data, i, NULL, type, pInput->uid, pResInfo); + doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, type, pInput->uid, pResInfo); } return TSDB_CODE_SUCCESS; @@ -1258,9 +1445,11 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par return (val1->v.d > val2->v.d) ? 1 : -1; } +void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, + uint64_t uid, SResultRowEntryInfo* pEntryInfo) { + STopBotRes *pRes = getTopBotOutputInfo(pCtx); + int32_t maxSize = pCtx->param[1].param.i; -void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, - uint64_t uid, SResultRowEntryInfo* pEntryInfo) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1272,22 +1461,9 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes]; pItem->v = val; pItem->uid = uid; - pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer - if (pRes->pageId == -1) { - SFilePage* pPage = getNewBufPage(NULL, 0, &pRes->pageId); - pPage->num = sizeof(SFilePage); - - // keep the current row data - for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); - bool isNull = colDataIsNull_s(pCol, rowIndex); - - - colDataGetData(pCol, rowIndex); - } - - } + // save the data of this tuple + saveTupleData(pCtx, rowIndex, pSrcBlock, pItem); // allocate the buffer and keep the data of this row into the new allocated buffer pEntryInfo->numOfRes++; @@ -1296,22 +1472,100 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { + // replace the old data and the coresponding tuple data STopBotResItem* pItem = &pItems[0]; pItem->v = val; pItem->uid = uid; - pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer + + // save the data of this tuple by over writing the old data + copyTupleData(pCtx, rowIndex, pSrcBlock, pItem); taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false); } } } -int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) { + SFilePage* pPage = NULL; + + int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); + + if (pCtx->curBufPage == -1) { + pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); + pPage->num = sizeof(SFilePage); + } else { + pPage = getBufPage(pCtx->pBuf, pCtx->curBufPage); + if (pPage->num + completeRowSize > getBufPageSize(pCtx->pBuf)) { + pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); + pPage->num = sizeof(SFilePage); + } + } + + pItem->tuplePos.pageId = pCtx->curBufPage; + + // keep the current row data, extract method + int32_t offset = 0; + bool* nullList = (bool*)((char*)pPage + pPage->num); + char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); + for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); + bool isNull = colDataIsNull_s(pCol, rowIndex); + if (isNull) { + nullList[i] = true; + continue; + } + + char* p = colDataGetData(pCol, rowIndex); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pStart + offset, p, varDataTLen(p)); + } else { + memcpy(pStart + offset, p, pCol->info.bytes); + } + + offset += pCol->info.bytes; + } + + pItem->tuplePos.offset = pPage->num; + pPage->num += completeRowSize; + + setBufPageDirty(pPage, true); + releaseBufPage(pCtx->pBuf, pPage); +} + +void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) { + SFilePage* pPage = getBufPage(pCtx->pBuf, pItem->tuplePos.pageId); + + bool* nullList = (bool*)((char*)pPage + pItem->tuplePos.offset); + char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); + + int32_t offset = 0; + for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); + if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { + continue; + } + + char* p = colDataGetData(pCol, rowIndex); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pStart + offset, p, varDataTLen(p)); + } else { + memcpy(pStart + offset, p, pCol->info.bytes); + } + + offset += pCol->info.bytes; + } + + setBufPageDirty(pPage, true); + releaseBufPage(pCtx->pBuf, pPage); +} + +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); pEntryInfo->complete = true; int32_t type = pCtx->input.pData[0]->info.type; + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); // todo assign the tag value and the corresponding row data @@ -1320,19 +1574,45 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; - colDataAppendInt32(pCol, currentRow++, (int32_t*)&pItem->v.i); + colDataAppendInt32(pCol, currentRow, (int32_t*)&pItem->v.i); int32_t pageId = pItem->tuplePos.pageId; int32_t offset = pItem->tuplePos.offset; - if (pageId != -1) { - // todo + if (pItem->tuplePos.pageId != -1) { + SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); + + bool* nullList = (bool*)((char*)pPage + offset); + char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); + + // todo set the offset value to optimize the performance. + for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { + SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; + + SFunctParam *pFuncParam = &pc->pExpr->base.pParam[0]; + int32_t srcSlotId = pFuncParam->pCol->slotId; + int32_t dstSlotId = pCtx->pExpr->base.resSchema.slotId; + + int32_t ps = 0; + for(int32_t k = 0; k < srcSlotId; ++k) { + SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, k); + ps += pSrcCol->info.bytes; + } + + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + if (nullList[srcSlotId]) { + colDataAppendNULL(pDstCol, currentRow); + } else { + colDataAppend(pDstCol, currentRow, (pStart + ps), false); + } + } } + + currentRow += 1; } + break; } } return pEntryInfo->numOfRes; - -// return functionFinalize(pCtx, pBlock, slotId); -} \ No newline at end of file +} diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index d44e3e251b..91ccdfb71e 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -24,7 +24,6 @@ typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; - SArray* pUdfTable; // SUdfInfo } SFuncMgtService; typedef struct SUdfInfo { @@ -49,18 +48,12 @@ static void doInitFunctionTable() { return; } } - - gFunMgtService.pUdfTable = NULL; -} - -static int8_t getUdfType(int32_t funcId) { - SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1); - return pUdf->funcType; } static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { if (fmIsUserDefinedFunc(funcId)) { - return getUdfType(funcId); + return FUNC_MGT_AGG_FUNC == classification ? FUNC_AGGREGATE_UDF_ID == funcId : + (FUNC_MGT_SCALAR_FUNC == classification ? FUNC_SCALAR_UDF_ID == funcId : false); } if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; @@ -68,33 +61,23 @@ static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification); } -static int32_t getUdfId(SFmGetFuncInfoParam* pParam, const char* pFuncName) { +static int32_t getUdfInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { SFuncInfo* pInfo = NULL; - int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFuncName, &pInfo); - if (TSDB_CODE_SUCCESS != code || NULL == pInfo) { - return -1; + int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFunc->functionName, &pInfo); + if (TSDB_CODE_SUCCESS != code) { + return code; } - if (NULL == gFunMgtService.pUdfTable) { - gFunMgtService.pUdfTable = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SUdfInfo)); + if (NULL == pInfo) { + snprintf(pParam->pErrBuf, pParam->errBufLen, "Invalid function name: %s", pFunc->functionName); + return TSDB_CODE_FUNC_INVALID_FUNTION; } - SUdfInfo info = { .outputDt.type = pInfo->outputType, .outputDt.bytes = pInfo->outputLen, .funcType = pInfo->funcType }; - taosArrayPush(gFunMgtService.pUdfTable, &info); + pFunc->funcType = FUNCTION_TYPE_UDF; + pFunc->funcId = TSDB_FUNC_TYPE_AGGREGATE == pInfo->funcType ? FUNC_AGGREGATE_UDF_ID : FUNC_SCALAR_UDF_ID; + pFunc->node.resType.type = pInfo->outputType; + pFunc->node.resType.bytes = pInfo->outputLen; + pFunc->udfBufSize = pInfo->bufSize; tFreeSFuncInfo(pInfo); taosMemoryFree(pInfo); - return taosArrayGetSize(gFunMgtService.pUdfTable) + FUNC_UDF_ID_START_OFFSET_VAL; -} - -static int32_t getFuncId(SFmGetFuncInfoParam* pParam, const char* pFuncName) { - void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName)); - if (NULL == pVal) { - return getUdfId(pParam, pFuncName); - } - return *(int32_t*)pVal; -} - -static int32_t getUdfResultType(SFunctionNode* pFunc) { - SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, pFunc->funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1); - pFunc->node.resType = pUdf->outputDt; return TSDB_CODE_SUCCESS; } @@ -103,28 +86,14 @@ int32_t fmFuncMgtInit() { return initFunctionCode; } -int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { - *pFuncId = getFuncId(pParam, pFuncName); - if (*pFuncId < 0) { - return TSDB_CODE_FAILED; +int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc) { + void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFunc->functionName, strlen(pFunc->functionName)); + if (NULL != pVal) { + pFunc->funcId = *(int32_t*)pVal; + pFunc->funcType = funcMgtBuiltins[pFunc->funcId].type; + return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pParam->pErrBuf, pParam->errBufLen); } - if (fmIsUserDefinedFunc(*pFuncId)) { - *pFuncType = FUNCTION_TYPE_UDF; - } else { - *pFuncType = funcMgtBuiltins[*pFuncId].type; - } - return TSDB_CODE_SUCCESS; -} - -int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (fmIsUserDefinedFunc(pFunc->funcId)) { - return getUdfResultType(pFunc); - } - - if (pFunc->funcId < 0 || pFunc->funcId >= funcMgtBuiltinsNum) { - return TSDB_CODE_FAILED; - } - return funcMgtBuiltins[pFunc->funcId].translateFunc(pFunc, pErrBuf, len); + return getUdfInfo(pParam, pFunc); } EFuncDataRequired fmFuncDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) { @@ -194,7 +163,7 @@ bool fmIsMultiResFunc(int32_t funcId) { } bool fmIsUserDefinedFunc(int32_t funcId) { - return funcId > FUNC_UDF_ID_START_OFFSET_VAL; + return funcId > FUNC_UDF_ID_START; } void fmFuncMgtDestroy() { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 317339af04..f8a7e77814 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -124,7 +124,7 @@ enum { int64_t gUdfTaskSeqNum = 0; typedef struct SUdfdProxy { - int32_t dnodeId; + char udfdPipeName[UDF_LISTEN_PIPE_NAME_LEN]; uv_barrier_t gUdfInitBarrier; uv_loop_t gUdfdLoop; @@ -137,11 +137,11 @@ typedef struct SUdfdProxy { int8_t gUdfcState; QUEUE gUdfTaskQueue; QUEUE gUvProcTaskQueue; - // int8_t gUdfcState = UDFC_STATE_INITAL; - // QUEUE gUdfTaskQueue = {0}; - // QUEUE gUvProcTaskQueue = {0}; + + int8_t initialized; } SUdfdProxy; +SUdfdProxy gUdfdProxy = {0}; typedef struct SUdfUvSession { SUdfdProxy *udfc; @@ -209,19 +209,27 @@ enum { UDFC_STATE_STARTNG, // starting after udfcOpen UDFC_STATE_READY, // started and begin to receive quests UDFC_STATE_STOPPING, // stopping after udfcClose - UDFC_STATUS_FINAL, // stopped }; +int32_t getUdfdPipeName(char* pipeName, int32_t size) { + char dnodeId[8] = {0}; + size_t dnodeIdSize; + int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize); + if (err != 0) { + dnodeId[0] = '1'; + } + snprintf(pipeName, size, "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); + return 0; +} + int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) { int32_t len = 0; len += taosEncodeBinary(buf, setup->udfName, TSDB_FUNC_NAME_LEN); - len += taosEncodeSEpSet(buf, &setup->epSet); return len; } void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) { buf = taosDecodeBinaryTo(buf, request->udfName, TSDB_FUNC_NAME_LEN); - buf = taosDecodeSEpSet((void*)buf, &request->epSet); return (void*)buf; } @@ -604,7 +612,7 @@ void onUdfcPipeClose(uv_handle_t *handle) { } int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { - debugPrint("%s", "get uv task result"); + fnDebug("udfc get uv task result. task: %p", task); if (uvTask->type == UV_TASK_REQ_RSP) { if (uvTask->rspBuf.base != NULL) { SUdfResponse rsp; @@ -647,7 +655,6 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT } void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint("%s", "client allocate buffer to receive from pipe"); SClientUvConn *conn = handle->data; SClientConnBuf *connBuf = &conn->readBuf; @@ -662,7 +669,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf buf->base = connBuf->buf; buf->len = connBuf->cap; } else { - //TODO: log error + fnError("udfc allocate buffer failure. size: %d", msgHeadSize); buf->base = NULL; buf->len = 0; } @@ -674,13 +681,13 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf buf->base = connBuf->buf + connBuf->len; buf->len = connBuf->cap - connBuf->len; } else { - //TODO: log error free connBuf->buf + fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap); buf->base = NULL; buf->len = 0; } } - debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); + fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); } @@ -689,6 +696,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { connBuf->total = *(int32_t *) (connBuf->buf); } if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + fnTrace("udfc complete message is received, now handle it"); return true; } return false; @@ -696,10 +704,10 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { void udfcUvHandleRsp(SClientUvConn *conn) { SClientConnBuf *connBuf = &conn->readBuf; - int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum if (QUEUE_EMPTY(&conn->taskQueue)) { - //LOG error + fnError("udfc no task waiting for response on connection"); return; } bool found = false; @@ -713,7 +721,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { found = true; taskFound = task; } else { - //LOG error; + fnError("udfc more than one task waiting for the same response"); continue; } } @@ -727,7 +735,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { uv_sem_post(&taskFound->taskSem); QUEUE_REMOVE(&taskFound->procTaskQueue); } else { - //TODO: LOG error + fnError("no task is waiting for the response."); } connBuf->buf = NULL; connBuf->total = -1; @@ -751,7 +759,7 @@ void udfcUvHandleError(SClientUvConn *conn) { } void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint("%s, nread: %zd", "client read from pipe", nread); + fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); if (nread == 0) return; SClientUvConn *conn = client->data; @@ -764,9 +772,9 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - debugPrint("\tclient read error: %s", uv_strerror(nread)); + fnError("udfc client pipe %p read error: %s", client, uv_strerror(nread)); if (nread == UV_EOF) { - //TODO: + fnError("udfc client pipe %p closed", client); } udfcUvHandleError(conn); } @@ -774,16 +782,15 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } void onUdfClientWrite(uv_write_t *write, int status) { - debugPrint("%s", "after writing to pipe"); SClientUvTaskNode *uvTask = write->data; + uv_pipe_t *pipe = uvTask->pipe; if (status == 0) { - uv_pipe_t *pipe = uvTask->pipe; SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); } else { - //TODO Log error; + fnError("udfc client %p write error.", pipe); } - debugPrint("\tlength:%zu", uvTask->reqBuf.len); + fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); taosMemoryFree(write); taosMemoryFree(uvTask->reqBuf.base); } @@ -841,7 +848,7 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN } int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint("%s, %d", "queue uv task", uvTask->type); + fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask); SUdfdProxy *udfc = uvTask->udfc; uv_mutex_lock(&udfc->gUdfTaskQueueMutex); QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); @@ -855,7 +862,7 @@ int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { } int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint("%s, type %d", "start uv task ", uvTask->type); + fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); @@ -874,8 +881,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; - - uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); + uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect); break; } case UV_TASK_REQ_RSP: { @@ -971,27 +977,37 @@ void constructUdfService(void *argsThread) { uv_loop_close(&udfc->gUdfdLoop); } -int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) { - SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy)); - proxy->dnodeId = dnodeId; +int32_t udfcOpen() { + int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 0, 1); + if (old == 1) { + return 0; + } + SUdfdProxy *proxy = &gUdfdProxy; + getUdfdPipeName(proxy->udfdPipeName, UDF_LISTEN_PIPE_NAME_LEN); proxy->gUdfcState = UDFC_STATE_STARTNG; uv_barrier_init(&proxy->gUdfInitBarrier, 2); uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); - uv_barrier_wait(&proxy->gUdfInitBarrier); + atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY); proxy->gUdfcState = UDFC_STATE_READY; - *udfc = proxy; + uv_barrier_wait(&proxy->gUdfInitBarrier); + fnInfo("udfc initialized") return 0; } -int32_t udfcClose(UdfcHandle udfcHandle) { - SUdfdProxy *udfc = udfcHandle; +int32_t udfcClose() { + int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 1, 0); + if (old == 0) { + return 0; + } + + SUdfdProxy *udfc = &gUdfdProxy; udfc->gUdfcState = UDFC_STATE_STOPPING; uv_async_send(&udfc->gUdfLoopStopAsync); uv_thread_join(&udfc->gUdfLoopThread); uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); uv_barrier_destroy(&udfc->gUdfInitBarrier); - udfc->gUdfcState = UDFC_STATUS_FINAL; - taosMemoryFree(udfc); + udfc->gUdfcState = UDFC_STATE_INITAL; + fnInfo("udfc cleaned up"); return 0; } @@ -1009,12 +1025,15 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { return task->errCode; } -int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) { - debugPrint("%s", "client setup udf"); +int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { + fnInfo("udfc setup udf. udfName: %s", udfName); + if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) { + return UDFC_CODE_INVALID_STATE; + } SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); - task->session->udfc = udfc; + task->session->udfc = &gUdfdProxy; task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; @@ -1022,15 +1041,20 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); if (errCode != 0) { - //TODO: log error - return -1; + fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName); + return UDFC_CODE_CONNECT_PIPE_ERR; } udfcRunUvTask(task, UV_TASK_REQ_RSP); SUdfSetupResponse *rsp = &task->_setup.rsp; task->session->severHandle = rsp->udfHandle; - *funcHandle = task->session; + if (task->errCode != 0) { + fnError("failed to setup udf. err: %d", task->errCode) + } else { + fnInfo("sucessfully setup udf func handle. handle: %p", task->session); + *funcHandle = task->session; + } int32_t err = task->errCode; taosMemoryFree(task); return err; @@ -1038,7 +1062,7 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { - debugPrint("%s", "client call udf"); + fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; @@ -1076,35 +1100,37 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf udfcRunUvTask(task, UV_TASK_REQ_RSP); - SUdfCallResponse *rsp = &task->_call.rsp; - switch (callType) { - case TSDB_UDF_CALL_AGG_INIT: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_MERGE: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_SCALA_PROC: { - *output = rsp->resultData; - break; + if (task->errCode != 0) { + fnError("call udf failure. err: %d", task->errCode); + } else { + SUdfCallResponse *rsp = &task->_call.rsp; + switch (callType) { + case TSDB_UDF_CALL_AGG_INIT: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_MERGE: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_SCALA_PROC: { + *output = rsp->resultData; + break; + } } } - taosMemoryFree(task); return task->errCode; } -//TODO: translate these calls to callUdf int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { int8_t callType = TSDB_UDF_CALL_AGG_INIT; @@ -1148,7 +1174,7 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu } int32_t teardownUdf(UdfcFuncHandle handle) { - debugPrint("%s", "client teardown udf"); + fnInfo("tear down udf. udf func handle: %p", handle); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; @@ -1160,7 +1186,6 @@ int32_t teardownUdf(UdfcFuncHandle handle) { udfcRunUvTask(task, UV_TASK_REQ_RSP); - SUdfTeardownResponse *rsp = &task->_teardown.rsp; int32_t err = task->errCode; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 6540851758..11bbd6442c 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -20,6 +20,7 @@ #include "tudf.h" #include "tudfInt.h" +#include "tdatablock.h" #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" @@ -31,8 +32,9 @@ typedef struct SUdfdContext { uv_signal_t intrSignal; char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; uv_pipe_t listeningPipe; - void *clientRpc; + void *clientRpc; + SCorEpSet mgmtEp; uv_mutex_t udfsMutex; SHashObj *udfsHash; @@ -63,8 +65,13 @@ typedef struct SUdf { uv_mutex_t lock; uv_cond_t condReady; - char name[16]; - int8_t type; + char name[TSDB_FUNC_NAME_LEN]; + int8_t funcType; + int8_t scriptType; + int8_t outputType; + int32_t outputLen; + int32_t bufSize; + char path[PATH_MAX]; uv_lib_t lib; @@ -78,17 +85,17 @@ typedef struct SUdfcFuncHandle { SUdf *udf; } SUdfcFuncHandle; -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf); +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); -int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) { +int32_t udfdLoadUdf(char *udfName, SUdf *udf) { strcpy(udf->name, udfName); - udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf); - + udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf); + //strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so"); int err = uv_dlopen(udf->path, &udf->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); - // TODO set error + return UDFC_CODE_LOAD_UDF_FAILURE; } // TODO: find all the functions char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; @@ -115,8 +122,8 @@ void udfdProcessRequest(uv_work_t *req) { SUdf *udf = NULL; uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN); - if (*udfInHash) { + SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName)); + if (udfInHash) { ++(*udfInHash)->refCount; udf = *udfInHash; uv_mutex_unlock(&global.udfsMutex); @@ -128,14 +135,14 @@ void udfdProcessRequest(uv_work_t *req) { uv_mutex_init(&udfNew->lock); uv_cond_init(&udfNew->condReady); udf = udfNew; - taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew)); + taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew)); uv_mutex_unlock(&global.udfsMutex); } uv_mutex_lock(&udf->lock); if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; - udfdLoadUdf(setup->udfName, &setup->epSet, udf); + udfdLoadUdf(setup->udfName, udf); udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -214,7 +221,7 @@ void udfdProcessRequest(uv_work_t *req) { udf->refCount--; if (udf->refCount == 0) { unloadUdf = true; - taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN); + taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); } uv_mutex_unlock(&global.udfsMutex); if (unloadUdf) { @@ -393,7 +400,48 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) { +int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { + pEpSet->version = 0; + + // init mnode ip set + SEpSet* mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return terrno; + } + + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); @@ -410,15 +458,21 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC; SRpcMsg rpcRsp = {0}; - rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp); + rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp); SRetrieveFuncRsp retrieveRsp = {0}; tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp); SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); + udf->funcType = pFuncInfo->funcType; + udf->scriptType = pFuncInfo->scriptType; + udf->outputType = pFuncInfo->funcType; + udf->outputLen = pFuncInfo->outputLen; + udf->bufSize = pFuncInfo->bufSize; + char path[PATH_MAX] = {0}; - taosGetTmpfilePath("/tmp", "libudf", path); - TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName); + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); // TODO check for failure of flush to disk taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); taosCloseFile(&file); @@ -494,7 +548,7 @@ static int32_t udfdParseArgs(int32_t argc, char *argv[]) { static int32_t udfdInitLog() { char logName[12] = {0}; snprintf(logName, sizeof(logName), "%slog", "udfd"); - return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); } void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { @@ -531,15 +585,7 @@ static int32_t udfdUvInit() { uv_pipe_open(&global.ctrlPipe, 0); uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); - char dnodeId[8] = {0}; - size_t dnodeIdSize; - int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); - if (err != 0) { - dnodeId[0] = '1'; - } - char listenPipeName[32] = {0}; - snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); - strcpy(global.listenPipeName, listenPipeName); + getUdfdPipeName(global.listenPipeName, UDF_LISTEN_PIPE_NAME_LEN); removeListeningPipe(); @@ -550,7 +596,7 @@ static int32_t udfdUvInit() { int r; fnInfo("bind to pipe %s", global.listenPipeName); - if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) { + if ((r = uv_pipe_bind(&global.listeningPipe, global.listenPipeName))) { fnError("Bind error %s", uv_err_name(r)); removeListeningPipe(); return -1; @@ -580,7 +626,7 @@ static int32_t udfdRun() { fnInfo("start the udfd"); int code = uv_run(global.loop, UV_RUN_DEFAULT); - fnInfo("udfd stopped. result: %s", uv_err_name(code)); + fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code); int codeClose = uv_loop_close(global.loop); fnDebug("uv loop close. result: %s", uv_err_name(codeClose)); udfdCloseClientRpc(); @@ -610,10 +656,11 @@ int main(int argc, char *argv[]) { return -1; } - if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) { + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { fnError("failed to start since read config error"); return -1; } + initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); return udfdRun(); } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 0727a4a1d2..a8d6fbd715 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -1,61 +1,84 @@ #include #include #include - #include "uv.h" + +#include "fnLog.h" #include "os.h" -#include "tudf.h" #include "tdatablock.h" +#include "tglobal.h" +#include "tudf.h" + +static int32_t parseArgs(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } + } + + return 0; +} + +static int32_t initLog() { + char logName[12] = {0}; + snprintf(logName, sizeof(logName), "%slog", "udfc"); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); +} int main(int argc, char *argv[]) { - UdfcHandle udfc; - udfcOpen(1, &udfc); - uv_sleep(1000); - char path[256] = {0}; - size_t cwdSize = 256; - int err = uv_cwd(path, &cwdSize); - if (err != 0) { - fprintf(stderr, "err cwd: %s\n", uv_strerror(err)); - return err; + parseArgs(argc, argv); + initLog(); + if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) { + fnError("failed to start since read config error"); + return -1; + } + + udfcOpen(); + uv_sleep(1000); + + UdfcFuncHandle handle; + + setupUdf("udf1", &handle); + + SSDataBlock block = {0}; + SSDataBlock *pBlock = █ + pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = 1; + pBlock->info.rows = 4; + char data[16] = {0}; + char bitmap[4] = {0}; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = data; + colInfo.nullbitmap = bitmap; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(&colInfo, j, &j); } - fprintf(stdout, "current working directory:%s\n", path); - strcat(path, "/libudf1.so"); + taosArrayPush(pBlock->pDataBlock, &colInfo); + } - UdfcFuncHandle handle; - SEpSet epSet; - setupUdf(udfc, "udf1", &epSet, &handle); + SScalarParam input = {0}; + input.numOfRows = pBlock->info.rows; + input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; + callUdfScalarFunc(handle, &input, 1, &output); - SSDataBlock block = {0}; - SSDataBlock* pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; - pBlock->info.rows = 4; - char data[16] = {0}; - char bitmap[4] = {0}; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = data; - colInfo.nullbitmap = bitmap; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); - } - - SScalarParam input = {0}; - input.numOfRows = pBlock->info.rows; - input.columnData = taosArrayGet(pBlock->pDataBlock, 0); - SScalarParam output = {0}; - callUdfScalarFunc(handle, &input, 1 , &output); - - SColumnInfoData *col = output.columnData; - for (int32_t i = 0; i < output.numOfRows; ++i) { - fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t))); - } - teardownUdf(handle); - - udfcClose(udfc); + SColumnInfoData *col = output.columnData; + for (int32_t i = 0; i < output.numOfRows; ++i) { + fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t))); + } + teardownUdf(handle); + udfcClose(); } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 676304f9a8..1d54448e55 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -671,9 +671,6 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) { static const char* jkScanPhysiPlanScanCols = "ScanCols"; static const char* jkScanPhysiPlanTableId = "TableId"; static const char* jkScanPhysiPlanTableType = "TableType"; -static const char* jkScanPhysiPlanScanOrder = "ScanOrder"; -static const char* jkScanPhysiPlanScanCount = "ScanCount"; -static const char* jkScanPhysiPlanReverseScanCount = "ReverseScanCount"; static const char* jkScanPhysiPlanTableName = "TableName"; static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { @@ -689,15 +686,6 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanOrder, pNode->order); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanCount, pNode->count); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanReverseScanCount, pNode->reverse); - } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanPhysiPlanTableName, nameToJson, &pNode->tableName); } @@ -718,15 +706,6 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanOrder, &pNode->order); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanCount, &pNode->count); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanPhysiPlanReverseScanCount, &pNode->reverse); - } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkScanPhysiPlanTableName, jsonToName, &pNode->tableName); } @@ -742,7 +721,8 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); } -static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag"; +static const char* jkTableScanPhysiPlanScanCount = "ScanCount"; +static const char* jkTableScanPhysiPlanReverseScanCount = "ReverseScanCount"; static const char* jkTableScanPhysiPlanStartKey = "StartKey"; static const char* jkTableScanPhysiPlanEndKey = "EndKey"; static const char* jkTableScanPhysiPlanRatio = "Ratio"; @@ -759,7 +739,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { int32_t code = physiScanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanFlag, pNode->scanFlag); + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanCount, pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanReverseScanCount, pNode->scanSeq[1]); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanStartKey, pNode->scanRange.skey); @@ -800,7 +783,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysiScanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanFlag, &pNode->scanFlag); + code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanCount, &pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanReverseScanCount, &pNode->scanSeq[1]); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanStartKey, &pNode->scanRange.skey); @@ -1677,7 +1663,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { return code; } -static const char* jkValueGenByCalc = "GenByCalc"; +static const char* jkValueLiteralSize = "LiteralSize"; static const char* jkValueLiteral = "Literal"; static const char* jkValueDuration = "Duration"; static const char* jkValueTranslate = "Translate"; @@ -1731,9 +1717,9 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) { int32_t code = exprNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkValueGenByCalc, pNode->genByCalc); + code = tjsonAddIntegerToObject(pJson, jkValueLiteralSize, NULL != pNode->literal ? strlen(pNode->literal) : 0); } - if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { + if (TSDB_CODE_SUCCESS == code && NULL != pNode->literal) { code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal); } if (TSDB_CODE_SUCCESS == code) { @@ -1803,10 +1789,11 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) { SValueNode* pNode = (SValueNode*)pObj; int32_t code = jsonToExprNode(pJson, pObj); + int32_t literalSize = 0; if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkValueGenByCalc, &pNode->genByCalc); + code = tjsonGetIntValue(pJson, jkValueLiteralSize, &literalSize); } - if (TSDB_CODE_SUCCESS == code && !pNode->genByCalc) { + if (TSDB_CODE_SUCCESS == code && literalSize > 0) { code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index f4e206af9d..712276be79 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -95,7 +95,6 @@ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pL SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); -SNode* createFunctionNodeNoArg(SAstCreateContext* pCxt, const SToken* pFuncName); SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt); SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index b1cf5759b7..42fd77ac1a 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -498,6 +498,7 @@ signed_literal(A) ::= NK_BOOL(B). signed_literal(A) ::= TIMESTAMP NK_STRING(B). { A = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &B); } signed_literal(A) ::= duration_literal(B). { A = releaseRawExprNode(pCxt, B); } signed_literal(A) ::= NULL. { A = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } +signed_literal(A) ::= literal_func(B). { A = releaseRawExprNode(pCxt, B); } %type literal_list { SNodeList* } %destructor literal_list { nodesDestroyList($$); } @@ -610,7 +611,10 @@ pseudo_column(A) ::= WDURATION(B). function_expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= star_func(B) NK_LP star_func_para_list(C) NK_RP(D). { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); } function_expression(A) ::= CAST(B) NK_LP expression(C) AS type_name(D) NK_RP(E). { A = createRawExprNodeExt(pCxt, &B, &E, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, C), D)); } -function_expression(A) ::= noarg_func(B) NK_LP NK_RP(C). { A = createRawExprNodeExt(pCxt, &B, &C, createFunctionNodeNoArg(pCxt, &B)); } +function_expression(A) ::= literal_func(B). { A = B; } + +literal_func(A) ::= noarg_func(B) NK_LP NK_RP(C). { A = createRawExprNodeExt(pCxt, &B, &C, createFunctionNode(pCxt, &B, NULL)); } +literal_func(A) ::= NOW(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); } %type noarg_func { SToken } %destructor noarg_func { } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index cbd57595bb..79e2c31154 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -368,39 +368,6 @@ SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNod return (SNode*)func; } -SNode* createFunctionNodeNoArg(SAstCreateContext* pCxt, const SToken* pFuncName) { - SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); - CHECK_OUT_OF_MEM(func); - char buf[64] = {0}; - - int32_t dataType; - switch (pFuncName->type) { - case TK_NOW: { - int64_t ts = taosGetTimestamp(TSDB_TIME_PRECISION_MILLI); - snprintf(buf, sizeof(buf), "%"PRId64, ts); - dataType = TSDB_DATA_TYPE_BIGINT; - break; - } - case TK_TODAY: { - int64_t ts = taosGetTimestampToday(TSDB_TIME_PRECISION_MILLI); - snprintf(buf, sizeof(buf), "%"PRId64, ts); - dataType = TSDB_DATA_TYPE_BIGINT; - break; - } - case TK_TIMEZONE: { - strncpy(buf, tsTimezoneStr, strlen(tsTimezoneStr)); - dataType = TSDB_DATA_TYPE_BINARY; - break; - } - } - SToken token = {.type = pFuncName->type, .n = strlen(buf), .z = buf}; - - SNodeList *pParameterList = createNodeList(pCxt, createValueNode(pCxt, dataType, &token)); - strncpy(func->functionName, pFuncName->z, pFuncName->n); - func->pParameterList = pParameterList; - return (SNode*)func; -} - SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt) { SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(func); @@ -443,10 +410,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTa } else { strncpy(realTable->table.tableAlias, pTableName->z, pTableName->n); } - strncpy(realTable->table.tableName, pTableName->z, pTableName->n); - if (NULL != pCxt->pQueryCxt->db) { - strcpy(realTable->useDbName, pCxt->pQueryCxt->db); - } + strncpy(realTable->table.tableName, pTableName->z, pTableName->n); return (SNode*)realTable; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index ef1f9ada01..88d0ca514d 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -24,109 +24,37 @@ typedef struct SCalcConstContext { int32_t code; } SCalcConstContext; -static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt); +static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery); + +static int32_t calcConstSubquery(SCalcConstContext* pCxt, STempTableNode* pTempTable) { + return calcConstQuery(pCxt, pTempTable->pSubquery, true); +} + +static int32_t calcConstNode(SNode** pNode) { + if (NULL == *pNode) { + return TSDB_CODE_SUCCESS; + } -static EDealRes doCalcConst(SNode** pNode, SCalcConstContext* pCxt) { SNode* pNew = NULL; - pCxt->code = scalarCalculateConstants(*pNode, &pNew); - if (TSDB_CODE_SUCCESS != pCxt->code) { - return DEAL_RES_ERROR; + int32_t code = scalarCalculateConstants(*pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + *pNode = pNew; } - ((SValueNode*)pNew)->genByCalc = true; - ((SValueNode*)pNew)->translate = true; - *pNode = pNew; - return DEAL_RES_CONTINUE; + return code; } -static bool isTimestampCol(SNode* pNode) { - if (NULL == pNode) { - return false; - } - return (QUERY_NODE_COLUMN == nodeType(pNode) && TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode*)pNode)->resType.type); -} - -static EDealRes stringToTimestamp(SCalcConstContext* pCxt, uint8_t precision, SValueNode* pVal) { - switch (pVal->node.resType.type) { - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_VARBINARY: { - int64_t val = 0; - if (TSDB_CODE_SUCCESS != convertStringToTimestamp(pVal->node.resType.type, pVal->datum.p, precision, &val)) { - pCxt->code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL, varDataVal(pVal->datum.p)); - return DEAL_RES_ERROR; - } - pVal->node.resType.type = TSDB_DATA_TYPE_TIMESTAMP; - pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; - taosMemoryFreeClear(pVal->datum.p); - pVal->datum.i = val; - break; - } - default: - break; - } - return DEAL_RES_CONTINUE; -} - -static EDealRes calcConstOperator(SOperatorNode** pNode, void* pContext) { - SCalcConstContext* pCxt = pContext; - SOperatorNode* pOp = *pNode; - if (QUERY_NODE_VALUE == nodeType(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { - return doCalcConst((SNode**)pNode, pCxt); - } - if (isTimestampCol(pOp->pLeft) && (NULL == pOp->pRight || QUERY_NODE_VALUE == nodeType(pOp->pRight))) { - return stringToTimestamp(pCxt, ((SColumnNode*)pOp->pLeft)->node.resType.precision, (SValueNode*)pOp->pRight); - } else if (isTimestampCol(pOp->pRight) && QUERY_NODE_VALUE == nodeType(pOp->pLeft)) { - return stringToTimestamp(pCxt, ((SColumnNode*)pOp->pRight)->node.resType.precision, (SValueNode*)pOp->pLeft); - } - return DEAL_RES_CONTINUE; -} - -static EDealRes calcConstFunction(SFunctionNode** pNode, void* pContext) { - SFunctionNode* pFunc = *pNode; - if (!fmIsScalarFunc(pFunc->funcId)) { - return DEAL_RES_CONTINUE; - } - SNode* pParam = NULL; - FOREACH(pParam, pFunc->pParameterList) { - if (QUERY_NODE_VALUE != nodeType(pParam)) { - return DEAL_RES_CONTINUE; +static int32_t calcConstList(SNodeList* pList) { + SNode* pNode = NULL; + FOREACH(pNode, pList) { + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(pNode, &pNew); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pNew); + } else { + return code; } } - return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); -} - -static EDealRes calcConstLogicCond(SLogicConditionNode** pNode, void* pContext) { - SLogicConditionNode* pCond = *pNode; - SNode* pParam = NULL; - FOREACH(pParam, pCond->pParameterList) { - // todo calc "true and c1 > 10" - if (QUERY_NODE_VALUE != nodeType(pParam)) { - return DEAL_RES_CONTINUE; - } - } - return doCalcConst((SNode**)pNode, (SCalcConstContext*)pContext); -} - -static EDealRes calcConstSubquery(STempTableNode** pNode, void* pContext) { - SCalcConstContext* pCxt = pContext; - pCxt->code = calcConstQuery(pCxt, (*pNode)->pSubquery); - return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); -} - -static EDealRes calcConst(SNode** pNode, void* pContext) { - switch (nodeType(*pNode)) { - case QUERY_NODE_OPERATOR: - return calcConstOperator((SOperatorNode**)pNode, pContext); - case QUERY_NODE_FUNCTION: - return calcConstFunction((SFunctionNode**)pNode, pContext); - case QUERY_NODE_LOGIC_CONDITION: - return calcConstLogicCond((SLogicConditionNode**)pNode, pContext); - case QUERY_NODE_TEMP_TABLE: - return calcConstSubquery((STempTableNode**)pNode, pContext); - default: - break; - } - return DEAL_RES_CONTINUE; + return TSDB_CODE_SUCCESS; } static bool isCondition(const SNode* pNode) { @@ -174,18 +102,41 @@ static int32_t rewriteCondition(SCalcConstContext* pCxt, SNode** pNode) { return pCxt->code; } -static int32_t rewriteConditionForFromTable(SCalcConstContext* pCxt, SNode* pTable) { - if (QUERY_NODE_JOIN_TABLE == nodeType(pTable)) { - SJoinTableNode* pJoin = (SJoinTableNode*)pTable; - pCxt->code = rewriteConditionForFromTable(pCxt, pJoin->pLeft); - if (TSDB_CODE_SUCCESS == pCxt->code) { - pCxt->code = rewriteConditionForFromTable(pCxt, pJoin->pRight); - } - if (TSDB_CODE_SUCCESS == pCxt->code && NULL != pJoin->pOnCond) { - pCxt->code = rewriteCondition(pCxt, &pJoin->pOnCond); - } +static int32_t calcConstCondition(SCalcConstContext* pCxt, SNode** pNode) { + int32_t code = rewriteCondition(pCxt, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = calcConstNode(pNode); } - return pCxt->code; + return code; +} + +static int32_t rewriteConditionForFromTable(SCalcConstContext* pCxt, SNode* pTable) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pTable)) { + case QUERY_NODE_TEMP_TABLE: { + code = calcConstSubquery(pCxt, (STempTableNode*)pTable); + break; + } + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoin = (SJoinTableNode*)pTable; + code = rewriteConditionForFromTable(pCxt, pJoin->pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = rewriteConditionForFromTable(pCxt, pJoin->pRight); + } + if (TSDB_CODE_SUCCESS == code && NULL != pJoin->pOnCond) { + code = calcConstCondition(pCxt, &pJoin->pOnCond); + } + // todo empty table + break; + } + default: + break; + } + return code; +} + +static int32_t calcConstFromTable(SCalcConstContext* pCxt, SSelectStmt* pSelect) { + return rewriteConditionForFromTable(pCxt, pSelect->pFromTable); } static void rewriteConstCondition(SSelectStmt* pSelect, SNode** pCond) { @@ -200,77 +151,133 @@ static void rewriteConstCondition(SSelectStmt* pSelect, SNode** pCond) { } } -static int32_t calcConstFromTable(SCalcConstContext* pCxt, SSelectStmt* pSelect) { - pCxt->code = rewriteConditionForFromTable(pCxt, pSelect->pFromTable); - if (TSDB_CODE_SUCCESS == pCxt->code) { - nodesRewriteExprPostOrder(&pSelect->pFromTable, calcConst, pCxt); - } - return pCxt->code; -} - -static int32_t calcConstCondition(SCalcConstContext* pCxt, SSelectStmt* pSelect, SNode** pCond) { +static int32_t calcConstSelectCondition(SCalcConstContext* pCxt, SSelectStmt* pSelect, SNode** pCond) { if (NULL == *pCond) { return TSDB_CODE_SUCCESS; } - pCxt->code = rewriteCondition(pCxt, pCond); - if (TSDB_CODE_SUCCESS == pCxt->code) { - nodesRewriteExprPostOrder(pCond, calcConst, pCxt); + int32_t code = rewriteCondition(pCxt, pCond); + if (TSDB_CODE_SUCCESS == code) { + code = calcConstNode(pCond); } - if (TSDB_CODE_SUCCESS == pCxt->code) { + if (TSDB_CODE_SUCCESS == code) { rewriteConstCondition(pSelect, pCond); } - return pCxt->code; + return code; } -static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect) { - nodesRewriteExprsPostOrder(pSelect->pProjectionList, calcConst, pCxt); - if (TSDB_CODE_SUCCESS == pCxt->code) { - pCxt->code = calcConstFromTable(pCxt, pSelect); +static int32_t calcConstProject(SNode* pProject, SNode** pNew) { + SArray* pAssociation = NULL; + if (NULL != ((SExprNode*)pProject)->pAssociation) { + pAssociation = taosArrayDup(((SExprNode*)pProject)->pAssociation); + if (NULL == pAssociation) { + return TSDB_CODE_OUT_OF_MEMORY; + } } - if (TSDB_CODE_SUCCESS == pCxt->code) { - pCxt->code = calcConstCondition(pCxt, pSelect, &pSelect->pWhere); + + int32_t code = scalarCalculateConstants(pProject, pNew); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE == nodeType(pNew) && NULL != pAssociation) { + int32_t size = taosArrayGetSize(pAssociation); + for (int32_t i = 0; i < size; ++i) { + SNode** pCol = taosArrayGet(pAssociation, i); + *pCol = nodesCloneNode(pNew); + if (NULL == *pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } } - if (TSDB_CODE_SUCCESS == pCxt->code) { - nodesRewriteExprsPostOrder(pSelect->pPartitionByList, calcConst, pCxt); - } - if (TSDB_CODE_SUCCESS == pCxt->code) { - nodesRewriteExprPostOrder(&pSelect->pWindow, calcConst, pCxt); - } - if (TSDB_CODE_SUCCESS == pCxt->code) { - nodesRewriteExprsPostOrder(pSelect->pGroupByList, calcConst, pCxt); - } - if (TSDB_CODE_SUCCESS == pCxt->code) { - pCxt->code = calcConstCondition(pCxt, pSelect, &pSelect->pHaving); - } - if (TSDB_CODE_SUCCESS == pCxt->code) { - nodesRewriteExprsPostOrder(pSelect->pOrderByList, calcConst, pCxt); - } - return pCxt->code; + return code; } -static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt) { - switch (nodeType(pStmt)) { - case QUERY_NODE_SELECT_STMT: - return calcConstSelect(pCxt, (SSelectStmt*)pStmt); - case QUERY_NODE_EXPLAIN_STMT: - return calcConstQuery(pCxt, ((SExplainStmt*)pStmt)->pQuery); - default: - break; +static int32_t calcConstProjections(SCalcConstContext* pCxt, SNodeList* pProjections, bool subquery) { + SNode* pProj = NULL; + WHERE_EACH(pProj, pProjections) { + if (subquery && NULL == ((SExprNode*)pProj)->pAssociation) { + ERASE_NODE(pProjections); + continue; + } + SNode* pNew = NULL; + int32_t code = calcConstProject(pProj, &pNew); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(pNew); + } else { + return code; + } + WHERE_NEXT; } return TSDB_CODE_SUCCESS; } -static bool isEmptyResultQuery(SNode* pStmt) { +static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { + int32_t code = calcConstProjections(pCxt, pSelect->pProjectionList, subquery); + if (TSDB_CODE_SUCCESS == code) { + code = calcConstFromTable(pCxt, pSelect); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstSelectCondition(pCxt, pSelect, &pSelect->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstList(pSelect->pPartitionByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstNode(&pSelect->pWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstList(pSelect->pGroupByList); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstSelectCondition(pCxt, pSelect, &pSelect->pHaving); + } + if (TSDB_CODE_SUCCESS == code) { + code = calcConstList(pSelect->pOrderByList); + } + return code; +} + +static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery) { + int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(pStmt)) { case QUERY_NODE_SELECT_STMT: - return ((SSelectStmt*)pStmt)->isEmptyResult; + code = calcConstSelect(pCxt, (SSelectStmt*)pStmt, subquery); + break; case QUERY_NODE_EXPLAIN_STMT: - return isEmptyResultQuery(((SExplainStmt*)pStmt)->pQuery); + code = calcConstQuery(pCxt, ((SExplainStmt*)pStmt)->pQuery, subquery); + break; + case QUERY_NODE_SET_OPERATOR: { + SSetOperator* pSetOp = (SSetOperator*)pStmt; + code = calcConstQuery(pCxt, pSetOp->pLeft, subquery); + if (TSDB_CODE_SUCCESS == code) { + code = calcConstQuery(pCxt, pSetOp->pRight, subquery); + } + break; + } default: break; } - return false; + return code; +} + +static bool isEmptyResultQuery(SNode* pStmt) { + bool isEmptyResult = false; + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + isEmptyResult = ((SSelectStmt*)pStmt)->isEmptyResult; + break; + case QUERY_NODE_EXPLAIN_STMT: + isEmptyResult = isEmptyResultQuery(((SExplainStmt*)pStmt)->pQuery); + break; + case QUERY_NODE_SET_OPERATOR: { + SSetOperator* pSetOp = (SSetOperator*)pStmt; + isEmptyResult = isEmptyResultQuery(pSetOp->pLeft); + if (isEmptyResult) { + isEmptyResult = isEmptyResultQuery(pSetOp->pRight); + } + break; + } + default: + break; + } + return isEmptyResult; } int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { @@ -280,9 +287,9 @@ int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) { .msgBuf.len = pParseCxt->msgLen, .code = TSDB_CODE_SUCCESS }; - int32_t code = calcConstQuery(&cxt, pQuery->pRoot); - if (TSDB_CODE_SUCCESS == code) { - pQuery->execMode = isEmptyResultQuery(pQuery->pRoot) ? QUERY_EXEC_MODE_EMPTY_RESULT : pQuery->execMode; + int32_t code = calcConstQuery(&cxt, pQuery->pRoot, false); + if (TSDB_CODE_SUCCESS == code && isEmptyResultQuery(pQuery->pRoot)) { + pQuery->execMode = QUERY_EXEC_MODE_EMPTY_RESULT; } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 01a386427d..01b46c7e1e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -19,6 +19,7 @@ #include "cmdnodes.h" #include "functionMgt.h" #include "parUtil.h" +#include "scalar.h" #include "tglobal.h" #include "ttime.h" @@ -198,7 +199,7 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo* pInfo) { SParseContext* pParCxt = pCxt->pParseCxt; - SName name; + SName name; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); @@ -305,7 +306,10 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SColumnNode* pCol) { pCol->pProjectRef = (SNode*)pExpr; - nodesListAppend(pExpr->pAssociationList, (SNode*)pCol); + if (NULL == pExpr->pAssociation) { + pExpr->pAssociation = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + } + taosArrayPush(pExpr->pAssociation, &pCol); if (NULL != pTable) { strcpy(pCol->tableAlias, pTable->tableAlias); } else if (QUERY_NODE_COLUMN == nodeType(pExpr)) { @@ -543,7 +547,9 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { TSDB_DATA_TYPE_BLOB == rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } - if (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) { + if ((TSDB_DATA_TYPE_TIMESTAMP == ldt.type && TSDB_DATA_TYPE_TIMESTAMP == rdt.type) || + (TSDB_DATA_TYPE_TIMESTAMP == ldt.type && IS_VAR_DATA_TYPE(rdt.type)) || + (TSDB_DATA_TYPE_TIMESTAMP == rdt.type && IS_VAR_DATA_TYPE(ldt.type))) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } @@ -558,8 +564,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; } } else if (nodesIsComparisonOp(pOp)) { - if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || - TSDB_DATA_TYPE_BLOB == rdt.type) { + if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { @@ -567,7 +572,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - } else if (nodesIsJsonOp(pOp)){ + } else if (nodesIsJsonOp(pOp)) { if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } @@ -586,11 +591,14 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) { } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { - SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); - } - pCxt->errCode = fmGetFuncResultType(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); + SFmGetFuncInfoParam param = { + .pCtg = pCxt->pParseCxt->pCatalog, + .pRpc = pCxt->pParseCxt->pTransporter, + .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet, + .pErrBuf = pCxt->msgBuf.buf, + .errBufLen = pCxt->msgBuf.len + }; + pCxt->errCode = fmGetFuncInfo(¶m, pFunc); if (TSDB_CODE_SUCCESS != pCxt->errCode) { return DEAL_RES_ERROR; } @@ -756,6 +764,19 @@ static int32_t toVgroupsInfo(SArray* pVgs, SVgroupsInfo** pVgsInfo) { return TSDB_CODE_SUCCESS; } +static int32_t addMnodeToVgroupList(const SEpSet* pEpSet, SArray** pVgroupList) { + if (NULL == *pVgroupList) { + *pVgroupList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVgroupInfo)); + if (NULL == *pVgroupList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + SVgroupInfo vg = { .vgId = MNODE_HANDLE }; + memcpy(&vg.epSet, pEpSet, sizeof(SEpSet)); + taosArrayPush(*pVgroupList, &vg); + return TSDB_CODE_SUCCESS; +} + static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { return TSDB_CODE_SUCCESS; @@ -763,12 +784,20 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea int32_t code = TSDB_CODE_SUCCESS; SArray* vgroupList = NULL; - if ('\0' != pRealTable->useDbName[0]) { - code = getDBVgInfo(pCxt, pRealTable->useDbName, &vgroupList); + if ('\0' != pRealTable->qualDbName[0]) { + // todo release after mnode can be processed + // if (0 != strcmp(pRealTable->qualDbName, TSDB_INFORMATION_SCHEMA_DB)) { + code = getDBVgInfo(pCxt, pRealTable->qualDbName, &vgroupList); + // } } else { code = getDBVgInfoImpl(pCxt, pName, &vgroupList); } + // todo release after mnode can be processed + // if (TSDB_CODE_SUCCESS == code) { + // code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); + // } + if (TSDB_CODE_SUCCESS == code) { code = toVgroupsInfo(vgroupList, &pRealTable->pVgroupList); } @@ -1266,7 +1295,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* static EDealRes checkStateExpr(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { STranslateContext* pCxt = pContext; - SColumnNode* pCol = (SColumnNode*)pNode; + SColumnNode* pCol = (SColumnNode*)pNode; if (!IS_INTEGER_TYPE(pCol->node.resType.type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); } @@ -1343,7 +1372,8 @@ static int32_t translateFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { } static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { - if ((NULL != pSelect->pLimit && pSelect->pLimit->offset < 0) || (NULL != pSelect->pSlimit && pSelect->pSlimit->offset < 0)) { + if ((NULL != pSelect->pLimit && pSelect->pLimit->offset < 0) || + (NULL != pSelect->pSlimit && pSelect->pSlimit->offset < 0)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_OFFSET_LESS_ZERO); } @@ -1436,7 +1466,7 @@ static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* p SExprNode* pLeftExpr = (SExprNode*)pLeft; SExprNode* pRightExpr = (SExprNode*)pRight; if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) { - SNode* pRightFunc = NULL; + SNode* pRightFunc = NULL; int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1648,7 +1678,8 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { (TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) || (pKeep2->isDuration && (TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, pKeep2->unit); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, + pKeep2->unit); } int32_t daysToKeep0 = getBigintFromValueNode(pKeep0); @@ -1657,7 +1688,7 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP || daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2, - TSDB_MIN_KEEP, TSDB_MAX_KEEP); + TSDB_MIN_KEEP, TSDB_MAX_KEEP); } if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) { @@ -1689,7 +1720,8 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return TSDB_CODE_SUCCESS; } -static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) { +static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, + bool alter) { if (NULL == pOptions->pDaysPerFile && NULL == pOptions->pKeep) { return TSDB_CODE_SUCCESS; } @@ -1697,7 +1729,7 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa int64_t daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pOptions->pKeep, 0), alter ? -1 : TSDB_DEFAULT_KEEP); if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) { SDbCfgInfo dbCfg; - int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); + int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -1710,7 +1742,8 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa return TSDB_CODE_SUCCESS; } -static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) { +static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, + bool alter) { int32_t code = checkRangeOption(pCxt, "totalBlocks", pOptions->pNumOfBlocks, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); if (TSDB_CODE_SUCCESS == code) { @@ -1916,7 +1949,7 @@ static int32_t checTableFactorOption(STranslateContext* pCxt, SValueNode* pVal) } if (pVal->datum.d < TSDB_MIN_DB_FILE_FACTOR || pVal->datum.d > TSDB_MAX_DB_FILE_FACTOR) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", pVal->datum.d, - TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR); + TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR); } } return TSDB_CODE_SUCCESS; @@ -1943,7 +1976,7 @@ static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) SNode* pNode; FOREACH(pNode, pStmt->pTags) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){ + if (pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); } } @@ -1958,11 +1991,6 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs if (1 != LIST_LENGTH(pFuncs)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION); } - SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0); - SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); - } return TSDB_CODE_SUCCESS; } @@ -1992,13 +2020,6 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt return code; } -static int32_t getAggregationMethod(SNodeList* pFuncs) { - if (NULL == pFuncs) { - return -1; - } - return ((SFunctionNode*)nodesListGetNode(pFuncs, 0))->funcId; -} - static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchema) { int8_t flags = 0; if (pCol->sma) { @@ -2014,10 +2035,10 @@ static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchem typedef struct SSampleAstInfo { const char* pDbName; const char* pTableName; - SNodeList* pFuncs; - SNode* pInterval; - SNode* pOffset; - SNode* pSliding; + SNodeList* pFuncs; + SNode* pInterval; + SNode* pOffset; + SNode* pSliding; STableMeta* pRollupTableMeta; } SSampleAstInfo; @@ -2087,8 +2108,8 @@ static SNode* makeIntervalVal(SRetention* pRetension, int8_t precision) { return NULL; } int64_t timeVal = convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit); - char buf[20] = {0}; - int32_t len = snprintf(buf, sizeof(buf), "%"PRId64"%c", timeVal, pRetension->freqUnit); + char buf[20] = {0}; + int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); pVal->literal = strndup(buf, len); if (NULL == pVal->literal) { nodesDestroyNode(pVal); @@ -2131,7 +2152,7 @@ static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) { SNode* pFunc = NULL; FOREACH(pFunc, pStmt->pOptions->pFuncs) { SNode* pCol = NULL; - bool primaryKey = true; + bool primaryKey = true; FOREACH(pCol, pStmt->pCols) { if (primaryKey) { primaryKey = false; @@ -2148,7 +2169,7 @@ static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) { } static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precision) { - int32_t numOfField = LIST_LENGTH(pStmt->pCols) + LIST_LENGTH(pStmt->pTags); + int32_t numOfField = LIST_LENGTH(pStmt->pCols) + LIST_LENGTH(pStmt->pTags); STableMeta* pMeta = taosMemoryCalloc(1, sizeof(STableMeta) + numOfField * sizeof(SSchema)); if (NULL == pMeta) { return NULL; @@ -2159,7 +2180,7 @@ static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precisi pMeta->tableInfo.numOfColumns = LIST_LENGTH(pStmt->pCols); int32_t index = 0; - SNode* pCol = NULL; + SNode* pCol = NULL; FOREACH(pCol, pStmt->pCols) { toSchema((SColumnDefNode*)pCol, index + 1, pMeta->schema + index); ++index; @@ -2173,8 +2194,8 @@ static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precisi return pMeta; } -static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, - SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, SSampleAstInfo* pInfo) { +static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, SRetention* pRetension, + int8_t precision, SSampleAstInfo* pInfo) { pInfo->pDbName = pStmt->dbName; pInfo->pTableName = pStmt->tableName; pInfo->pFuncs = createRollupFuncs(pStmt); @@ -2186,10 +2207,10 @@ static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, return TSDB_CODE_SUCCESS; } -static int32_t getRollupAst(STranslateContext* pCxt, - SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, char** pAst, int32_t* pLen) { +static int32_t getRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, + char** pAst, int32_t* pLen) { SSampleAstInfo info = {0}; - int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info); + int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info); if (TSDB_CODE_SUCCESS == code) { code = buildSampleAst(pCxt, &info, pAst, pLen); } @@ -2199,17 +2220,17 @@ static int32_t getRollupAst(STranslateContext* pCxt, static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); - int32_t num = taosArrayGetSize(dbCfg.pRetensions); + int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); + int32_t num = taosArrayGetSize(dbCfg.pRetensions); if (TSDB_CODE_SUCCESS != code || num < 2) { return code; } for (int32_t i = 1; i < num; ++i) { - SRetention *pRetension = taosArrayGet(dbCfg.pRetensions, i); + SRetention* pRetension = taosArrayGet(dbCfg.pRetensions, i); STranslateContext cxt = {0}; initTranslateContext(pCxt->pParseCxt, &cxt); - code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, - 1 == i ? &pReq->pAst1 : &pReq->pAst2, 1 == i ? &pReq->ast1Len : &pReq->ast2Len); + code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2, + 1 == i ? &pReq->ast1Len : &pReq->ast2Len); destroyTranslateContext(&cxt); if (TSDB_CODE_SUCCESS != code) { break; @@ -2220,7 +2241,6 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { pReq->igExists = pStmt->ignoreExists; - pReq->aggregationMethod = getAggregationMethod(pStmt->pOptions->pFuncs); pReq->xFilesFactor = GET_OPTION_VAL(pStmt->pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR); pReq->delay = GET_OPTION_VAL(pStmt->pOptions->pDelay, TSDB_DEFAULT_DB_DELAY); columnDefNodeToField(pStmt->pCols, &pReq->pColumns); @@ -2243,7 +2263,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { SMCreateStbReq createReq = {0}; - int32_t code = checkCreateTable(pCxt, pStmt); + int32_t code = checkCreateTable(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { code = buildCreateStbReq(pCxt, pStmt, &createReq); } @@ -2287,7 +2307,8 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { SName tableName; - return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pStmt->ignoreNotExists); + return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), + pStmt->ignoreNotExists); } static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) { @@ -2419,7 +2440,7 @@ static int32_t nodeTypeToShowType(ENodeType nt) { case QUERY_NODE_SHOW_QUERIES_STMT: return TSDB_MGMT_TABLE_QUERIES; case QUERY_NODE_SHOW_VARIABLE_STMT: - return 0; // todo + return 0; // todo default: break; } @@ -2461,8 +2482,8 @@ static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexSt pInfo->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); pInfo->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); if (NULL == pInfo->pFuncs || NULL == pInfo->pInterval || - (NULL != pStmt->pOptions->pOffset && NULL == pInfo->pOffset) || - (NULL != pStmt->pOptions->pSliding && NULL == pInfo->pSliding)) { + (NULL != pStmt->pOptions->pOffset && NULL == pInfo->pOffset) || + (NULL != pStmt->pOptions->pSliding && NULL == pInfo->pSliding)) { return TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; @@ -2470,7 +2491,7 @@ static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexSt static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) { SSampleAstInfo info = {0}; - int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info); + int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info); if (TSDB_CODE_SUCCESS == code) { code = buildSampleAst(pCxt, &info, pAst, pLen); } @@ -2614,37 +2635,66 @@ static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponen (FSerializeFunc)tSerializeSCreateDropMQSBNodeReq, &dropReq); } -static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { - SCMCreateTopicReq createReq = {0}; +static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SCMCreateTopicReq* pReq) { + SName name; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); + tNameGetFullDbName(&name, pReq->name); + /*tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name);*/ + pReq->igExists = pStmt->ignoreExists; + pReq->withTbName = pStmt->pOptions->withTable; + pReq->withSchema = pStmt->pOptions->withSchema; + pReq->withTag = pStmt->pOptions->withTag; - if (NULL != pStmt->pQuery) { - pCxt->pParseCxt->topicQuery = true; - int32_t code = translateQuery(pCxt, pStmt->pQuery); - if (TSDB_CODE_SUCCESS == code) { - code = nodesNodeToString(pStmt->pQuery, false, &createReq.ast, NULL); - } - if (TSDB_CODE_SUCCESS != code) { - return code; - } - } else { - strcpy(createReq.subscribeDbName, pStmt->subscribeDbName); - } - - createReq.sql = strdup(pCxt->pParseCxt->pSql); - if (NULL == createReq.sql) { + pReq->sql = strdup(pCxt->pParseCxt->pSql); + if (NULL == pReq->sql) { return TSDB_CODE_OUT_OF_MEMORY; } - SName name; - // tNameSetDbName(&name, pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, strlen(pCxt->pParseCxt->db)); - // tNameGetFullDbName(&name, createReq.name); - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), createReq.name); - createReq.igExists = pStmt->ignoreExists; - createReq.withTbName = pStmt->pOptions->withTable; - createReq.withSchema = pStmt->pOptions->withSchema; - createReq.withTag = pStmt->pOptions->withTag; + int32_t code = TSDB_CODE_SUCCESS; - int32_t code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq); + const char* dbName; + if (NULL != pStmt->pQuery) { + dbName = ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName; + pCxt->pParseCxt->topicQuery = true; + code = translateQuery(pCxt, pStmt->pQuery); + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); + } + } else { + dbName = pStmt->subscribeDbName; + } + tNameSetDbName(&name, pCxt->pParseCxt->acctId, dbName, strlen(dbName)); + tNameGetFullDbName(&name, pReq->subscribeDbName); + + return code; +} + +static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { + if (NULL == pStmt->pQuery) { + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { + SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; + if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) && + NULL == pSelect->pGroupByList && NULL == pSelect->pLimit && NULL == pSelect->pSlimit && + NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) { + return TSDB_CODE_SUCCESS; + } + } + + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TOPIC_QUERY); +} + +static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { + SCMCreateTopicReq createReq = {0}; + int32_t code = checkCreateTopic(pCxt, pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = buildCreateTopicReq(pCxt, pStmt, &createReq); + } + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq); + } tFreeSCMCreateTopicReq(&createReq); return code; } @@ -2736,7 +2786,7 @@ static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pSt return TSDB_CODE_SUCCESS; } -static int32_t readFromFile(char* pName, int32_t *len, char **buf) { +static int32_t readFromFile(char* pName, int32_t* len, char** buf) { int64_t filesize = 0; if (taosStatFile(pName, &filesize, NULL) < 0) { return TAOS_SYSTEM_ERROR(errno); @@ -3154,7 +3204,7 @@ static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) } if (NULL != pShow->pDbName) { - strcpy(((SRealTableNode*)pSelect->pFromTable)->useDbName, ((SValueNode*)pShow->pDbName)->literal); + strcpy(((SRealTableNode*)pSelect->pFromTable)->qualDbName, ((SValueNode*)pShow->pDbName)->literal); } return TSDB_CODE_SUCCESS; @@ -3196,14 +3246,6 @@ static int32_t buildSmaParam(STableOptions* pOptions, SVCreateTbReq* pReq) { } pReq->ntbCfg.pRSmaParam->delay = GET_OPTION_VAL(pOptions->pDelay, TSDB_DEFAULT_DB_DELAY); pReq->ntbCfg.pRSmaParam->xFilesFactor = GET_OPTION_VAL(pOptions->pFilesFactor, TSDB_DEFAULT_DB_FILE_FACTOR); - pReq->ntbCfg.pRSmaParam->nFuncIds = LIST_LENGTH(pOptions->pFuncs); - pReq->ntbCfg.pRSmaParam->pFuncIds = taosMemoryCalloc(pReq->ntbCfg.pRSmaParam->nFuncIds, sizeof(func_id_t)); - if (NULL == pReq->ntbCfg.pRSmaParam->pFuncIds) { - return TSDB_CODE_OUT_OF_MEMORY; - } - int32_t index = 0; - SNode* pFunc = NULL; - FOREACH(pFunc, pOptions->pFuncs) { pReq->ntbCfg.pRSmaParam->pFuncIds[index++] = ((SFunctionNode*)pFunc)->funcId; } return TSDB_CODE_SUCCESS; } @@ -3381,27 +3423,41 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, SKVRowBuilder* pBuilder) { - if(pSchema->type == TSDB_DATA_TYPE_JSON){ - if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + if (pSchema->type == TSDB_DATA_TYPE_JSON) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); } return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); } - if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { - return pCxt->errCode; - } - - if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){ + if (pVal->node.resType.type == TSDB_DATA_TYPE_NULL) { // todo - }else{ - tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); + } else { + tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), + IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); } return TSDB_CODE_SUCCESS; } +static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) { + if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) { + return pCxt->errCode; + } + return scalarCalculateConstants((SNode*)pFunc, (SNode**)pVal); +} + +static int32_t translateTagVal(STranslateContext* pCxt, SNode* pNode, SValueNode** pVal) { + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + return createValueFromFunction(pCxt, (SFunctionNode*)pNode, pVal); + } else if (QUERY_NODE_VALUE == nodeType(pNode)) { + return (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pNode) ? pCxt->errCode : TSDB_CODE_SUCCESS); + } else { + return TSDB_CODE_FAILED; + } +} + static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, SKVRowBuilder* pBuilder) { int32_t numOfTags = getNumOfTags(pSuperTableMeta); @@ -3411,8 +3467,8 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); - SNode * pTag, *pVal; - FORBOTH(pTag, pStmt->pSpecificTags, pVal, pStmt->pValsOfTags) { + SNode * pTag, *pNode; + FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) { SColumnNode* pCol = (SColumnNode*)pTag; SSchema* pSchema = NULL; for (int32_t i = 0; i < numOfTags; ++i) { @@ -3424,7 +3480,18 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla if (NULL == pSchema) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); } - int32_t code = addValToKVRow(pCxt, (SValueNode*)pVal, pSchema, pBuilder); + SValueNode* pVal = NULL; + int32_t code = translateTagVal(pCxt, pNode, &pVal); + if (TSDB_CODE_SUCCESS == code) { + if (NULL == pVal) { + pVal = (SValueNode*)pNode; + } else { + REPLACE_LIST2_NODE(pVal); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = addValToKVRow(pCxt, pVal, pSchema, pBuilder); + } if (TSDB_CODE_SUCCESS != code) { return code; } @@ -3440,10 +3507,21 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); - SNode* pVal; + SNode* pNode; int32_t index = 0; - FOREACH(pVal, pStmt->pValsOfTags) { - int32_t code = addValToKVRow(pCxt, (SValueNode*)pVal, pTagSchema + index++, pBuilder); + FOREACH(pNode, pStmt->pValsOfTags) { + SValueNode* pVal = NULL; + int32_t code = translateTagVal(pCxt, pNode, &pVal); + if (TSDB_CODE_SUCCESS == code) { + if (NULL == pVal) { + pVal = (SValueNode*)pNode; + } else { + REPLACE_NODE(pVal); + } + } + if (TSDB_CODE_SUCCESS == code) { + code = addValToKVRow(pCxt, pVal, pTagSchema + index++, pBuilder); + } if (TSDB_CODE_SUCCESS != code) { return code; } @@ -3633,7 +3711,9 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } - pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot); + if (nodeType(pQuery->pRoot) == QUERY_NODE_SELECT_STMT) { + pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot); + } } if (NULL != pCxt->pDbs) { @@ -3665,7 +3745,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = {0}; - int32_t code = initTranslateContext(pParseCxt, &cxt); + int32_t code = initTranslateContext(pParseCxt, &cxt); if (TSDB_CODE_SUCCESS == code) { code = fmFuncMgtInit(); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 7e41bbe3fd..8335d491ed 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -30,8 +30,6 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Column ambiguously defined: %s"; case TSDB_CODE_PAR_WRONG_VALUE_TYPE: return "Invalid value type: %s"; - case TSDB_CODE_PAR_INVALID_FUNTION: - return "Invalid function name: %s"; case TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION: return "There mustn't be aggregation"; case TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT: @@ -128,6 +126,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "soffset/offset can not be less than 0"; case TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY: return "slimit/soffset only available for PARTITION BY query"; + case TSDB_CODE_PAR_INVALID_TOPIC_QUERY: + return "Invalid topic query"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 97f718c151..18f0809dfd 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -32,7 +32,6 @@ #include #include -#include "os.h" #include "functionMgt.h" #include "nodes.h" #include "parToken.h" @@ -101,24 +100,24 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 346 +#define YYNOCODE 347 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SAlterOption yy145; - EOrder yy250; - EOperatorType yy348; - int32_t yy376; - SDataType yy380; - SNode* yy456; - SToken yy517; - EFillMode yy534; - ENullOrder yy645; - SNodeList* yy652; - bool yy673; - EJoinType yy684; + EJoinType yy84; + SDataType yy156; + ENullOrder yy181; + EOrder yy272; + EFillMode yy284; + EOperatorType yy304; + SAlterOption yy475; + SToken yy555; + SNodeList* yy568; + int32_t yy610; + bool yy617; + SNode* yy662; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -134,17 +133,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 574 -#define YYNRULE 441 +#define YYNSTATE 575 +#define YYNRULE 444 #define YYNTOKEN 231 -#define YY_MAX_SHIFT 573 -#define YY_MIN_SHIFTREDUCE 855 -#define YY_MAX_SHIFTREDUCE 1295 -#define YY_ERROR_ACTION 1296 -#define YY_ACCEPT_ACTION 1297 -#define YY_NO_ACTION 1298 -#define YY_MIN_REDUCE 1299 -#define YY_MAX_REDUCE 1739 +#define YY_MAX_SHIFT 574 +#define YY_MIN_SHIFTREDUCE 857 +#define YY_MAX_SHIFTREDUCE 1300 +#define YY_ERROR_ACTION 1301 +#define YY_ACCEPT_ACTION 1302 +#define YY_NO_ACTION 1303 +#define YY_MIN_REDUCE 1304 +#define YY_MAX_REDUCE 1747 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -211,544 +210,573 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1906) +#define YY_ACTTAB_COUNT (2021) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 487, 1718, 1590, 1502, 273, 474, 322, 290, 285, 78, - /* 10 */ 146, 1514, 33, 31, 1717, 1410, 386, 407, 1715, 487, - /* 20 */ 282, 1576, 1115, 1470, 1576, 1421, 1606, 125, 78, 272, - /* 30 */ 447, 1378, 415, 450, 1468, 393, 1572, 1578, 1113, 1572, - /* 40 */ 1578, 26, 205, 470, 1421, 1718, 174, 1562, 1576, 410, - /* 50 */ 12, 33, 31, 1237, 404, 105, 486, 1121, 137, 282, - /* 60 */ 173, 1115, 1715, 1572, 1579, 54, 74, 1591, 473, 1593, - /* 70 */ 1594, 469, 486, 464, 59, 1, 1656, 1113, 100, 487, - /* 80 */ 275, 1652, 132, 30, 29, 28, 1416, 45, 326, 12, - /* 90 */ 44, 54, 103, 486, 201, 293, 1121, 286, 570, 1348, - /* 100 */ 430, 1683, 1505, 1507, 1421, 122, 449, 133, 1663, 1664, - /* 110 */ 1114, 1668, 1417, 1423, 1, 1300, 983, 510, 509, 508, - /* 120 */ 987, 507, 989, 990, 506, 992, 503, 36, 998, 500, - /* 130 */ 1000, 1001, 497, 494, 9, 8, 89, 570, 1136, 88, - /* 140 */ 87, 86, 85, 84, 83, 82, 81, 80, 36, 1114, - /* 150 */ 1116, 546, 545, 544, 543, 297, 365, 542, 541, 540, - /* 160 */ 108, 535, 534, 533, 532, 531, 530, 529, 528, 115, - /* 170 */ 524, 1119, 1120, 63, 1164, 1165, 1166, 1167, 1168, 1169, - /* 180 */ 1170, 466, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1116, - /* 190 */ 891, 1606, 292, 402, 1414, 396, 416, 1297, 471, 401, - /* 200 */ 122, 138, 102, 1140, 397, 395, 384, 398, 1423, 1213, - /* 210 */ 1119, 1120, 394, 1164, 1165, 1166, 1167, 1168, 1169, 1170, - /* 220 */ 466, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 33, 31, - /* 230 */ 101, 1590, 414, 440, 389, 1718, 282, 250, 1115, 1136, - /* 240 */ 34, 32, 30, 29, 28, 412, 344, 1718, 137, 356, - /* 250 */ 1670, 298, 1715, 138, 1113, 1606, 1299, 392, 357, 1470, - /* 260 */ 1716, 321, 471, 320, 1715, 287, 12, 33, 31, 487, - /* 270 */ 1468, 138, 470, 1121, 1667, 282, 1562, 1115, 327, 138, - /* 280 */ 98, 97, 96, 95, 94, 93, 92, 91, 90, 1412, - /* 290 */ 1718, 1, 1138, 1113, 1421, 74, 1591, 473, 1593, 1594, - /* 300 */ 469, 328, 464, 137, 1251, 1656, 128, 1715, 69, 275, - /* 310 */ 1652, 1730, 1121, 138, 570, 1261, 1470, 1462, 65, 138, - /* 320 */ 1690, 1077, 294, 200, 56, 270, 1114, 1468, 175, 1079, - /* 330 */ 7, 251, 355, 264, 1562, 350, 349, 348, 347, 346, - /* 340 */ 6, 343, 342, 341, 340, 339, 335, 334, 333, 332, - /* 350 */ 331, 330, 329, 570, 434, 1259, 1260, 1262, 1263, 101, - /* 360 */ 892, 441, 891, 389, 89, 1114, 1116, 88, 87, 86, - /* 370 */ 85, 84, 83, 82, 81, 80, 265, 1322, 263, 262, - /* 380 */ 893, 388, 437, 124, 48, 1311, 392, 1119, 1120, 1078, - /* 390 */ 1164, 1165, 1166, 1167, 1168, 1169, 1170, 466, 1175, 1176, - /* 400 */ 1177, 1178, 1179, 1180, 1181, 1116, 1398, 22, 295, 34, - /* 410 */ 32, 30, 29, 28, 1351, 523, 122, 34, 32, 30, - /* 420 */ 29, 28, 1562, 1141, 1423, 228, 1119, 1120, 1451, 1164, - /* 430 */ 1165, 1166, 1167, 1168, 1169, 1170, 466, 1175, 1176, 1177, - /* 440 */ 1178, 1179, 1180, 1181, 33, 31, 1182, 254, 24, 1201, - /* 450 */ 442, 438, 282, 424, 1115, 1590, 138, 945, 34, 32, - /* 460 */ 30, 29, 28, 566, 565, 487, 402, 107, 396, 1206, - /* 470 */ 1113, 518, 401, 1151, 336, 102, 947, 397, 395, 1606, - /* 480 */ 398, 1199, 313, 33, 31, 394, 471, 391, 390, 1121, - /* 490 */ 1421, 282, 1139, 1115, 521, 1321, 470, 1232, 425, 474, - /* 500 */ 1562, 315, 487, 1320, 23, 1515, 451, 7, 1590, 1113, - /* 510 */ 1137, 337, 1319, 517, 516, 515, 487, 514, 1318, 73, - /* 520 */ 1591, 473, 1593, 1594, 469, 364, 464, 1421, 1121, 1656, - /* 530 */ 570, 1200, 1606, 253, 1652, 487, 351, 1718, 70, 471, - /* 540 */ 1562, 1421, 1114, 186, 1418, 1718, 7, 122, 1562, 470, - /* 550 */ 137, 1205, 106, 1562, 1715, 1424, 431, 1562, 137, 1413, - /* 560 */ 1421, 513, 1715, 1562, 34, 32, 30, 29, 28, 570, - /* 570 */ 144, 487, 249, 1591, 473, 1593, 1594, 469, 1317, 464, - /* 580 */ 1543, 1114, 1116, 148, 147, 1316, 25, 280, 1194, 1195, - /* 590 */ 1196, 1197, 1198, 1202, 1203, 1204, 1421, 52, 453, 1315, - /* 600 */ 51, 400, 399, 1119, 1120, 526, 1164, 1165, 1166, 1167, - /* 610 */ 1168, 1169, 1170, 466, 1175, 1176, 1177, 1178, 1179, 1180, - /* 620 */ 1181, 1116, 1314, 1562, 1313, 34, 32, 30, 29, 28, - /* 630 */ 1562, 457, 1310, 34, 32, 30, 29, 28, 1399, 520, - /* 640 */ 519, 1236, 1119, 1120, 1562, 1164, 1165, 1166, 1167, 1168, - /* 650 */ 1169, 1170, 466, 1175, 1176, 1177, 1178, 1179, 1180, 1181, - /* 660 */ 33, 31, 1309, 254, 1670, 1590, 1670, 1562, 282, 1562, - /* 670 */ 1115, 34, 32, 30, 29, 28, 447, 1562, 34, 32, - /* 680 */ 30, 29, 28, 289, 288, 1406, 1113, 365, 1666, 1606, - /* 690 */ 1665, 1397, 1583, 1129, 487, 1292, 450, 1199, 527, 487, - /* 700 */ 1393, 105, 487, 484, 1581, 1121, 470, 1562, 485, 1122, - /* 710 */ 1562, 219, 539, 537, 1506, 1507, 487, 1408, 1590, 1421, - /* 720 */ 1308, 451, 1404, 1, 1421, 296, 1307, 1421, 1121, 74, - /* 730 */ 1591, 473, 1593, 1594, 469, 1151, 464, 178, 103, 1656, - /* 740 */ 523, 1421, 1606, 275, 1652, 132, 570, 1200, 1306, 471, - /* 750 */ 1305, 1101, 1102, 198, 1663, 446, 1304, 445, 1114, 470, - /* 760 */ 1718, 121, 538, 1562, 1684, 1562, 316, 1205, 1303, 488, - /* 770 */ 1302, 1562, 1244, 137, 1291, 1675, 1232, 1715, 1138, 1470, - /* 780 */ 458, 1125, 75, 1591, 473, 1593, 1594, 469, 1187, 464, - /* 790 */ 1469, 454, 1656, 1562, 1138, 1562, 1655, 1652, 1116, 9, - /* 800 */ 8, 1562, 25, 280, 1194, 1195, 1196, 1197, 1198, 1202, - /* 810 */ 1203, 1204, 465, 1562, 1551, 1562, 1590, 40, 252, 1119, - /* 820 */ 1120, 1130, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 466, - /* 830 */ 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1396, 123, 1235, - /* 840 */ 1606, 166, 1133, 234, 164, 1338, 168, 471, 1590, 167, - /* 850 */ 305, 170, 1333, 1115, 169, 232, 172, 470, 113, 171, - /* 860 */ 47, 1562, 426, 512, 1258, 1294, 1295, 403, 149, 1113, - /* 870 */ 1331, 189, 1606, 35, 405, 191, 461, 1207, 1312, 471, - /* 880 */ 74, 1591, 473, 1593, 1594, 469, 423, 464, 1121, 470, - /* 890 */ 1656, 1379, 408, 1562, 275, 1652, 1730, 35, 107, 35, - /* 900 */ 208, 1171, 518, 1072, 210, 1713, 110, 111, 202, 435, - /* 910 */ 479, 216, 74, 1591, 473, 1593, 1594, 469, 1463, 464, - /* 920 */ 113, 1124, 1656, 1123, 976, 521, 275, 1652, 1730, 570, - /* 930 */ 195, 47, 72, 417, 383, 971, 917, 1674, 455, 492, - /* 940 */ 448, 1114, 1686, 1004, 517, 516, 515, 111, 514, 112, - /* 950 */ 447, 1008, 113, 1015, 111, 918, 1014, 1607, 114, 2, - /* 960 */ 204, 1136, 300, 50, 49, 325, 304, 143, 1590, 259, - /* 970 */ 945, 261, 319, 224, 145, 105, 338, 1590, 1085, 1504, - /* 980 */ 345, 1116, 352, 353, 260, 354, 311, 358, 307, 303, - /* 990 */ 140, 1145, 1606, 1127, 359, 1126, 150, 1191, 360, 471, - /* 1000 */ 1144, 1606, 1119, 1120, 361, 153, 362, 1143, 471, 470, - /* 1010 */ 156, 363, 103, 1562, 1142, 53, 366, 159, 470, 451, - /* 1020 */ 385, 138, 1562, 387, 1411, 79, 269, 134, 1663, 1664, - /* 1030 */ 1121, 1668, 241, 1591, 473, 1593, 1594, 469, 1590, 464, - /* 1040 */ 163, 75, 1591, 473, 1593, 1594, 469, 1407, 464, 165, - /* 1050 */ 116, 1656, 117, 1409, 1405, 460, 1652, 1547, 1718, 573, - /* 1060 */ 118, 119, 1606, 428, 447, 418, 1141, 1687, 176, 468, - /* 1070 */ 225, 137, 419, 223, 427, 1715, 99, 179, 181, 470, - /* 1080 */ 422, 184, 562, 1562, 558, 554, 550, 222, 436, 105, - /* 1090 */ 226, 1697, 1590, 187, 477, 433, 444, 190, 1696, 194, - /* 1100 */ 274, 439, 248, 1591, 473, 1593, 1594, 469, 467, 464, - /* 1110 */ 462, 1628, 5, 432, 71, 4, 1606, 217, 1232, 104, - /* 1120 */ 1140, 130, 37, 471, 1590, 197, 103, 196, 1677, 1671, - /* 1130 */ 276, 459, 456, 470, 16, 1513, 475, 1562, 481, 476, - /* 1140 */ 1590, 135, 1663, 1664, 1512, 1668, 480, 483, 1606, 212, - /* 1150 */ 203, 1733, 1714, 284, 1637, 471, 126, 1591, 473, 1593, - /* 1160 */ 1594, 469, 214, 464, 1606, 470, 62, 482, 1422, 1562, - /* 1170 */ 227, 471, 64, 1394, 490, 229, 429, 221, 569, 182, - /* 1180 */ 43, 470, 233, 129, 271, 1562, 235, 231, 244, 1591, - /* 1190 */ 473, 1593, 1594, 469, 1556, 464, 1093, 236, 177, 1590, - /* 1200 */ 452, 1731, 1555, 299, 75, 1591, 473, 1593, 1594, 469, - /* 1210 */ 301, 464, 1590, 1552, 1656, 302, 1109, 1110, 141, 1653, - /* 1220 */ 306, 1550, 308, 1606, 309, 310, 443, 1549, 312, 1548, - /* 1230 */ 471, 314, 1533, 142, 317, 318, 1606, 1088, 1087, 1527, - /* 1240 */ 470, 1526, 323, 471, 1562, 324, 1525, 1524, 1055, 1497, - /* 1250 */ 1496, 1495, 1494, 470, 1493, 1492, 1491, 1562, 109, 1480, - /* 1260 */ 279, 1490, 1489, 126, 1591, 473, 1593, 1594, 469, 1590, - /* 1270 */ 464, 155, 1400, 910, 1349, 1057, 249, 1591, 473, 1593, - /* 1280 */ 1594, 469, 1488, 464, 1487, 1486, 1485, 1484, 1483, 1482, - /* 1290 */ 1481, 1479, 1478, 1606, 1477, 1476, 1475, 1474, 1473, 1472, - /* 1300 */ 468, 1471, 1350, 1541, 1535, 1519, 1510, 1347, 1732, 369, - /* 1310 */ 470, 367, 368, 1345, 1562, 371, 372, 1590, 373, 1343, - /* 1320 */ 1341, 375, 377, 1330, 381, 1329, 1326, 1402, 77, 376, - /* 1330 */ 162, 1401, 536, 248, 1591, 473, 1593, 1594, 469, 538, - /* 1340 */ 464, 1606, 1629, 1020, 380, 1339, 1590, 379, 471, 1023, - /* 1350 */ 944, 266, 943, 942, 941, 940, 937, 1334, 470, 267, - /* 1360 */ 936, 1332, 1562, 406, 268, 281, 409, 1325, 411, 1324, - /* 1370 */ 1606, 413, 76, 1540, 46, 1534, 1095, 471, 120, 180, - /* 1380 */ 420, 249, 1591, 473, 1593, 1594, 469, 470, 464, 1518, - /* 1390 */ 1517, 1562, 161, 183, 283, 131, 1590, 421, 1509, 185, - /* 1400 */ 13, 382, 3, 378, 374, 370, 160, 57, 35, 188, - /* 1410 */ 249, 1591, 473, 1593, 1594, 469, 41, 464, 38, 14, - /* 1420 */ 1606, 11, 1581, 1257, 1250, 1280, 127, 471, 192, 20, - /* 1430 */ 58, 8, 193, 55, 21, 1590, 158, 470, 39, 1229, - /* 1440 */ 1228, 1562, 1279, 199, 1590, 277, 1285, 1284, 15, 1283, - /* 1450 */ 278, 17, 136, 1174, 1173, 463, 27, 1159, 139, 1606, - /* 1460 */ 237, 1591, 473, 1593, 1594, 469, 471, 464, 1606, 1508, - /* 1470 */ 1172, 1192, 10, 206, 18, 471, 470, 19, 478, 207, - /* 1480 */ 1562, 1255, 213, 472, 1131, 470, 65, 1580, 209, 1562, - /* 1490 */ 211, 60, 1590, 61, 157, 218, 152, 42, 154, 243, - /* 1500 */ 1591, 473, 1593, 1594, 469, 1590, 464, 215, 245, 1591, - /* 1510 */ 473, 1593, 1594, 469, 489, 464, 1606, 151, 491, 1005, - /* 1520 */ 291, 493, 495, 471, 1002, 496, 498, 999, 499, 1606, - /* 1530 */ 993, 501, 502, 470, 991, 504, 471, 1562, 505, 982, - /* 1540 */ 997, 996, 66, 511, 995, 994, 470, 1017, 1016, 67, - /* 1550 */ 1562, 68, 1013, 1010, 908, 522, 238, 1591, 473, 1593, - /* 1560 */ 1594, 469, 933, 464, 951, 525, 220, 931, 1590, 246, - /* 1570 */ 1591, 473, 1593, 1594, 469, 930, 464, 1590, 929, 928, - /* 1580 */ 927, 926, 925, 924, 948, 946, 921, 920, 919, 1346, - /* 1590 */ 916, 915, 1606, 914, 913, 547, 548, 549, 1344, 471, - /* 1600 */ 551, 1606, 552, 1342, 556, 555, 553, 557, 471, 470, - /* 1610 */ 1340, 559, 560, 1562, 561, 1328, 1590, 563, 470, 564, - /* 1620 */ 1327, 1323, 1562, 567, 1117, 568, 571, 1590, 230, 572, - /* 1630 */ 1298, 1298, 239, 1591, 473, 1593, 1594, 469, 1298, 464, - /* 1640 */ 1606, 247, 1591, 473, 1593, 1594, 469, 471, 464, 1298, - /* 1650 */ 1298, 1606, 1298, 1298, 1298, 1298, 1298, 470, 471, 1298, - /* 1660 */ 1298, 1562, 1298, 1298, 1298, 1298, 1298, 1298, 470, 1298, - /* 1670 */ 1298, 1298, 1562, 1298, 1298, 1298, 1590, 1298, 1298, 1298, - /* 1680 */ 240, 1591, 473, 1593, 1594, 469, 1298, 464, 1298, 1298, - /* 1690 */ 1298, 1602, 1591, 473, 1593, 1594, 469, 1298, 464, 1298, - /* 1700 */ 1606, 1298, 1298, 1298, 1298, 1298, 1298, 471, 1298, 1298, - /* 1710 */ 1298, 1298, 1298, 1298, 1298, 1590, 1298, 470, 1298, 1298, - /* 1720 */ 1298, 1562, 1298, 1298, 1590, 1298, 1298, 1298, 1298, 1298, - /* 1730 */ 1298, 1298, 1298, 1590, 1298, 1298, 1298, 1298, 1298, 1606, - /* 1740 */ 1601, 1591, 473, 1593, 1594, 469, 471, 464, 1606, 1298, - /* 1750 */ 1298, 1298, 1298, 1298, 1298, 471, 470, 1606, 1298, 1298, - /* 1760 */ 1562, 1298, 1298, 1298, 471, 470, 1298, 1298, 1298, 1562, - /* 1770 */ 1298, 1298, 1590, 1298, 470, 1298, 1298, 1298, 1562, 1600, - /* 1780 */ 1591, 473, 1593, 1594, 469, 1590, 464, 1298, 257, 1591, - /* 1790 */ 473, 1593, 1594, 469, 1298, 464, 1606, 256, 1591, 473, - /* 1800 */ 1593, 1594, 469, 471, 464, 1298, 1298, 1298, 1298, 1606, - /* 1810 */ 1298, 1298, 1298, 470, 1298, 1298, 471, 1562, 1298, 1298, - /* 1820 */ 1298, 1298, 1298, 1298, 1298, 1298, 470, 1298, 1298, 1298, - /* 1830 */ 1562, 1298, 1298, 1298, 1590, 1298, 258, 1591, 473, 1593, - /* 1840 */ 1594, 469, 1298, 464, 1298, 1298, 1298, 1298, 1298, 255, - /* 1850 */ 1591, 473, 1593, 1594, 469, 1298, 464, 1298, 1606, 1298, - /* 1860 */ 1298, 1298, 1298, 1298, 1298, 471, 1298, 1298, 1298, 1298, - /* 1870 */ 1298, 1298, 1298, 1298, 1298, 470, 1298, 1298, 1298, 1562, - /* 1880 */ 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, - /* 1890 */ 1298, 1298, 1298, 1298, 1298, 1298, 1298, 1298, 242, 1591, - /* 1900 */ 473, 1593, 1594, 469, 1298, 464, + /* 0 */ 273, 124, 485, 1316, 1596, 485, 485, 322, 128, 1581, + /* 10 */ 472, 78, 33, 31, 78, 326, 1520, 1581, 386, 1467, + /* 20 */ 282, 393, 1117, 447, 1577, 1585, 1583, 1426, 1612, 484, + /* 30 */ 1426, 1426, 1577, 1584, 1583, 469, 488, 485, 1115, 34, + /* 40 */ 32, 30, 29, 28, 488, 468, 327, 1726, 105, 1567, + /* 50 */ 12, 33, 31, 1242, 484, 451, 484, 1123, 59, 282, + /* 60 */ 137, 1117, 1426, 125, 1723, 1726, 1624, 1383, 1726, 73, + /* 70 */ 1597, 471, 1599, 1600, 467, 1, 488, 1115, 1725, 1664, + /* 80 */ 22, 1724, 1723, 253, 1660, 1723, 103, 328, 485, 12, + /* 90 */ 34, 32, 30, 29, 28, 1726, 1123, 336, 571, 1353, + /* 100 */ 449, 133, 1671, 1672, 365, 1676, 26, 205, 137, 36, + /* 110 */ 1116, 36, 1723, 1426, 1, 1305, 454, 251, 985, 511, + /* 120 */ 510, 509, 989, 508, 991, 992, 507, 994, 504, 947, + /* 130 */ 1000, 501, 1002, 1003, 498, 495, 89, 571, 1141, 88, + /* 140 */ 87, 86, 85, 84, 83, 82, 81, 80, 949, 1116, + /* 150 */ 1118, 547, 546, 545, 544, 297, 441, 543, 542, 541, + /* 160 */ 108, 536, 535, 534, 533, 532, 531, 530, 529, 115, + /* 170 */ 525, 1121, 1122, 1139, 1167, 1168, 1169, 1170, 1171, 1172, + /* 180 */ 1173, 464, 486, 1181, 1182, 1183, 1184, 1185, 1186, 1118, + /* 190 */ 30, 29, 28, 402, 416, 396, 69, 1475, 321, 401, + /* 200 */ 320, 138, 102, 272, 397, 395, 65, 398, 1473, 54, + /* 210 */ 1121, 1122, 394, 1167, 1168, 1169, 1170, 1171, 1172, 1173, + /* 220 */ 464, 486, 1181, 1182, 1183, 1184, 1185, 1186, 33, 31, + /* 230 */ 1422, 447, 63, 1596, 1726, 138, 282, 250, 1117, 1139, + /* 240 */ 34, 32, 30, 29, 28, 447, 344, 137, 54, 356, + /* 250 */ 138, 1723, 1079, 1419, 1115, 138, 105, 1612, 357, 472, + /* 260 */ 1081, 100, 285, 455, 466, 1519, 12, 33, 31, 1421, + /* 270 */ 105, 485, 485, 1123, 468, 282, 451, 1117, 1567, 89, + /* 280 */ 337, 364, 88, 87, 86, 85, 84, 83, 82, 81, + /* 290 */ 80, 1, 1142, 1115, 103, 1624, 1426, 1426, 248, 1597, + /* 300 */ 471, 1599, 1600, 467, 465, 488, 462, 1636, 103, 198, + /* 310 */ 1671, 446, 1123, 445, 571, 1356, 1726, 1140, 56, 270, + /* 320 */ 1080, 1612, 175, 134, 1671, 1672, 1116, 1676, 469, 137, + /* 330 */ 7, 1304, 355, 1723, 138, 350, 349, 348, 347, 346, + /* 340 */ 1404, 343, 342, 341, 340, 339, 335, 334, 333, 332, + /* 350 */ 331, 330, 329, 571, 138, 98, 97, 96, 95, 94, + /* 360 */ 93, 92, 91, 90, 440, 1116, 1118, 402, 1507, 396, + /* 370 */ 1143, 1297, 524, 401, 1475, 146, 102, 514, 397, 395, + /* 380 */ 287, 398, 1266, 567, 566, 1473, 394, 1121, 1122, 365, + /* 390 */ 1167, 1168, 1169, 1170, 1171, 1172, 1173, 464, 486, 1181, + /* 400 */ 1182, 1183, 1184, 1185, 1186, 1118, 34, 32, 30, 29, + /* 410 */ 28, 293, 286, 34, 32, 30, 29, 28, 1510, 1512, + /* 420 */ 122, 434, 1264, 1265, 1267, 1268, 1121, 1122, 1428, 1167, + /* 430 */ 1168, 1169, 1170, 1171, 1172, 1173, 464, 486, 1181, 1182, + /* 440 */ 1183, 1184, 1185, 1186, 33, 31, 1187, 1475, 391, 390, + /* 450 */ 1296, 254, 282, 294, 1117, 485, 138, 1596, 1473, 48, + /* 460 */ 161, 70, 290, 131, 1423, 527, 894, 485, 893, 382, + /* 470 */ 1115, 378, 374, 370, 160, 106, 1548, 1154, 485, 1581, + /* 480 */ 1426, 1612, 1418, 33, 31, 1204, 895, 482, 450, 1123, + /* 490 */ 200, 282, 1426, 1117, 1577, 1584, 1583, 122, 468, 1417, + /* 500 */ 351, 55, 1567, 1426, 158, 1429, 488, 7, 485, 1115, + /* 510 */ 1415, 34, 32, 30, 29, 28, 101, 483, 437, 1624, + /* 520 */ 389, 1206, 74, 1597, 471, 1599, 1600, 467, 1123, 488, + /* 530 */ 571, 1402, 1664, 1426, 121, 1205, 275, 1660, 132, 424, + /* 540 */ 485, 1211, 1116, 392, 1567, 292, 7, 148, 147, 219, + /* 550 */ 201, 295, 1411, 122, 485, 1210, 430, 1691, 1144, 122, + /* 560 */ 24, 1428, 157, 296, 152, 1426, 154, 1428, 1302, 571, + /* 570 */ 34, 32, 30, 29, 28, 1256, 23, 1475, 1327, 1426, + /* 580 */ 524, 1116, 1118, 1413, 425, 151, 442, 438, 1474, 893, + /* 590 */ 25, 280, 1199, 1200, 1201, 1202, 1203, 1207, 1208, 1209, + /* 600 */ 1409, 9, 8, 1121, 1122, 384, 1167, 1168, 1169, 1170, + /* 610 */ 1171, 1172, 1173, 464, 486, 1181, 1182, 1183, 1184, 1185, + /* 620 */ 1186, 1118, 298, 1567, 1726, 34, 32, 30, 29, 28, + /* 630 */ 34, 32, 30, 29, 28, 6, 1326, 137, 1325, 447, + /* 640 */ 1324, 1723, 1121, 1122, 144, 1167, 1168, 1169, 1170, 1171, + /* 650 */ 1172, 1173, 464, 486, 1181, 1182, 1183, 1184, 1185, 1186, + /* 660 */ 33, 31, 1726, 254, 105, 400, 399, 1678, 282, 1596, + /* 670 */ 1117, 52, 1678, 453, 51, 137, 1218, 1323, 186, 1723, + /* 680 */ 1678, 1567, 1322, 1567, 178, 1567, 1115, 1321, 1320, 521, + /* 690 */ 520, 1675, 1241, 1612, 1154, 1319, 1674, 1204, 1318, 539, + /* 700 */ 450, 463, 103, 316, 1673, 1123, 1315, 1314, 1313, 1312, + /* 710 */ 468, 528, 1311, 1398, 1567, 1310, 1309, 135, 1671, 1672, + /* 720 */ 1308, 1676, 1567, 1, 540, 538, 1307, 1567, 457, 264, + /* 730 */ 1403, 1624, 1567, 1567, 74, 1597, 471, 1599, 1600, 467, + /* 740 */ 1567, 488, 228, 1567, 1664, 1456, 571, 1205, 275, 1660, + /* 750 */ 132, 1567, 1567, 1567, 1567, 101, 1401, 1567, 1116, 389, + /* 760 */ 1567, 1567, 1511, 1512, 1249, 1567, 313, 1210, 513, 1692, + /* 770 */ 1141, 1567, 265, 461, 263, 262, 1556, 388, 1683, 1237, + /* 780 */ 1192, 919, 392, 423, 166, 315, 1141, 164, 414, 1317, + /* 790 */ 168, 107, 1237, 167, 170, 519, 172, 169, 1118, 171, + /* 800 */ 920, 412, 25, 280, 1199, 1200, 1201, 1202, 1203, 1207, + /* 810 */ 1208, 1209, 305, 1103, 1104, 1384, 1596, 107, 522, 1121, + /* 820 */ 1122, 519, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 464, + /* 830 */ 486, 1181, 1182, 1183, 1184, 1185, 1186, 518, 517, 516, + /* 840 */ 1612, 515, 113, 202, 522, 1468, 426, 469, 1596, 1343, + /* 850 */ 43, 252, 42, 9, 8, 189, 1263, 468, 35, 191, + /* 860 */ 435, 1567, 1212, 518, 517, 516, 35, 515, 1299, 1300, + /* 870 */ 1174, 403, 1612, 417, 195, 383, 1694, 458, 1624, 469, + /* 880 */ 1596, 74, 1597, 471, 1599, 1600, 467, 1338, 488, 468, + /* 890 */ 1240, 1664, 448, 1567, 1196, 275, 1660, 1738, 35, 208, + /* 900 */ 1589, 110, 1074, 210, 1612, 477, 1698, 1613, 1139, 405, + /* 910 */ 1624, 469, 1587, 74, 1597, 471, 1599, 1600, 467, 1336, + /* 920 */ 488, 468, 111, 1664, 1126, 1567, 216, 275, 1660, 1738, + /* 930 */ 204, 113, 1596, 2, 123, 978, 1125, 300, 1721, 234, + /* 940 */ 42, 408, 1624, 407, 973, 74, 1597, 471, 1599, 1600, + /* 950 */ 467, 232, 488, 304, 259, 1664, 1612, 261, 415, 275, + /* 960 */ 1660, 1738, 493, 469, 149, 947, 1006, 111, 1087, 224, + /* 970 */ 1682, 1010, 174, 468, 112, 410, 338, 1567, 1017, 1117, + /* 980 */ 404, 1596, 1509, 451, 113, 111, 173, 145, 1016, 114, + /* 990 */ 353, 345, 352, 354, 1624, 1115, 1129, 241, 1597, 471, + /* 1000 */ 1599, 1600, 467, 358, 488, 1612, 1148, 150, 1128, 359, + /* 1010 */ 360, 1147, 469, 46, 1123, 361, 45, 153, 362, 1146, + /* 1020 */ 363, 156, 468, 1726, 53, 366, 1567, 1145, 72, 159, + /* 1030 */ 387, 385, 1416, 1596, 1123, 163, 137, 79, 269, 1412, + /* 1040 */ 1723, 1552, 165, 1624, 225, 176, 126, 1597, 471, 1599, + /* 1050 */ 1600, 467, 179, 488, 116, 571, 117, 1612, 1414, 50, + /* 1060 */ 49, 325, 1410, 143, 469, 1596, 118, 1116, 319, 119, + /* 1070 */ 427, 418, 226, 181, 468, 184, 422, 419, 1567, 428, + /* 1080 */ 260, 1144, 311, 436, 307, 303, 140, 1705, 1695, 1612, + /* 1090 */ 452, 1739, 475, 1704, 187, 1624, 469, 433, 75, 1597, + /* 1100 */ 471, 1599, 1600, 467, 190, 488, 468, 1118, 1664, 274, + /* 1110 */ 1567, 439, 1663, 1660, 444, 1596, 5, 138, 1237, 432, + /* 1120 */ 104, 130, 1143, 1685, 194, 4, 37, 1624, 1121, 1122, + /* 1130 */ 75, 1597, 471, 1599, 1600, 467, 1679, 488, 196, 1612, + /* 1140 */ 1664, 456, 16, 276, 460, 1660, 469, 574, 459, 197, + /* 1150 */ 1722, 1518, 203, 1741, 1645, 1596, 468, 473, 474, 284, + /* 1160 */ 1567, 223, 1517, 478, 99, 214, 479, 480, 212, 62, + /* 1170 */ 563, 1427, 559, 555, 551, 222, 64, 1624, 227, 1612, + /* 1180 */ 75, 1597, 471, 1599, 1600, 467, 469, 488, 1399, 229, + /* 1190 */ 1664, 221, 491, 271, 570, 1661, 468, 44, 129, 235, + /* 1200 */ 1567, 231, 71, 431, 233, 217, 236, 1561, 1596, 1560, + /* 1210 */ 299, 1557, 301, 302, 1111, 1112, 141, 1624, 306, 1555, + /* 1220 */ 249, 1597, 471, 1599, 1600, 467, 308, 488, 309, 310, + /* 1230 */ 1554, 312, 1612, 1553, 314, 481, 1538, 142, 317, 469, + /* 1240 */ 318, 1090, 1089, 1532, 1531, 323, 324, 1530, 1529, 468, + /* 1250 */ 1057, 1502, 1501, 1567, 1500, 1499, 1498, 1596, 109, 1485, + /* 1260 */ 1484, 1483, 1482, 1481, 429, 1497, 1496, 182, 1596, 1495, + /* 1270 */ 1624, 1494, 1493, 244, 1597, 471, 1599, 1600, 467, 1492, + /* 1280 */ 488, 1612, 1491, 1490, 1095, 1489, 177, 1488, 469, 1487, + /* 1290 */ 1486, 1480, 1612, 1479, 1478, 1059, 1477, 1476, 468, 469, + /* 1300 */ 1355, 1546, 1567, 1540, 1524, 1515, 1405, 155, 912, 468, + /* 1310 */ 1354, 443, 1352, 1567, 367, 1596, 279, 368, 369, 1624, + /* 1320 */ 1350, 373, 126, 1597, 471, 1599, 1600, 467, 372, 488, + /* 1330 */ 1624, 1596, 162, 249, 1597, 471, 1599, 1600, 467, 1612, + /* 1340 */ 488, 1348, 377, 1346, 376, 371, 466, 1596, 1335, 375, + /* 1350 */ 1334, 379, 380, 1331, 1407, 1612, 468, 1025, 381, 1022, + /* 1360 */ 1567, 77, 469, 1406, 946, 945, 944, 1740, 943, 942, + /* 1370 */ 537, 1612, 468, 939, 539, 1344, 1567, 1624, 469, 281, + /* 1380 */ 248, 1597, 471, 1599, 1600, 467, 266, 488, 468, 1637, + /* 1390 */ 938, 1339, 1567, 1624, 267, 283, 249, 1597, 471, 1599, + /* 1400 */ 1600, 467, 1337, 488, 406, 1596, 268, 409, 1330, 1624, + /* 1410 */ 411, 1329, 249, 1597, 471, 1599, 1600, 467, 1596, 488, + /* 1420 */ 413, 76, 1545, 1097, 47, 1539, 420, 1523, 421, 1612, + /* 1430 */ 1522, 1514, 180, 57, 3, 120, 469, 1596, 183, 35, + /* 1440 */ 13, 127, 1612, 192, 185, 40, 468, 188, 1587, 469, + /* 1450 */ 1567, 14, 1262, 20, 193, 1255, 58, 199, 21, 468, + /* 1460 */ 1162, 1612, 1285, 1567, 39, 1234, 1233, 1624, 469, 38, + /* 1470 */ 237, 1597, 471, 1599, 1600, 467, 1290, 488, 468, 15, + /* 1480 */ 1624, 136, 1567, 243, 1597, 471, 1599, 1600, 467, 1284, + /* 1490 */ 488, 11, 1596, 277, 1289, 1288, 278, 8, 17, 1624, + /* 1500 */ 1197, 139, 245, 1597, 471, 1599, 1600, 467, 1596, 488, + /* 1510 */ 1176, 27, 470, 1175, 10, 18, 1612, 19, 476, 206, + /* 1520 */ 207, 1260, 209, 469, 211, 60, 1513, 213, 215, 61, + /* 1530 */ 1133, 65, 1612, 468, 1627, 1586, 1178, 1567, 487, 469, + /* 1540 */ 218, 41, 492, 1007, 291, 494, 497, 490, 1596, 468, + /* 1550 */ 1004, 496, 1001, 1567, 1624, 499, 500, 238, 1597, 471, + /* 1560 */ 1599, 1600, 467, 995, 488, 502, 1596, 503, 505, 993, + /* 1570 */ 1624, 506, 1612, 246, 1597, 471, 1599, 1600, 467, 469, + /* 1580 */ 488, 1596, 999, 998, 984, 997, 996, 512, 1019, 468, + /* 1590 */ 1612, 66, 1015, 1567, 67, 68, 1012, 469, 1018, 910, + /* 1600 */ 953, 523, 935, 933, 526, 1612, 220, 468, 932, 931, + /* 1610 */ 1624, 1567, 469, 239, 1597, 471, 1599, 1600, 467, 930, + /* 1620 */ 488, 929, 468, 928, 927, 926, 1567, 948, 1624, 950, + /* 1630 */ 1596, 247, 1597, 471, 1599, 1600, 467, 923, 488, 922, + /* 1640 */ 921, 1351, 918, 1624, 917, 916, 240, 1597, 471, 1599, + /* 1650 */ 1600, 467, 915, 488, 1612, 548, 549, 1349, 550, 552, + /* 1660 */ 553, 469, 554, 1347, 556, 557, 558, 1345, 560, 561, + /* 1670 */ 562, 468, 1333, 564, 565, 1567, 1332, 1596, 1328, 573, + /* 1680 */ 568, 569, 1303, 1119, 230, 572, 1303, 1303, 1303, 1303, + /* 1690 */ 1303, 1303, 1624, 1303, 1303, 1608, 1597, 471, 1599, 1600, + /* 1700 */ 467, 1612, 488, 1303, 1303, 1303, 1303, 1596, 469, 1303, + /* 1710 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 468, 1303, + /* 1720 */ 1303, 1303, 1567, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1730 */ 1303, 1612, 1303, 1303, 1303, 1303, 1303, 1596, 469, 1624, + /* 1740 */ 1303, 1303, 1607, 1597, 471, 1599, 1600, 467, 468, 488, + /* 1750 */ 1303, 1303, 1567, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1760 */ 1303, 1612, 1303, 1303, 1303, 1303, 1303, 1596, 469, 1624, + /* 1770 */ 1303, 1303, 1606, 1597, 471, 1599, 1600, 467, 468, 488, + /* 1780 */ 1303, 1303, 1567, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1790 */ 1303, 1612, 1303, 1303, 1303, 1303, 1303, 1596, 469, 1624, + /* 1800 */ 1303, 1303, 257, 1597, 471, 1599, 1600, 467, 468, 488, + /* 1810 */ 1303, 1303, 1567, 1303, 1303, 1303, 1596, 1303, 1303, 1303, + /* 1820 */ 1303, 1612, 1303, 1303, 1303, 1303, 1303, 1303, 469, 1624, + /* 1830 */ 1303, 1303, 256, 1597, 471, 1599, 1600, 467, 468, 488, + /* 1840 */ 1612, 1303, 1567, 1303, 1303, 1303, 1596, 469, 289, 288, + /* 1850 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 468, 1131, 1624, + /* 1860 */ 1303, 1567, 258, 1597, 471, 1599, 1600, 467, 1303, 488, + /* 1870 */ 1612, 1303, 1303, 1303, 1124, 1303, 1303, 469, 1624, 1303, + /* 1880 */ 1303, 255, 1597, 471, 1599, 1600, 467, 468, 488, 1303, + /* 1890 */ 1303, 1567, 1303, 1123, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1900 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1624, 1303, + /* 1910 */ 1303, 242, 1597, 471, 1599, 1600, 467, 1303, 488, 1303, + /* 1920 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1930 */ 1303, 1303, 1303, 1303, 489, 1303, 1303, 1303, 1303, 1303, + /* 1940 */ 1303, 1303, 1303, 1303, 1303, 1303, 1127, 1303, 1303, 1303, + /* 1950 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1960 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1970 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 1980 */ 1303, 1303, 1303, 1303, 1303, 1303, 1132, 1303, 1303, 1303, + /* 1990 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, + /* 2000 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1135, 1303, 1303, + /* 2010 */ 1303, 1303, 1303, 1303, 1303, 1303, 1303, 1303, 486, 1181, + /* 2020 */ 1182, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 240, 324, 234, 265, 262, 275, 285, 262, 278, 249, - /* 10 */ 272, 281, 12, 13, 337, 259, 256, 4, 341, 240, - /* 20 */ 20, 279, 22, 258, 279, 265, 258, 243, 249, 264, - /* 30 */ 240, 247, 19, 265, 269, 256, 294, 295, 38, 294, - /* 40 */ 295, 309, 310, 275, 265, 324, 33, 279, 279, 36, - /* 50 */ 50, 12, 13, 14, 41, 265, 20, 57, 337, 20, - /* 60 */ 47, 22, 341, 294, 295, 242, 298, 299, 300, 301, - /* 70 */ 302, 303, 20, 305, 4, 75, 308, 38, 255, 240, - /* 80 */ 312, 313, 314, 14, 15, 16, 263, 74, 249, 50, - /* 90 */ 77, 242, 302, 20, 326, 267, 57, 250, 98, 0, - /* 100 */ 332, 333, 274, 275, 265, 258, 316, 317, 318, 319, - /* 110 */ 110, 321, 263, 266, 75, 0, 89, 90, 91, 92, - /* 120 */ 93, 94, 95, 96, 97, 98, 99, 75, 101, 102, - /* 130 */ 103, 104, 105, 106, 1, 2, 21, 98, 20, 24, - /* 140 */ 25, 26, 27, 28, 29, 30, 31, 32, 75, 110, - /* 150 */ 150, 52, 53, 54, 55, 56, 49, 58, 59, 60, + /* 0 */ 262, 233, 240, 235, 234, 240, 240, 285, 257, 279, + /* 10 */ 275, 249, 12, 13, 249, 249, 281, 279, 256, 268, + /* 20 */ 20, 256, 22, 240, 294, 295, 296, 265, 258, 20, + /* 30 */ 265, 265, 294, 295, 296, 265, 306, 240, 38, 12, + /* 40 */ 13, 14, 15, 16, 306, 275, 249, 325, 265, 279, + /* 50 */ 50, 12, 13, 14, 20, 285, 20, 57, 4, 20, + /* 60 */ 338, 22, 265, 243, 342, 325, 296, 247, 325, 299, + /* 70 */ 300, 301, 302, 303, 304, 75, 306, 38, 338, 309, + /* 80 */ 2, 338, 342, 313, 314, 342, 303, 240, 240, 50, + /* 90 */ 12, 13, 14, 15, 16, 325, 57, 249, 98, 0, + /* 100 */ 317, 318, 319, 320, 49, 322, 310, 311, 338, 75, + /* 110 */ 110, 75, 342, 265, 75, 0, 72, 270, 89, 90, + /* 120 */ 91, 92, 93, 94, 95, 96, 97, 98, 99, 38, + /* 130 */ 101, 102, 103, 104, 105, 106, 21, 98, 20, 24, + /* 140 */ 25, 26, 27, 28, 29, 30, 31, 32, 57, 110, + /* 150 */ 150, 52, 53, 54, 55, 56, 20, 58, 59, 60, /* 160 */ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, - /* 170 */ 71, 171, 172, 239, 174, 175, 176, 177, 178, 179, + /* 170 */ 71, 171, 172, 20, 174, 175, 176, 177, 178, 179, /* 180 */ 180, 181, 182, 183, 184, 185, 186, 187, 188, 150, - /* 190 */ 22, 258, 250, 52, 260, 54, 285, 231, 265, 58, - /* 200 */ 258, 201, 61, 20, 63, 64, 38, 66, 266, 76, + /* 190 */ 14, 15, 16, 52, 285, 54, 75, 258, 149, 58, + /* 200 */ 151, 201, 61, 264, 63, 64, 85, 66, 269, 242, /* 210 */ 171, 172, 71, 174, 175, 176, 177, 178, 179, 180, /* 220 */ 181, 182, 183, 184, 185, 186, 187, 188, 12, 13, - /* 230 */ 61, 234, 21, 300, 65, 324, 20, 18, 22, 20, - /* 240 */ 12, 13, 14, 15, 16, 34, 27, 324, 337, 30, - /* 250 */ 296, 285, 341, 201, 38, 258, 0, 88, 39, 258, - /* 260 */ 337, 149, 265, 151, 341, 264, 50, 12, 13, 240, - /* 270 */ 269, 201, 275, 57, 320, 20, 279, 22, 249, 201, - /* 280 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 234, - /* 290 */ 324, 75, 20, 38, 265, 298, 299, 300, 301, 302, - /* 300 */ 303, 240, 305, 337, 76, 308, 257, 341, 75, 312, - /* 310 */ 313, 314, 57, 201, 98, 171, 258, 268, 85, 201, - /* 320 */ 323, 74, 264, 140, 159, 160, 110, 269, 163, 82, - /* 330 */ 75, 270, 113, 35, 279, 116, 117, 118, 119, 120, - /* 340 */ 43, 122, 123, 124, 125, 126, 127, 128, 129, 130, - /* 350 */ 131, 132, 133, 98, 210, 211, 212, 213, 214, 61, - /* 360 */ 20, 20, 22, 65, 21, 110, 150, 24, 25, 26, - /* 370 */ 27, 28, 29, 30, 31, 32, 78, 234, 80, 81, - /* 380 */ 40, 83, 138, 233, 3, 235, 88, 171, 172, 142, + /* 230 */ 263, 240, 239, 234, 325, 201, 20, 18, 22, 20, + /* 240 */ 12, 13, 14, 15, 16, 240, 27, 338, 242, 30, + /* 250 */ 201, 342, 74, 260, 38, 201, 265, 258, 39, 275, + /* 260 */ 82, 255, 278, 219, 265, 281, 50, 12, 13, 263, + /* 270 */ 265, 240, 240, 57, 275, 20, 285, 22, 279, 21, + /* 280 */ 249, 249, 24, 25, 26, 27, 28, 29, 30, 31, + /* 290 */ 32, 75, 20, 38, 303, 296, 265, 265, 299, 300, + /* 300 */ 301, 302, 303, 304, 305, 306, 307, 308, 303, 318, + /* 310 */ 319, 320, 57, 322, 98, 0, 325, 20, 159, 160, + /* 320 */ 142, 258, 163, 318, 319, 320, 110, 322, 265, 338, + /* 330 */ 75, 0, 113, 342, 201, 116, 117, 118, 119, 120, + /* 340 */ 0, 122, 123, 124, 125, 126, 127, 128, 129, 130, + /* 350 */ 131, 132, 133, 98, 201, 24, 25, 26, 27, 28, + /* 360 */ 29, 30, 31, 32, 301, 110, 150, 52, 265, 54, + /* 370 */ 20, 143, 49, 58, 258, 272, 61, 86, 63, 64, + /* 380 */ 264, 66, 171, 237, 238, 269, 71, 171, 172, 49, /* 390 */ 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, - /* 400 */ 184, 185, 186, 187, 188, 150, 0, 2, 250, 12, - /* 410 */ 13, 14, 15, 16, 0, 49, 258, 12, 13, 14, - /* 420 */ 15, 16, 279, 20, 266, 251, 171, 172, 254, 174, + /* 400 */ 184, 185, 186, 187, 188, 150, 12, 13, 14, 15, + /* 410 */ 16, 267, 250, 12, 13, 14, 15, 16, 274, 275, + /* 420 */ 258, 210, 211, 212, 213, 214, 171, 172, 266, 174, /* 430 */ 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, - /* 440 */ 185, 186, 187, 188, 12, 13, 14, 50, 2, 134, - /* 450 */ 206, 207, 20, 240, 22, 234, 201, 38, 12, 13, - /* 460 */ 14, 15, 16, 237, 238, 240, 52, 61, 54, 154, - /* 470 */ 38, 65, 58, 76, 249, 61, 57, 63, 64, 258, - /* 480 */ 66, 84, 146, 12, 13, 71, 265, 244, 245, 57, - /* 490 */ 265, 20, 20, 22, 88, 234, 275, 200, 285, 275, - /* 500 */ 279, 165, 240, 234, 189, 281, 285, 75, 234, 38, - /* 510 */ 20, 249, 234, 107, 108, 109, 240, 111, 234, 298, - /* 520 */ 299, 300, 301, 302, 303, 249, 305, 265, 57, 308, - /* 530 */ 98, 134, 258, 312, 313, 240, 67, 324, 239, 265, - /* 540 */ 279, 265, 110, 140, 249, 324, 75, 258, 279, 275, - /* 550 */ 337, 154, 253, 279, 341, 266, 282, 279, 337, 260, - /* 560 */ 265, 86, 341, 279, 12, 13, 14, 15, 16, 98, - /* 570 */ 47, 240, 298, 299, 300, 301, 302, 303, 234, 305, - /* 580 */ 249, 110, 150, 114, 115, 234, 189, 190, 191, 192, - /* 590 */ 193, 194, 195, 196, 197, 198, 265, 74, 217, 234, - /* 600 */ 77, 244, 245, 171, 172, 57, 174, 175, 176, 177, + /* 440 */ 185, 186, 187, 188, 12, 13, 14, 258, 244, 245, + /* 450 */ 222, 50, 20, 264, 22, 240, 201, 234, 269, 3, + /* 460 */ 33, 239, 262, 36, 249, 57, 20, 240, 22, 42, + /* 470 */ 38, 44, 45, 46, 47, 253, 249, 76, 240, 279, + /* 480 */ 265, 258, 260, 12, 13, 84, 40, 249, 265, 57, + /* 490 */ 140, 20, 265, 22, 294, 295, 296, 258, 275, 234, + /* 500 */ 67, 74, 279, 265, 77, 266, 306, 75, 240, 38, + /* 510 */ 259, 12, 13, 14, 15, 16, 61, 249, 138, 296, + /* 520 */ 65, 134, 299, 300, 301, 302, 303, 304, 57, 306, + /* 530 */ 98, 0, 309, 265, 140, 134, 313, 314, 315, 240, + /* 540 */ 240, 154, 110, 88, 279, 250, 75, 114, 115, 249, + /* 550 */ 327, 250, 259, 258, 240, 154, 333, 334, 20, 258, + /* 560 */ 2, 266, 135, 249, 137, 265, 139, 266, 231, 98, + /* 570 */ 12, 13, 14, 15, 16, 76, 189, 258, 234, 265, + /* 580 */ 49, 110, 150, 259, 285, 158, 206, 207, 269, 22, + /* 590 */ 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + /* 600 */ 259, 1, 2, 171, 172, 38, 174, 175, 176, 177, /* 610 */ 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, - /* 620 */ 188, 150, 234, 279, 234, 12, 13, 14, 15, 16, - /* 630 */ 279, 72, 234, 12, 13, 14, 15, 16, 0, 244, - /* 640 */ 245, 4, 171, 172, 279, 174, 175, 176, 177, 178, + /* 620 */ 188, 150, 285, 279, 325, 12, 13, 14, 15, 16, + /* 630 */ 12, 13, 14, 15, 16, 43, 234, 338, 234, 240, + /* 640 */ 234, 342, 171, 172, 47, 174, 175, 176, 177, 178, /* 650 */ 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, - /* 660 */ 12, 13, 234, 50, 296, 234, 296, 279, 20, 279, - /* 670 */ 22, 12, 13, 14, 15, 16, 240, 279, 12, 13, - /* 680 */ 14, 15, 16, 12, 13, 259, 38, 49, 320, 258, - /* 690 */ 320, 0, 75, 22, 240, 143, 265, 84, 246, 240, - /* 700 */ 248, 265, 240, 249, 87, 57, 275, 279, 249, 38, - /* 710 */ 279, 249, 244, 245, 274, 275, 240, 259, 234, 265, - /* 720 */ 234, 285, 259, 75, 265, 249, 234, 265, 57, 298, - /* 730 */ 299, 300, 301, 302, 303, 76, 305, 259, 302, 308, - /* 740 */ 49, 265, 258, 312, 313, 314, 98, 134, 234, 265, - /* 750 */ 234, 161, 162, 317, 318, 319, 234, 321, 110, 275, - /* 760 */ 324, 140, 72, 279, 333, 279, 76, 154, 234, 98, - /* 770 */ 234, 279, 14, 337, 222, 199, 200, 341, 20, 258, - /* 780 */ 221, 110, 298, 299, 300, 301, 302, 303, 14, 305, - /* 790 */ 269, 72, 308, 279, 20, 279, 312, 313, 150, 1, - /* 800 */ 2, 279, 189, 190, 191, 192, 193, 194, 195, 196, - /* 810 */ 197, 198, 259, 279, 0, 279, 234, 140, 141, 171, - /* 820 */ 172, 150, 174, 175, 176, 177, 178, 179, 180, 181, - /* 830 */ 182, 183, 184, 185, 186, 187, 188, 0, 18, 202, - /* 840 */ 258, 79, 171, 23, 82, 0, 79, 265, 234, 82, - /* 850 */ 36, 79, 0, 22, 82, 35, 79, 275, 72, 82, - /* 860 */ 72, 279, 76, 259, 76, 186, 187, 22, 48, 38, - /* 870 */ 0, 72, 258, 72, 22, 76, 50, 76, 235, 265, - /* 880 */ 298, 299, 300, 301, 302, 303, 288, 305, 57, 275, - /* 890 */ 308, 247, 22, 279, 312, 313, 314, 72, 61, 72, - /* 900 */ 72, 76, 65, 76, 76, 323, 72, 72, 344, 335, - /* 910 */ 76, 76, 298, 299, 300, 301, 302, 303, 268, 305, - /* 920 */ 72, 38, 308, 38, 76, 88, 312, 313, 314, 98, - /* 930 */ 329, 72, 112, 292, 237, 76, 38, 323, 219, 72, - /* 940 */ 322, 110, 297, 76, 107, 108, 109, 72, 111, 72, - /* 950 */ 240, 76, 72, 76, 72, 57, 76, 258, 76, 325, - /* 960 */ 338, 20, 240, 143, 144, 145, 36, 147, 234, 293, - /* 970 */ 38, 244, 152, 286, 121, 265, 240, 234, 148, 240, - /* 980 */ 273, 150, 271, 134, 164, 271, 166, 240, 168, 169, - /* 990 */ 170, 20, 258, 110, 290, 110, 242, 171, 275, 265, - /* 1000 */ 20, 258, 171, 172, 283, 242, 265, 20, 265, 275, - /* 1010 */ 242, 276, 302, 279, 20, 242, 240, 242, 275, 285, - /* 1020 */ 236, 201, 279, 258, 258, 240, 236, 317, 318, 319, - /* 1030 */ 57, 321, 298, 299, 300, 301, 302, 303, 234, 305, - /* 1040 */ 258, 298, 299, 300, 301, 302, 303, 258, 305, 258, - /* 1050 */ 258, 308, 258, 258, 258, 312, 313, 279, 324, 19, - /* 1060 */ 258, 258, 258, 276, 240, 157, 20, 297, 239, 265, - /* 1070 */ 290, 337, 289, 33, 265, 341, 36, 239, 239, 275, - /* 1080 */ 275, 239, 42, 279, 44, 45, 46, 47, 209, 265, - /* 1090 */ 283, 334, 234, 280, 208, 279, 215, 280, 334, 330, - /* 1100 */ 279, 279, 298, 299, 300, 301, 302, 303, 304, 305, - /* 1110 */ 306, 307, 216, 204, 74, 203, 258, 77, 200, 265, - /* 1120 */ 20, 328, 121, 265, 234, 315, 302, 327, 331, 296, - /* 1130 */ 223, 220, 218, 275, 75, 280, 279, 279, 277, 279, - /* 1140 */ 234, 317, 318, 319, 280, 321, 137, 107, 258, 265, - /* 1150 */ 339, 345, 340, 279, 311, 265, 298, 299, 300, 301, - /* 1160 */ 302, 303, 239, 305, 258, 275, 239, 276, 265, 279, - /* 1170 */ 254, 265, 75, 248, 261, 240, 136, 239, 236, 139, - /* 1180 */ 287, 275, 232, 291, 284, 279, 252, 241, 298, 299, - /* 1190 */ 300, 301, 302, 303, 0, 305, 156, 252, 158, 234, - /* 1200 */ 342, 343, 0, 64, 298, 299, 300, 301, 302, 303, - /* 1210 */ 38, 305, 234, 0, 308, 167, 38, 38, 38, 313, - /* 1220 */ 167, 0, 38, 258, 38, 167, 336, 0, 38, 0, - /* 1230 */ 265, 38, 0, 75, 154, 153, 258, 110, 150, 0, - /* 1240 */ 275, 0, 53, 265, 279, 146, 0, 0, 87, 0, - /* 1250 */ 0, 0, 0, 275, 0, 0, 0, 279, 121, 0, - /* 1260 */ 282, 0, 0, 298, 299, 300, 301, 302, 303, 234, - /* 1270 */ 305, 43, 0, 51, 0, 22, 298, 299, 300, 301, - /* 1280 */ 302, 303, 0, 305, 0, 0, 0, 0, 0, 0, - /* 1290 */ 0, 0, 0, 258, 0, 0, 0, 0, 0, 0, - /* 1300 */ 265, 0, 0, 0, 0, 0, 0, 0, 343, 43, - /* 1310 */ 275, 38, 36, 0, 279, 38, 36, 234, 43, 0, - /* 1320 */ 0, 38, 43, 0, 43, 0, 0, 0, 84, 36, - /* 1330 */ 82, 0, 72, 298, 299, 300, 301, 302, 303, 72, - /* 1340 */ 305, 258, 307, 22, 36, 0, 234, 38, 265, 38, - /* 1350 */ 38, 22, 38, 38, 38, 38, 38, 0, 275, 22, - /* 1360 */ 38, 0, 279, 39, 22, 282, 38, 0, 22, 0, - /* 1370 */ 258, 22, 20, 0, 140, 0, 38, 265, 155, 137, - /* 1380 */ 22, 298, 299, 300, 301, 302, 303, 275, 305, 0, - /* 1390 */ 0, 279, 33, 43, 282, 36, 234, 140, 0, 135, - /* 1400 */ 205, 42, 72, 44, 45, 46, 47, 75, 72, 76, - /* 1410 */ 298, 299, 300, 301, 302, 303, 72, 305, 199, 205, - /* 1420 */ 258, 205, 87, 76, 76, 38, 75, 265, 75, 75, - /* 1430 */ 75, 2, 72, 74, 72, 234, 77, 275, 72, 76, - /* 1440 */ 76, 279, 38, 87, 234, 38, 76, 38, 72, 38, - /* 1450 */ 38, 72, 87, 76, 76, 75, 75, 22, 87, 258, - /* 1460 */ 298, 299, 300, 301, 302, 303, 265, 305, 258, 0, - /* 1470 */ 76, 171, 75, 87, 75, 265, 275, 75, 138, 76, - /* 1480 */ 279, 76, 43, 173, 22, 275, 85, 87, 75, 279, - /* 1490 */ 75, 75, 234, 75, 135, 87, 137, 75, 139, 298, - /* 1500 */ 299, 300, 301, 302, 303, 234, 305, 135, 298, 299, - /* 1510 */ 300, 301, 302, 303, 86, 305, 258, 158, 38, 76, - /* 1520 */ 38, 75, 38, 265, 76, 75, 38, 76, 75, 258, - /* 1530 */ 76, 38, 75, 275, 76, 38, 265, 279, 75, 22, - /* 1540 */ 100, 100, 75, 88, 100, 100, 275, 38, 110, 75, - /* 1550 */ 279, 75, 38, 22, 51, 50, 298, 299, 300, 301, - /* 1560 */ 302, 303, 38, 305, 57, 73, 72, 38, 234, 298, - /* 1570 */ 299, 300, 301, 302, 303, 38, 305, 234, 38, 38, - /* 1580 */ 38, 38, 38, 22, 57, 38, 38, 38, 38, 0, - /* 1590 */ 38, 38, 258, 38, 38, 38, 36, 43, 0, 265, - /* 1600 */ 38, 258, 36, 0, 36, 38, 43, 43, 265, 275, - /* 1610 */ 0, 38, 36, 279, 43, 0, 234, 38, 275, 37, - /* 1620 */ 0, 0, 279, 22, 22, 21, 21, 234, 22, 20, - /* 1630 */ 346, 346, 298, 299, 300, 301, 302, 303, 346, 305, - /* 1640 */ 258, 298, 299, 300, 301, 302, 303, 265, 305, 346, - /* 1650 */ 346, 258, 346, 346, 346, 346, 346, 275, 265, 346, - /* 1660 */ 346, 279, 346, 346, 346, 346, 346, 346, 275, 346, - /* 1670 */ 346, 346, 279, 346, 346, 346, 234, 346, 346, 346, - /* 1680 */ 298, 299, 300, 301, 302, 303, 346, 305, 346, 346, - /* 1690 */ 346, 298, 299, 300, 301, 302, 303, 346, 305, 346, - /* 1700 */ 258, 346, 346, 346, 346, 346, 346, 265, 346, 346, - /* 1710 */ 346, 346, 346, 346, 346, 234, 346, 275, 346, 346, - /* 1720 */ 346, 279, 346, 346, 234, 346, 346, 346, 346, 346, - /* 1730 */ 346, 346, 346, 234, 346, 346, 346, 346, 346, 258, - /* 1740 */ 298, 299, 300, 301, 302, 303, 265, 305, 258, 346, - /* 1750 */ 346, 346, 346, 346, 346, 265, 275, 258, 346, 346, - /* 1760 */ 279, 346, 346, 346, 265, 275, 346, 346, 346, 279, - /* 1770 */ 346, 346, 234, 346, 275, 346, 346, 346, 279, 298, - /* 1780 */ 299, 300, 301, 302, 303, 234, 305, 346, 298, 299, - /* 1790 */ 300, 301, 302, 303, 346, 305, 258, 298, 299, 300, - /* 1800 */ 301, 302, 303, 265, 305, 346, 346, 346, 346, 258, - /* 1810 */ 346, 346, 346, 275, 346, 346, 265, 279, 346, 346, - /* 1820 */ 346, 346, 346, 346, 346, 346, 275, 346, 346, 346, - /* 1830 */ 279, 346, 346, 346, 234, 346, 298, 299, 300, 301, - /* 1840 */ 302, 303, 346, 305, 346, 346, 346, 346, 346, 298, - /* 1850 */ 299, 300, 301, 302, 303, 346, 305, 346, 258, 346, - /* 1860 */ 346, 346, 346, 346, 346, 265, 346, 346, 346, 346, - /* 1870 */ 346, 346, 346, 346, 346, 275, 346, 346, 346, 279, - /* 1880 */ 346, 346, 346, 346, 346, 346, 346, 346, 346, 346, - /* 1890 */ 346, 346, 346, 346, 346, 346, 346, 346, 298, 299, - /* 1900 */ 300, 301, 302, 303, 346, 305, + /* 660 */ 12, 13, 325, 50, 265, 244, 245, 297, 20, 234, + /* 670 */ 22, 74, 297, 217, 77, 338, 76, 234, 140, 342, + /* 680 */ 297, 279, 234, 279, 259, 279, 38, 234, 234, 244, + /* 690 */ 245, 321, 4, 258, 76, 234, 321, 84, 234, 72, + /* 700 */ 265, 259, 303, 76, 321, 57, 234, 234, 234, 234, + /* 710 */ 275, 246, 234, 248, 279, 234, 234, 318, 319, 320, + /* 720 */ 234, 322, 279, 75, 244, 245, 234, 279, 72, 35, + /* 730 */ 0, 296, 279, 279, 299, 300, 301, 302, 303, 304, + /* 740 */ 279, 306, 251, 279, 309, 254, 98, 134, 313, 314, + /* 750 */ 315, 279, 279, 279, 279, 61, 0, 279, 110, 65, + /* 760 */ 279, 279, 274, 275, 14, 279, 146, 154, 259, 334, + /* 770 */ 20, 279, 78, 50, 80, 81, 0, 83, 199, 200, + /* 780 */ 14, 38, 88, 288, 79, 165, 20, 82, 21, 235, + /* 790 */ 79, 61, 200, 82, 79, 65, 79, 82, 150, 82, + /* 800 */ 57, 34, 189, 190, 191, 192, 193, 194, 195, 196, + /* 810 */ 197, 198, 36, 161, 162, 247, 234, 61, 88, 171, + /* 820 */ 172, 65, 174, 175, 176, 177, 178, 179, 180, 181, + /* 830 */ 182, 183, 184, 185, 186, 187, 188, 107, 108, 109, + /* 840 */ 258, 111, 72, 345, 88, 268, 76, 265, 234, 0, + /* 850 */ 140, 141, 72, 1, 2, 72, 76, 275, 72, 76, + /* 860 */ 336, 279, 76, 107, 108, 109, 72, 111, 186, 187, + /* 870 */ 76, 22, 258, 292, 330, 237, 298, 221, 296, 265, + /* 880 */ 234, 299, 300, 301, 302, 303, 304, 0, 306, 275, + /* 890 */ 202, 309, 323, 279, 171, 313, 314, 315, 72, 72, + /* 900 */ 75, 72, 76, 76, 258, 76, 324, 258, 20, 22, + /* 910 */ 296, 265, 87, 299, 300, 301, 302, 303, 304, 0, + /* 920 */ 306, 275, 72, 309, 38, 279, 76, 313, 314, 315, + /* 930 */ 339, 72, 234, 326, 18, 76, 38, 240, 324, 23, + /* 940 */ 72, 22, 296, 4, 76, 299, 300, 301, 302, 303, + /* 950 */ 304, 35, 306, 36, 293, 309, 258, 244, 19, 313, + /* 960 */ 314, 315, 72, 265, 48, 38, 76, 72, 148, 286, + /* 970 */ 324, 76, 33, 275, 72, 36, 240, 279, 76, 22, + /* 980 */ 41, 234, 240, 285, 72, 72, 47, 121, 76, 76, + /* 990 */ 134, 273, 271, 271, 296, 38, 110, 299, 300, 301, + /* 1000 */ 302, 303, 304, 240, 306, 258, 20, 242, 110, 290, + /* 1010 */ 275, 20, 265, 74, 57, 283, 77, 242, 265, 20, + /* 1020 */ 276, 242, 275, 325, 242, 240, 279, 20, 112, 242, + /* 1030 */ 258, 236, 258, 234, 57, 258, 338, 240, 236, 258, + /* 1040 */ 342, 279, 258, 296, 290, 239, 299, 300, 301, 302, + /* 1050 */ 303, 304, 239, 306, 258, 98, 258, 258, 258, 143, + /* 1060 */ 144, 145, 258, 147, 265, 234, 258, 110, 152, 258, + /* 1070 */ 265, 157, 283, 239, 275, 239, 275, 289, 279, 276, + /* 1080 */ 164, 20, 166, 209, 168, 169, 170, 335, 298, 258, + /* 1090 */ 343, 344, 208, 335, 280, 296, 265, 279, 299, 300, + /* 1100 */ 301, 302, 303, 304, 280, 306, 275, 150, 309, 279, + /* 1110 */ 279, 279, 313, 314, 215, 234, 216, 201, 200, 204, + /* 1120 */ 265, 329, 20, 332, 331, 203, 121, 296, 171, 172, + /* 1130 */ 299, 300, 301, 302, 303, 304, 297, 306, 328, 258, + /* 1140 */ 309, 218, 75, 223, 313, 314, 265, 19, 220, 316, + /* 1150 */ 341, 280, 340, 346, 312, 234, 275, 279, 279, 279, + /* 1160 */ 279, 33, 280, 137, 36, 239, 277, 276, 265, 239, + /* 1170 */ 42, 265, 44, 45, 46, 47, 75, 296, 254, 258, + /* 1180 */ 299, 300, 301, 302, 303, 304, 265, 306, 248, 240, + /* 1190 */ 309, 239, 261, 284, 236, 314, 275, 287, 291, 252, + /* 1200 */ 279, 241, 74, 282, 232, 77, 252, 0, 234, 0, + /* 1210 */ 64, 0, 38, 167, 38, 38, 38, 296, 167, 0, + /* 1220 */ 299, 300, 301, 302, 303, 304, 38, 306, 38, 167, + /* 1230 */ 0, 38, 258, 0, 38, 107, 0, 75, 154, 265, + /* 1240 */ 153, 110, 150, 0, 0, 53, 146, 0, 0, 275, + /* 1250 */ 87, 0, 0, 279, 0, 0, 0, 234, 121, 0, + /* 1260 */ 0, 0, 0, 0, 136, 0, 0, 139, 234, 0, + /* 1270 */ 296, 0, 0, 299, 300, 301, 302, 303, 304, 0, + /* 1280 */ 306, 258, 0, 0, 156, 0, 158, 0, 265, 0, + /* 1290 */ 0, 0, 258, 0, 0, 22, 0, 0, 275, 265, + /* 1300 */ 0, 0, 279, 0, 0, 0, 0, 43, 51, 275, + /* 1310 */ 0, 337, 0, 279, 38, 234, 282, 36, 43, 296, + /* 1320 */ 0, 43, 299, 300, 301, 302, 303, 304, 36, 306, + /* 1330 */ 296, 234, 82, 299, 300, 301, 302, 303, 304, 258, + /* 1340 */ 306, 0, 43, 0, 36, 38, 265, 234, 0, 38, + /* 1350 */ 0, 38, 36, 0, 0, 258, 275, 38, 43, 22, + /* 1360 */ 279, 84, 265, 0, 38, 38, 38, 344, 38, 38, + /* 1370 */ 72, 258, 275, 38, 72, 0, 279, 296, 265, 282, + /* 1380 */ 299, 300, 301, 302, 303, 304, 22, 306, 275, 308, + /* 1390 */ 38, 0, 279, 296, 22, 282, 299, 300, 301, 302, + /* 1400 */ 303, 304, 0, 306, 39, 234, 22, 38, 0, 296, + /* 1410 */ 22, 0, 299, 300, 301, 302, 303, 304, 234, 306, + /* 1420 */ 22, 20, 0, 38, 140, 0, 22, 0, 140, 258, + /* 1430 */ 0, 0, 137, 75, 72, 155, 265, 234, 43, 72, + /* 1440 */ 205, 75, 258, 75, 135, 72, 275, 76, 87, 265, + /* 1450 */ 279, 205, 76, 75, 72, 76, 75, 87, 72, 275, + /* 1460 */ 22, 258, 38, 279, 72, 76, 76, 296, 265, 199, + /* 1470 */ 299, 300, 301, 302, 303, 304, 76, 306, 275, 72, + /* 1480 */ 296, 87, 279, 299, 300, 301, 302, 303, 304, 38, + /* 1490 */ 306, 205, 234, 38, 38, 38, 38, 2, 72, 296, + /* 1500 */ 171, 87, 299, 300, 301, 302, 303, 304, 234, 306, + /* 1510 */ 76, 75, 173, 76, 75, 75, 258, 75, 138, 87, + /* 1520 */ 76, 76, 75, 265, 75, 75, 0, 43, 135, 75, + /* 1530 */ 22, 85, 258, 275, 75, 87, 76, 279, 75, 265, + /* 1540 */ 87, 75, 38, 76, 38, 75, 75, 86, 234, 275, + /* 1550 */ 76, 38, 76, 279, 296, 38, 75, 299, 300, 301, + /* 1560 */ 302, 303, 304, 76, 306, 38, 234, 75, 38, 76, + /* 1570 */ 296, 75, 258, 299, 300, 301, 302, 303, 304, 265, + /* 1580 */ 306, 234, 100, 100, 22, 100, 100, 88, 38, 275, + /* 1590 */ 258, 75, 38, 279, 75, 75, 22, 265, 110, 51, + /* 1600 */ 57, 50, 38, 38, 73, 258, 72, 275, 38, 38, + /* 1610 */ 296, 279, 265, 299, 300, 301, 302, 303, 304, 38, + /* 1620 */ 306, 38, 275, 38, 38, 22, 279, 38, 296, 57, + /* 1630 */ 234, 299, 300, 301, 302, 303, 304, 38, 306, 38, + /* 1640 */ 38, 0, 38, 296, 38, 38, 299, 300, 301, 302, + /* 1650 */ 303, 304, 38, 306, 258, 38, 36, 0, 43, 38, + /* 1660 */ 36, 265, 43, 0, 38, 36, 43, 0, 38, 36, + /* 1670 */ 43, 275, 0, 38, 37, 279, 0, 234, 0, 20, + /* 1680 */ 22, 21, 347, 22, 22, 21, 347, 347, 347, 347, + /* 1690 */ 347, 347, 296, 347, 347, 299, 300, 301, 302, 303, + /* 1700 */ 304, 258, 306, 347, 347, 347, 347, 234, 265, 347, + /* 1710 */ 347, 347, 347, 347, 347, 347, 347, 347, 275, 347, + /* 1720 */ 347, 347, 279, 347, 347, 347, 347, 347, 347, 347, + /* 1730 */ 347, 258, 347, 347, 347, 347, 347, 234, 265, 296, + /* 1740 */ 347, 347, 299, 300, 301, 302, 303, 304, 275, 306, + /* 1750 */ 347, 347, 279, 347, 347, 347, 347, 347, 347, 347, + /* 1760 */ 347, 258, 347, 347, 347, 347, 347, 234, 265, 296, + /* 1770 */ 347, 347, 299, 300, 301, 302, 303, 304, 275, 306, + /* 1780 */ 347, 347, 279, 347, 347, 347, 347, 347, 347, 347, + /* 1790 */ 347, 258, 347, 347, 347, 347, 347, 234, 265, 296, + /* 1800 */ 347, 347, 299, 300, 301, 302, 303, 304, 275, 306, + /* 1810 */ 347, 347, 279, 347, 347, 347, 234, 347, 347, 347, + /* 1820 */ 347, 258, 347, 347, 347, 347, 347, 347, 265, 296, + /* 1830 */ 347, 347, 299, 300, 301, 302, 303, 304, 275, 306, + /* 1840 */ 258, 347, 279, 347, 347, 347, 234, 265, 12, 13, + /* 1850 */ 347, 347, 347, 347, 347, 347, 347, 275, 22, 296, + /* 1860 */ 347, 279, 299, 300, 301, 302, 303, 304, 347, 306, + /* 1870 */ 258, 347, 347, 347, 38, 347, 347, 265, 296, 347, + /* 1880 */ 347, 299, 300, 301, 302, 303, 304, 275, 306, 347, + /* 1890 */ 347, 279, 347, 57, 347, 347, 347, 347, 347, 347, + /* 1900 */ 347, 347, 347, 347, 347, 347, 347, 347, 296, 347, + /* 1910 */ 347, 299, 300, 301, 302, 303, 304, 347, 306, 347, + /* 1920 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 1930 */ 347, 347, 347, 347, 98, 347, 347, 347, 347, 347, + /* 1940 */ 347, 347, 347, 347, 347, 347, 110, 347, 347, 347, + /* 1950 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 1960 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 1970 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 1980 */ 347, 347, 347, 347, 347, 347, 150, 347, 347, 347, + /* 1990 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 2000 */ 347, 347, 347, 347, 347, 347, 347, 171, 347, 347, + /* 2010 */ 347, 347, 347, 347, 347, 347, 347, 347, 182, 183, + /* 2020 */ 184, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 2030 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 2040 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 2050 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 2060 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, + /* 2070 */ 347, 347, 347, 347, 347, 347, 347, 347, 347, 347, }; -#define YY_SHIFT_COUNT (573) +#define YY_SHIFT_COUNT (574) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1621) +#define YY_SHIFT_MAX (1836) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 820, 0, 39, 216, 216, 216, 216, 255, 216, 216, + /* 0 */ 916, 0, 39, 216, 216, 216, 216, 255, 216, 216, /* 10 */ 432, 471, 648, 471, 471, 471, 471, 471, 471, 471, /* 20 */ 471, 471, 471, 471, 471, 471, 471, 471, 471, 471, - /* 30 */ 471, 471, 471, 471, 471, 471, 52, 73, 73, 73, - /* 40 */ 118, 671, 671, 112, 36, 36, 78, 671, 70, 36, - /* 50 */ 36, 36, 36, 36, 36, 107, 36, 272, 341, 78, - /* 60 */ 472, 272, 36, 36, 272, 36, 272, 472, 272, 272, - /* 70 */ 36, 366, 219, 397, 613, 613, 343, 831, 298, 141, - /* 80 */ 831, 831, 831, 831, 831, 831, 831, 831, 831, 831, - /* 90 */ 831, 831, 831, 831, 831, 831, 831, 831, 831, 340, - /* 100 */ 638, 419, 419, 183, 183, 183, 691, 419, 419, 490, - /* 110 */ 472, 272, 472, 272, 475, 548, 27, 27, 27, 27, - /* 120 */ 27, 27, 27, 1040, 115, 414, 552, 144, 169, 165, - /* 130 */ 244, 168, 403, 576, 297, 576, 758, 381, 637, 774, - /* 140 */ 941, 930, 932, 830, 941, 941, 853, 849, 849, 941, - /* 150 */ 971, 107, 472, 980, 107, 490, 987, 107, 107, 941, - /* 160 */ 107, 994, 272, 272, 272, 272, 272, 272, 272, 272, - /* 170 */ 272, 272, 272, 941, 994, 973, 971, 366, 908, 472, - /* 180 */ 366, 980, 366, 490, 987, 366, 1046, 879, 886, 973, - /* 190 */ 879, 886, 973, 973, 896, 881, 909, 912, 918, 490, - /* 200 */ 1100, 1001, 907, 911, 914, 1059, 272, 886, 973, 973, - /* 210 */ 886, 973, 1009, 490, 987, 366, 475, 366, 490, 1097, - /* 220 */ 548, 941, 366, 994, 1906, 1906, 1906, 1906, 1906, 1906, - /* 230 */ 1906, 99, 1359, 256, 13, 406, 837, 228, 405, 446, - /* 240 */ 621, 659, 666, 666, 666, 666, 666, 666, 666, 666, - /* 250 */ 523, 469, 247, 133, 315, 69, 69, 69, 69, 814, - /* 260 */ 336, 690, 762, 767, 772, 777, 845, 852, 870, 211, - /* 270 */ 590, 677, 786, 788, 799, 798, 679, 719, 559, 801, - /* 280 */ 826, 825, 617, 827, 828, 834, 835, 848, 883, 885, - /* 290 */ 859, 867, 875, 877, 880, 882, 233, 898, 1194, 1202, - /* 300 */ 1139, 1213, 1172, 1048, 1178, 1179, 1180, 1053, 1221, 1184, - /* 310 */ 1186, 1058, 1227, 1190, 1229, 1193, 1232, 1158, 1080, 1082, - /* 320 */ 1127, 1088, 1239, 1241, 1189, 1099, 1246, 1247, 1161, 1249, - /* 330 */ 1250, 1251, 1252, 1254, 1255, 1256, 1261, 1262, 1282, 1284, - /* 340 */ 1285, 1286, 1287, 1288, 1289, 1290, 1137, 1259, 1291, 1292, - /* 350 */ 1294, 1295, 1296, 1253, 1297, 1298, 1299, 1301, 1302, 1303, - /* 360 */ 1304, 1305, 1306, 1228, 1272, 1222, 1274, 1307, 1273, 1276, - /* 370 */ 1266, 1313, 1277, 1280, 1275, 1319, 1283, 1293, 1279, 1320, - /* 380 */ 1309, 1308, 1281, 1323, 1325, 1326, 1327, 1244, 1248, 1311, - /* 390 */ 1260, 1267, 1321, 1331, 1312, 1314, 1315, 1316, 1317, 1260, - /* 400 */ 1267, 1318, 1322, 1345, 1329, 1357, 1337, 1324, 1361, 1342, - /* 410 */ 1328, 1367, 1346, 1369, 1349, 1352, 1373, 1234, 1338, 1375, - /* 420 */ 1223, 1358, 1257, 1242, 1389, 1390, 1398, 1332, 1350, 1264, - /* 430 */ 1330, 1336, 1195, 1333, 1344, 1347, 1351, 1353, 1354, 1348, - /* 440 */ 1360, 1335, 1355, 1362, 1214, 1363, 1364, 1356, 1219, 1366, - /* 450 */ 1365, 1370, 1376, 1216, 1387, 1404, 1407, 1409, 1411, 1412, - /* 460 */ 1429, 1300, 1379, 1377, 1380, 1378, 1381, 1394, 1371, 1397, - /* 470 */ 1399, 1386, 1435, 1310, 1402, 1403, 1405, 1413, 1415, 1340, - /* 480 */ 1416, 1469, 1439, 1372, 1418, 1401, 1400, 1408, 1462, 1422, - /* 490 */ 1428, 1443, 1480, 1482, 1446, 1448, 1484, 1450, 1451, 1488, - /* 500 */ 1453, 1454, 1493, 1457, 1458, 1497, 1463, 1440, 1441, 1444, - /* 510 */ 1445, 1517, 1455, 1467, 1509, 1438, 1474, 1476, 1514, 1260, - /* 520 */ 1267, 1531, 1503, 1505, 1524, 1507, 1492, 1494, 1529, 1537, - /* 530 */ 1540, 1541, 1542, 1543, 1544, 1561, 1527, 1260, 1547, 1267, - /* 540 */ 1548, 1549, 1550, 1552, 1553, 1555, 1556, 1589, 1557, 1560, - /* 550 */ 1554, 1598, 1562, 1566, 1563, 1603, 1567, 1568, 1564, 1610, - /* 560 */ 1573, 1576, 1571, 1615, 1579, 1582, 1620, 1621, 1601, 1604, - /* 570 */ 1602, 1606, 1605, 1609, + /* 30 */ 471, 471, 471, 471, 471, 471, 34, 36, 36, 36, + /* 40 */ 1836, 1836, 1836, 153, 49, 9, 9, 133, 54, 9, + /* 50 */ 9, 9, 9, 9, 9, 55, 9, 118, 136, 133, + /* 60 */ 272, 118, 9, 9, 118, 9, 118, 272, 118, 118, + /* 70 */ 9, 323, 219, 401, 613, 613, 258, 957, 694, 141, + /* 80 */ 957, 957, 957, 957, 957, 957, 957, 957, 957, 957, + /* 90 */ 957, 957, 957, 957, 957, 957, 957, 957, 957, 446, + /* 100 */ 340, 91, 91, 350, 350, 350, 531, 91, 91, 297, + /* 110 */ 272, 118, 272, 118, 291, 408, 29, 29, 29, 29, + /* 120 */ 29, 29, 29, 1128, 115, 315, 228, 211, 455, 159, + /* 130 */ 380, 567, 538, 579, 592, 579, 750, 456, 688, 766, + /* 140 */ 888, 917, 927, 820, 888, 888, 866, 856, 856, 888, + /* 150 */ 986, 55, 272, 991, 55, 297, 999, 55, 55, 888, + /* 160 */ 55, 1007, 118, 118, 118, 118, 118, 118, 118, 118, + /* 170 */ 118, 118, 118, 888, 1007, 977, 986, 323, 914, 272, + /* 180 */ 323, 991, 323, 297, 999, 323, 1061, 874, 884, 977, + /* 190 */ 874, 884, 977, 977, 900, 899, 915, 922, 918, 297, + /* 200 */ 1102, 1005, 920, 928, 923, 1067, 118, 884, 977, 977, + /* 210 */ 884, 977, 1026, 297, 999, 323, 291, 323, 297, 1101, + /* 220 */ 408, 888, 323, 1007, 2021, 2021, 2021, 2021, 2021, 2021, + /* 230 */ 2021, 99, 427, 331, 939, 730, 756, 499, 78, 558, + /* 240 */ 394, 618, 27, 27, 27, 27, 27, 27, 27, 27, + /* 250 */ 597, 433, 178, 600, 387, 176, 176, 176, 176, 776, + /* 260 */ 620, 627, 705, 711, 715, 717, 849, 887, 919, 767, + /* 270 */ 652, 710, 770, 780, 783, 852, 682, 44, 656, 786, + /* 280 */ 723, 794, 825, 826, 827, 829, 850, 859, 886, 898, + /* 290 */ 868, 890, 895, 902, 912, 913, 121, 743, 1207, 1209, + /* 300 */ 1146, 1211, 1174, 1046, 1176, 1177, 1178, 1051, 1219, 1188, + /* 310 */ 1190, 1062, 1230, 1193, 1233, 1196, 1236, 1162, 1084, 1087, + /* 320 */ 1131, 1092, 1243, 1244, 1192, 1100, 1247, 1248, 1163, 1251, + /* 330 */ 1252, 1254, 1255, 1256, 1265, 1266, 1269, 1271, 1272, 1279, + /* 340 */ 1282, 1283, 1285, 1287, 1289, 1290, 1137, 1259, 1260, 1261, + /* 350 */ 1262, 1263, 1291, 1273, 1293, 1294, 1296, 1297, 1300, 1301, + /* 360 */ 1303, 1304, 1305, 1264, 1306, 1257, 1310, 1312, 1276, 1281, + /* 370 */ 1275, 1320, 1307, 1292, 1278, 1341, 1311, 1308, 1299, 1343, + /* 380 */ 1313, 1316, 1315, 1348, 1350, 1353, 1354, 1277, 1250, 1319, + /* 390 */ 1298, 1302, 1337, 1363, 1326, 1327, 1328, 1330, 1331, 1298, + /* 400 */ 1302, 1335, 1352, 1375, 1364, 1391, 1372, 1365, 1402, 1384, + /* 410 */ 1369, 1408, 1388, 1411, 1398, 1401, 1422, 1284, 1385, 1425, + /* 420 */ 1280, 1404, 1288, 1295, 1427, 1430, 1431, 1358, 1395, 1309, + /* 430 */ 1362, 1367, 1235, 1371, 1373, 1376, 1366, 1368, 1378, 1379, + /* 440 */ 1382, 1361, 1381, 1386, 1246, 1389, 1390, 1370, 1270, 1392, + /* 450 */ 1394, 1400, 1407, 1286, 1424, 1451, 1455, 1456, 1457, 1458, + /* 460 */ 1495, 1329, 1426, 1434, 1436, 1437, 1414, 1439, 1440, 1432, + /* 470 */ 1438, 1339, 1442, 1444, 1445, 1447, 1449, 1380, 1450, 1526, + /* 480 */ 1484, 1393, 1454, 1446, 1448, 1453, 1459, 1460, 1463, 1508, + /* 490 */ 1466, 1461, 1467, 1504, 1506, 1470, 1474, 1513, 1471, 1476, + /* 500 */ 1517, 1481, 1487, 1527, 1492, 1493, 1530, 1496, 1482, 1483, + /* 510 */ 1485, 1486, 1562, 1499, 1516, 1550, 1488, 1519, 1520, 1554, + /* 520 */ 1298, 1302, 1574, 1548, 1551, 1564, 1543, 1531, 1534, 1565, + /* 530 */ 1570, 1571, 1581, 1583, 1585, 1586, 1603, 1572, 1298, 1589, + /* 540 */ 1302, 1599, 1601, 1602, 1604, 1606, 1607, 1614, 1641, 1617, + /* 550 */ 1620, 1615, 1657, 1621, 1624, 1619, 1663, 1626, 1629, 1623, + /* 560 */ 1667, 1630, 1633, 1627, 1672, 1635, 1637, 1676, 1678, 1658, + /* 570 */ 1660, 1661, 1662, 1664, 1659, }; #define YY_REDUCE_COUNT (230) -#define YY_REDUCE_MIN (-323) -#define YY_REDUCE_MAX (1600) +#define YY_REDUCE_MIN (-278) +#define YY_REDUCE_MAX (1612) static const short yy_reduce_ofst[] = { - /* 0 */ -34, 221, -232, 431, -3, 582, 614, 734, 484, 743, - /* 10 */ 804, 858, 906, 274, 890, 965, 978, 1035, 1083, 1112, - /* 20 */ 1162, 1201, 1210, 1258, 1271, 1334, 1343, 1382, 1393, 1442, - /* 30 */ 1481, 1490, 1499, 1538, 1551, 1600, 436, -210, 710, 824, - /* 40 */ 213, -258, -255, -279, -240, -221, -89, -231, -323, -161, - /* 50 */ 29, 225, 262, 276, 295, -177, 331, -235, -67, -77, - /* 60 */ -270, -153, 454, 459, 1, 462, -58, -172, 58, 158, - /* 70 */ 476, 299, 61, -268, -268, -268, 150, 55, 49, -216, - /* 80 */ 143, 261, 269, 278, 284, 344, 351, 365, 388, 390, - /* 90 */ 398, 428, 486, 492, 514, 516, 522, 534, 536, 226, - /* 100 */ -151, 243, 357, -46, 368, 370, -66, 395, 468, -262, - /* 110 */ 224, 289, 440, 521, 174, 452, -244, 426, 458, 463, - /* 120 */ 478, 553, 604, 598, 643, 644, 564, 574, 650, 641, - /* 130 */ 601, 697, 645, 618, 618, 618, 699, 622, 634, 699, - /* 140 */ 722, 676, 727, 687, 736, 739, 707, 711, 714, 747, - /* 150 */ 704, 754, 723, 721, 763, 741, 735, 768, 773, 776, - /* 160 */ 775, 784, 765, 766, 782, 789, 791, 792, 794, 795, - /* 170 */ 796, 802, 803, 785, 790, 778, 780, 829, 783, 805, - /* 180 */ 838, 807, 839, 809, 787, 842, 770, 757, 813, 816, - /* 190 */ 764, 817, 821, 822, 797, 769, 793, 800, 618, 854, - /* 200 */ 833, 810, 806, 812, 811, 843, 699, 855, 857, 860, - /* 210 */ 864, 874, 861, 884, 891, 923, 916, 927, 903, 913, - /* 220 */ 925, 935, 938, 942, 893, 892, 900, 934, 945, 946, - /* 230 */ 950, + /* 0 */ 337, -230, 223, 435, 582, 614, 646, 698, 799, 831, + /* 10 */ -1, 747, 881, 921, 974, 1023, 1034, 1081, 1097, 1113, + /* 20 */ 1171, 1184, 1203, 1258, 1274, 1314, 1332, 1347, 1396, 1443, + /* 30 */ 1473, 1503, 1533, 1563, 1582, 1612, -9, -217, 5, 399, + /* 40 */ -262, 200, -270, 299, -278, -238, -235, -91, -260, -234, + /* 50 */ -203, -152, 31, 32, 215, 6, 227, -61, 63, -257, + /* 60 */ -16, 162, 238, 268, 116, 300, 295, 144, 189, 301, + /* 70 */ 314, 222, -153, -204, -204, -204, -232, 265, -249, -180, + /* 80 */ 344, 402, 404, 406, 443, 448, 453, 454, 461, 464, + /* 90 */ 472, 473, 474, 475, 478, 481, 482, 486, 492, 146, + /* 100 */ -33, 204, 421, 370, 375, 383, -7, 445, 480, 103, + /* 110 */ -265, 239, 488, 319, 491, 465, 251, 293, 324, 341, + /* 120 */ 425, 442, 509, 495, 554, 568, 498, 524, 577, 581, + /* 130 */ 544, 638, 578, 569, 569, 569, 649, 591, 607, 649, + /* 140 */ 697, 661, 713, 683, 736, 742, 718, 721, 722, 763, + /* 150 */ 719, 765, 735, 732, 775, 753, 744, 779, 782, 785, + /* 160 */ 787, 795, 772, 774, 777, 781, 784, 796, 798, 800, + /* 170 */ 804, 808, 811, 797, 802, 762, 754, 806, 788, 801, + /* 180 */ 813, 789, 834, 805, 803, 836, 790, 752, 814, 818, + /* 190 */ 758, 824, 830, 832, 791, 793, 792, 810, 569, 855, + /* 200 */ 839, 833, 807, 809, 812, 842, 649, 871, 878, 879, + /* 210 */ 882, 880, 889, 903, 891, 926, 924, 930, 906, 931, + /* 220 */ 940, 949, 952, 958, 910, 907, 909, 947, 954, 960, + /* 230 */ 972, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 10 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 20 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 30 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 40 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 50 */ 1296, 1296, 1296, 1296, 1296, 1355, 1296, 1296, 1296, 1296, - /* 60 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 70 */ 1296, 1353, 1498, 1296, 1658, 1296, 1296, 1296, 1296, 1296, - /* 80 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 90 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 100 */ 1355, 1296, 1296, 1669, 1669, 1669, 1353, 1296, 1296, 1296, - /* 110 */ 1296, 1296, 1296, 1296, 1450, 1296, 1296, 1296, 1296, 1296, - /* 120 */ 1296, 1296, 1296, 1536, 1296, 1296, 1734, 1296, 1403, 1542, - /* 130 */ 1693, 1296, 1685, 1661, 1675, 1662, 1296, 1719, 1678, 1296, - /* 140 */ 1296, 1296, 1296, 1528, 1296, 1296, 1503, 1500, 1500, 1296, - /* 150 */ 1296, 1355, 1296, 1296, 1355, 1296, 1296, 1355, 1355, 1296, - /* 160 */ 1355, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 170 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1353, 1538, 1296, - /* 180 */ 1353, 1296, 1353, 1296, 1296, 1353, 1296, 1700, 1698, 1296, - /* 190 */ 1700, 1698, 1296, 1296, 1712, 1708, 1691, 1689, 1675, 1296, - /* 200 */ 1296, 1296, 1737, 1725, 1721, 1296, 1296, 1698, 1296, 1296, - /* 210 */ 1698, 1296, 1511, 1296, 1296, 1353, 1296, 1353, 1296, 1419, - /* 220 */ 1296, 1296, 1353, 1296, 1530, 1544, 1520, 1453, 1453, 1356, - /* 230 */ 1301, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 240 */ 1296, 1296, 1605, 1711, 1710, 1634, 1633, 1632, 1630, 1604, - /* 250 */ 1296, 1296, 1296, 1296, 1296, 1598, 1599, 1597, 1596, 1296, - /* 260 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 270 */ 1296, 1296, 1296, 1296, 1296, 1659, 1296, 1722, 1726, 1296, - /* 280 */ 1296, 1296, 1582, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 290 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 300 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 310 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 320 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 330 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 340 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 350 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 360 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 370 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 380 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 390 */ 1466, 1465, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1383, - /* 400 */ 1382, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 410 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 420 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 430 */ 1682, 1692, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 440 */ 1296, 1582, 1296, 1709, 1296, 1668, 1664, 1296, 1296, 1660, - /* 450 */ 1296, 1296, 1720, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 460 */ 1654, 1296, 1627, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 470 */ 1296, 1296, 1296, 1592, 1296, 1296, 1296, 1296, 1296, 1296, - /* 480 */ 1296, 1296, 1296, 1296, 1296, 1296, 1581, 1296, 1296, 1296, - /* 490 */ 1296, 1296, 1296, 1296, 1447, 1296, 1296, 1296, 1296, 1296, - /* 500 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1432, 1430, 1429, - /* 510 */ 1428, 1296, 1425, 1296, 1296, 1296, 1296, 1296, 1296, 1456, - /* 520 */ 1455, 1296, 1296, 1296, 1296, 1296, 1296, 1376, 1296, 1296, - /* 530 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1367, 1296, 1366, - /* 540 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 550 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 560 */ 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, 1296, - /* 570 */ 1296, 1296, 1296, 1296, + /* 0 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 10 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 20 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 30 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 40 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 50 */ 1301, 1301, 1301, 1301, 1301, 1360, 1301, 1301, 1301, 1301, + /* 60 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 70 */ 1301, 1358, 1503, 1301, 1666, 1301, 1301, 1301, 1301, 1301, + /* 80 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 90 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 100 */ 1360, 1301, 1301, 1677, 1677, 1677, 1358, 1301, 1301, 1301, + /* 110 */ 1301, 1301, 1301, 1301, 1455, 1301, 1301, 1301, 1301, 1301, + /* 120 */ 1301, 1301, 1301, 1541, 1301, 1301, 1742, 1301, 1408, 1547, + /* 130 */ 1701, 1301, 1693, 1669, 1683, 1670, 1301, 1727, 1686, 1301, + /* 140 */ 1301, 1301, 1301, 1533, 1301, 1301, 1508, 1505, 1505, 1301, + /* 150 */ 1301, 1360, 1301, 1301, 1360, 1301, 1301, 1360, 1360, 1301, + /* 160 */ 1360, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 170 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1358, 1543, 1301, + /* 180 */ 1358, 1301, 1358, 1301, 1301, 1358, 1301, 1708, 1706, 1301, + /* 190 */ 1708, 1706, 1301, 1301, 1720, 1716, 1699, 1697, 1683, 1301, + /* 200 */ 1301, 1301, 1745, 1733, 1729, 1301, 1301, 1706, 1301, 1301, + /* 210 */ 1706, 1301, 1516, 1301, 1301, 1358, 1301, 1358, 1301, 1424, + /* 220 */ 1301, 1301, 1358, 1301, 1535, 1549, 1525, 1458, 1458, 1361, + /* 230 */ 1306, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 240 */ 1301, 1301, 1611, 1719, 1718, 1642, 1641, 1640, 1638, 1610, + /* 250 */ 1301, 1301, 1301, 1301, 1301, 1604, 1605, 1603, 1602, 1301, + /* 260 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 270 */ 1301, 1301, 1301, 1301, 1301, 1667, 1301, 1730, 1734, 1301, + /* 280 */ 1301, 1301, 1588, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 290 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 300 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 310 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 320 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 330 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 340 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 350 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 360 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 370 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 380 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 390 */ 1471, 1470, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1388, + /* 400 */ 1387, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 410 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 420 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 430 */ 1690, 1700, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 440 */ 1301, 1588, 1301, 1717, 1301, 1676, 1672, 1301, 1301, 1668, + /* 450 */ 1301, 1301, 1728, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 460 */ 1662, 1301, 1635, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 470 */ 1301, 1598, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 480 */ 1301, 1301, 1301, 1301, 1587, 1301, 1626, 1301, 1301, 1301, + /* 490 */ 1301, 1301, 1301, 1301, 1301, 1452, 1301, 1301, 1301, 1301, + /* 500 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1437, 1435, + /* 510 */ 1434, 1433, 1301, 1430, 1301, 1301, 1301, 1301, 1301, 1301, + /* 520 */ 1461, 1460, 1301, 1301, 1301, 1301, 1301, 1301, 1381, 1301, + /* 530 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1372, 1301, + /* 540 */ 1371, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 550 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 560 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, + /* 570 */ 1301, 1301, 1301, 1301, 1301, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1382,56 +1410,57 @@ static const char *const yyTokenName[] = { /* 293 */ "dnode_list", /* 294 */ "signed", /* 295 */ "signed_literal", - /* 296 */ "table_alias", - /* 297 */ "column_alias", - /* 298 */ "expression", - /* 299 */ "pseudo_column", - /* 300 */ "column_reference", - /* 301 */ "function_expression", - /* 302 */ "subquery", - /* 303 */ "star_func", - /* 304 */ "star_func_para_list", - /* 305 */ "noarg_func", - /* 306 */ "other_para_list", - /* 307 */ "star_func_para", - /* 308 */ "predicate", - /* 309 */ "compare_op", - /* 310 */ "in_op", - /* 311 */ "in_predicate_value", - /* 312 */ "boolean_value_expression", - /* 313 */ "boolean_primary", - /* 314 */ "common_expression", - /* 315 */ "from_clause", - /* 316 */ "table_reference_list", - /* 317 */ "table_reference", - /* 318 */ "table_primary", - /* 319 */ "joined_table", - /* 320 */ "alias_opt", - /* 321 */ "parenthesized_joined_table", - /* 322 */ "join_type", - /* 323 */ "search_condition", - /* 324 */ "query_specification", - /* 325 */ "set_quantifier_opt", - /* 326 */ "select_list", - /* 327 */ "where_clause_opt", - /* 328 */ "partition_by_clause_opt", - /* 329 */ "twindow_clause_opt", - /* 330 */ "group_by_clause_opt", - /* 331 */ "having_clause_opt", - /* 332 */ "select_sublist", - /* 333 */ "select_item", - /* 334 */ "fill_opt", - /* 335 */ "fill_mode", - /* 336 */ "group_by_list", - /* 337 */ "query_expression_body", - /* 338 */ "order_by_clause_opt", - /* 339 */ "slimit_clause_opt", - /* 340 */ "limit_clause_opt", - /* 341 */ "query_primary", - /* 342 */ "sort_specification_list", - /* 343 */ "sort_specification", - /* 344 */ "ordering_specification_opt", - /* 345 */ "null_ordering_opt", + /* 296 */ "literal_func", + /* 297 */ "table_alias", + /* 298 */ "column_alias", + /* 299 */ "expression", + /* 300 */ "pseudo_column", + /* 301 */ "column_reference", + /* 302 */ "function_expression", + /* 303 */ "subquery", + /* 304 */ "star_func", + /* 305 */ "star_func_para_list", + /* 306 */ "noarg_func", + /* 307 */ "other_para_list", + /* 308 */ "star_func_para", + /* 309 */ "predicate", + /* 310 */ "compare_op", + /* 311 */ "in_op", + /* 312 */ "in_predicate_value", + /* 313 */ "boolean_value_expression", + /* 314 */ "boolean_primary", + /* 315 */ "common_expression", + /* 316 */ "from_clause", + /* 317 */ "table_reference_list", + /* 318 */ "table_reference", + /* 319 */ "table_primary", + /* 320 */ "joined_table", + /* 321 */ "alias_opt", + /* 322 */ "parenthesized_joined_table", + /* 323 */ "join_type", + /* 324 */ "search_condition", + /* 325 */ "query_specification", + /* 326 */ "set_quantifier_opt", + /* 327 */ "select_list", + /* 328 */ "where_clause_opt", + /* 329 */ "partition_by_clause_opt", + /* 330 */ "twindow_clause_opt", + /* 331 */ "group_by_clause_opt", + /* 332 */ "having_clause_opt", + /* 333 */ "select_sublist", + /* 334 */ "select_item", + /* 335 */ "fill_opt", + /* 336 */ "fill_mode", + /* 337 */ "group_by_list", + /* 338 */ "query_expression_body", + /* 339 */ "order_by_clause_opt", + /* 340 */ "slimit_clause_opt", + /* 341 */ "limit_clause_opt", + /* 342 */ "query_primary", + /* 343 */ "sort_specification_list", + /* 344 */ "sort_specification", + /* 345 */ "ordering_specification_opt", + /* 346 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1718,168 +1747,171 @@ static const char *const yyRuleName[] = { /* 276 */ "signed_literal ::= TIMESTAMP NK_STRING", /* 277 */ "signed_literal ::= duration_literal", /* 278 */ "signed_literal ::= NULL", - /* 279 */ "literal_list ::= signed_literal", - /* 280 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 281 */ "db_name ::= NK_ID", - /* 282 */ "table_name ::= NK_ID", - /* 283 */ "column_name ::= NK_ID", - /* 284 */ "function_name ::= NK_ID", - /* 285 */ "table_alias ::= NK_ID", - /* 286 */ "column_alias ::= NK_ID", - /* 287 */ "user_name ::= NK_ID", - /* 288 */ "index_name ::= NK_ID", - /* 289 */ "topic_name ::= NK_ID", - /* 290 */ "stream_name ::= NK_ID", - /* 291 */ "expression ::= literal", - /* 292 */ "expression ::= pseudo_column", - /* 293 */ "expression ::= column_reference", - /* 294 */ "expression ::= function_expression", - /* 295 */ "expression ::= subquery", - /* 296 */ "expression ::= NK_LP expression NK_RP", - /* 297 */ "expression ::= NK_PLUS expression", - /* 298 */ "expression ::= NK_MINUS expression", - /* 299 */ "expression ::= expression NK_PLUS expression", - /* 300 */ "expression ::= expression NK_MINUS expression", - /* 301 */ "expression ::= expression NK_STAR expression", - /* 302 */ "expression ::= expression NK_SLASH expression", - /* 303 */ "expression ::= expression NK_REM expression", - /* 304 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 305 */ "expression_list ::= expression", - /* 306 */ "expression_list ::= expression_list NK_COMMA expression", - /* 307 */ "column_reference ::= column_name", - /* 308 */ "column_reference ::= table_name NK_DOT column_name", - /* 309 */ "pseudo_column ::= ROWTS", - /* 310 */ "pseudo_column ::= TBNAME", - /* 311 */ "pseudo_column ::= QSTARTTS", - /* 312 */ "pseudo_column ::= QENDTS", - /* 313 */ "pseudo_column ::= WSTARTTS", - /* 314 */ "pseudo_column ::= WENDTS", - /* 315 */ "pseudo_column ::= WDURATION", - /* 316 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 317 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 318 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", - /* 319 */ "function_expression ::= noarg_func NK_LP NK_RP", - /* 320 */ "noarg_func ::= NOW", - /* 321 */ "noarg_func ::= TODAY", - /* 322 */ "noarg_func ::= TIMEZONE", - /* 323 */ "star_func ::= COUNT", - /* 324 */ "star_func ::= FIRST", - /* 325 */ "star_func ::= LAST", - /* 326 */ "star_func ::= LAST_ROW", - /* 327 */ "star_func_para_list ::= NK_STAR", - /* 328 */ "star_func_para_list ::= other_para_list", - /* 329 */ "other_para_list ::= star_func_para", - /* 330 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 331 */ "star_func_para ::= expression", - /* 332 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 333 */ "predicate ::= expression compare_op expression", - /* 334 */ "predicate ::= expression BETWEEN expression AND expression", - /* 335 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 336 */ "predicate ::= expression IS NULL", - /* 337 */ "predicate ::= expression IS NOT NULL", - /* 338 */ "predicate ::= expression in_op in_predicate_value", - /* 339 */ "compare_op ::= NK_LT", - /* 340 */ "compare_op ::= NK_GT", - /* 341 */ "compare_op ::= NK_LE", - /* 342 */ "compare_op ::= NK_GE", - /* 343 */ "compare_op ::= NK_NE", - /* 344 */ "compare_op ::= NK_EQ", - /* 345 */ "compare_op ::= LIKE", - /* 346 */ "compare_op ::= NOT LIKE", - /* 347 */ "compare_op ::= MATCH", - /* 348 */ "compare_op ::= NMATCH", - /* 349 */ "compare_op ::= CONTAINS", - /* 350 */ "in_op ::= IN", - /* 351 */ "in_op ::= NOT IN", - /* 352 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 353 */ "boolean_value_expression ::= boolean_primary", - /* 354 */ "boolean_value_expression ::= NOT boolean_primary", - /* 355 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 356 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 357 */ "boolean_primary ::= predicate", - /* 358 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 359 */ "common_expression ::= expression", - /* 360 */ "common_expression ::= boolean_value_expression", - /* 361 */ "from_clause ::= FROM table_reference_list", - /* 362 */ "table_reference_list ::= table_reference", - /* 363 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 364 */ "table_reference ::= table_primary", - /* 365 */ "table_reference ::= joined_table", - /* 366 */ "table_primary ::= table_name alias_opt", - /* 367 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 368 */ "table_primary ::= subquery alias_opt", - /* 369 */ "table_primary ::= parenthesized_joined_table", - /* 370 */ "alias_opt ::=", - /* 371 */ "alias_opt ::= table_alias", - /* 372 */ "alias_opt ::= AS table_alias", - /* 373 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 374 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 375 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 376 */ "join_type ::=", - /* 377 */ "join_type ::= INNER", - /* 378 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 379 */ "set_quantifier_opt ::=", - /* 380 */ "set_quantifier_opt ::= DISTINCT", - /* 381 */ "set_quantifier_opt ::= ALL", - /* 382 */ "select_list ::= NK_STAR", - /* 383 */ "select_list ::= select_sublist", - /* 384 */ "select_sublist ::= select_item", - /* 385 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 386 */ "select_item ::= common_expression", - /* 387 */ "select_item ::= common_expression column_alias", - /* 388 */ "select_item ::= common_expression AS column_alias", - /* 389 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 390 */ "where_clause_opt ::=", - /* 391 */ "where_clause_opt ::= WHERE search_condition", - /* 392 */ "partition_by_clause_opt ::=", - /* 393 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 394 */ "twindow_clause_opt ::=", - /* 395 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 396 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", - /* 397 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 398 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 399 */ "sliding_opt ::=", - /* 400 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 401 */ "fill_opt ::=", - /* 402 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 403 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 404 */ "fill_mode ::= NONE", - /* 405 */ "fill_mode ::= PREV", - /* 406 */ "fill_mode ::= NULL", - /* 407 */ "fill_mode ::= LINEAR", - /* 408 */ "fill_mode ::= NEXT", - /* 409 */ "group_by_clause_opt ::=", - /* 410 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 411 */ "group_by_list ::= expression", - /* 412 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 413 */ "having_clause_opt ::=", - /* 414 */ "having_clause_opt ::= HAVING search_condition", - /* 415 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 416 */ "query_expression_body ::= query_primary", - /* 417 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 418 */ "query_expression_body ::= query_expression_body UNION query_expression_body", - /* 419 */ "query_primary ::= query_specification", - /* 420 */ "order_by_clause_opt ::=", - /* 421 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 422 */ "slimit_clause_opt ::=", - /* 423 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 424 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 425 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 426 */ "limit_clause_opt ::=", - /* 427 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 428 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 429 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 430 */ "subquery ::= NK_LP query_expression NK_RP", - /* 431 */ "search_condition ::= common_expression", - /* 432 */ "sort_specification_list ::= sort_specification", - /* 433 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 434 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 435 */ "ordering_specification_opt ::=", - /* 436 */ "ordering_specification_opt ::= ASC", - /* 437 */ "ordering_specification_opt ::= DESC", - /* 438 */ "null_ordering_opt ::=", - /* 439 */ "null_ordering_opt ::= NULLS FIRST", - /* 440 */ "null_ordering_opt ::= NULLS LAST", + /* 279 */ "signed_literal ::= literal_func", + /* 280 */ "literal_list ::= signed_literal", + /* 281 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 282 */ "db_name ::= NK_ID", + /* 283 */ "table_name ::= NK_ID", + /* 284 */ "column_name ::= NK_ID", + /* 285 */ "function_name ::= NK_ID", + /* 286 */ "table_alias ::= NK_ID", + /* 287 */ "column_alias ::= NK_ID", + /* 288 */ "user_name ::= NK_ID", + /* 289 */ "index_name ::= NK_ID", + /* 290 */ "topic_name ::= NK_ID", + /* 291 */ "stream_name ::= NK_ID", + /* 292 */ "expression ::= literal", + /* 293 */ "expression ::= pseudo_column", + /* 294 */ "expression ::= column_reference", + /* 295 */ "expression ::= function_expression", + /* 296 */ "expression ::= subquery", + /* 297 */ "expression ::= NK_LP expression NK_RP", + /* 298 */ "expression ::= NK_PLUS expression", + /* 299 */ "expression ::= NK_MINUS expression", + /* 300 */ "expression ::= expression NK_PLUS expression", + /* 301 */ "expression ::= expression NK_MINUS expression", + /* 302 */ "expression ::= expression NK_STAR expression", + /* 303 */ "expression ::= expression NK_SLASH expression", + /* 304 */ "expression ::= expression NK_REM expression", + /* 305 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 306 */ "expression_list ::= expression", + /* 307 */ "expression_list ::= expression_list NK_COMMA expression", + /* 308 */ "column_reference ::= column_name", + /* 309 */ "column_reference ::= table_name NK_DOT column_name", + /* 310 */ "pseudo_column ::= ROWTS", + /* 311 */ "pseudo_column ::= TBNAME", + /* 312 */ "pseudo_column ::= QSTARTTS", + /* 313 */ "pseudo_column ::= QENDTS", + /* 314 */ "pseudo_column ::= WSTARTTS", + /* 315 */ "pseudo_column ::= WENDTS", + /* 316 */ "pseudo_column ::= WDURATION", + /* 317 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 318 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 319 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", + /* 320 */ "function_expression ::= literal_func", + /* 321 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 322 */ "literal_func ::= NOW", + /* 323 */ "noarg_func ::= NOW", + /* 324 */ "noarg_func ::= TODAY", + /* 325 */ "noarg_func ::= TIMEZONE", + /* 326 */ "star_func ::= COUNT", + /* 327 */ "star_func ::= FIRST", + /* 328 */ "star_func ::= LAST", + /* 329 */ "star_func ::= LAST_ROW", + /* 330 */ "star_func_para_list ::= NK_STAR", + /* 331 */ "star_func_para_list ::= other_para_list", + /* 332 */ "other_para_list ::= star_func_para", + /* 333 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 334 */ "star_func_para ::= expression", + /* 335 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 336 */ "predicate ::= expression compare_op expression", + /* 337 */ "predicate ::= expression BETWEEN expression AND expression", + /* 338 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 339 */ "predicate ::= expression IS NULL", + /* 340 */ "predicate ::= expression IS NOT NULL", + /* 341 */ "predicate ::= expression in_op in_predicate_value", + /* 342 */ "compare_op ::= NK_LT", + /* 343 */ "compare_op ::= NK_GT", + /* 344 */ "compare_op ::= NK_LE", + /* 345 */ "compare_op ::= NK_GE", + /* 346 */ "compare_op ::= NK_NE", + /* 347 */ "compare_op ::= NK_EQ", + /* 348 */ "compare_op ::= LIKE", + /* 349 */ "compare_op ::= NOT LIKE", + /* 350 */ "compare_op ::= MATCH", + /* 351 */ "compare_op ::= NMATCH", + /* 352 */ "compare_op ::= CONTAINS", + /* 353 */ "in_op ::= IN", + /* 354 */ "in_op ::= NOT IN", + /* 355 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 356 */ "boolean_value_expression ::= boolean_primary", + /* 357 */ "boolean_value_expression ::= NOT boolean_primary", + /* 358 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 359 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 360 */ "boolean_primary ::= predicate", + /* 361 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 362 */ "common_expression ::= expression", + /* 363 */ "common_expression ::= boolean_value_expression", + /* 364 */ "from_clause ::= FROM table_reference_list", + /* 365 */ "table_reference_list ::= table_reference", + /* 366 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 367 */ "table_reference ::= table_primary", + /* 368 */ "table_reference ::= joined_table", + /* 369 */ "table_primary ::= table_name alias_opt", + /* 370 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 371 */ "table_primary ::= subquery alias_opt", + /* 372 */ "table_primary ::= parenthesized_joined_table", + /* 373 */ "alias_opt ::=", + /* 374 */ "alias_opt ::= table_alias", + /* 375 */ "alias_opt ::= AS table_alias", + /* 376 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 377 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 378 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 379 */ "join_type ::=", + /* 380 */ "join_type ::= INNER", + /* 381 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 382 */ "set_quantifier_opt ::=", + /* 383 */ "set_quantifier_opt ::= DISTINCT", + /* 384 */ "set_quantifier_opt ::= ALL", + /* 385 */ "select_list ::= NK_STAR", + /* 386 */ "select_list ::= select_sublist", + /* 387 */ "select_sublist ::= select_item", + /* 388 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 389 */ "select_item ::= common_expression", + /* 390 */ "select_item ::= common_expression column_alias", + /* 391 */ "select_item ::= common_expression AS column_alias", + /* 392 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 393 */ "where_clause_opt ::=", + /* 394 */ "where_clause_opt ::= WHERE search_condition", + /* 395 */ "partition_by_clause_opt ::=", + /* 396 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 397 */ "twindow_clause_opt ::=", + /* 398 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 399 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", + /* 400 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 401 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 402 */ "sliding_opt ::=", + /* 403 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 404 */ "fill_opt ::=", + /* 405 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 406 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 407 */ "fill_mode ::= NONE", + /* 408 */ "fill_mode ::= PREV", + /* 409 */ "fill_mode ::= NULL", + /* 410 */ "fill_mode ::= LINEAR", + /* 411 */ "fill_mode ::= NEXT", + /* 412 */ "group_by_clause_opt ::=", + /* 413 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 414 */ "group_by_list ::= expression", + /* 415 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 416 */ "having_clause_opt ::=", + /* 417 */ "having_clause_opt ::= HAVING search_condition", + /* 418 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 419 */ "query_expression_body ::= query_primary", + /* 420 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 421 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 422 */ "query_primary ::= query_specification", + /* 423 */ "order_by_clause_opt ::=", + /* 424 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 425 */ "slimit_clause_opt ::=", + /* 426 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 427 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 428 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 429 */ "limit_clause_opt ::=", + /* 430 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 431 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 432 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 433 */ "subquery ::= NK_LP query_expression NK_RP", + /* 434 */ "search_condition ::= common_expression", + /* 435 */ "sort_specification_list ::= sort_specification", + /* 436 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 437 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 438 */ "ordering_specification_opt ::=", + /* 439 */ "ordering_specification_opt ::= ASC", + /* 440 */ "ordering_specification_opt ::= DESC", + /* 441 */ "null_ordering_opt ::=", + /* 442 */ "null_ordering_opt ::= NULLS FIRST", + /* 443 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2035,37 +2067,38 @@ static void yy_destructor( case 292: /* into_opt */ case 294: /* signed */ case 295: /* signed_literal */ - case 298: /* expression */ - case 299: /* pseudo_column */ - case 300: /* column_reference */ - case 301: /* function_expression */ - case 302: /* subquery */ - case 307: /* star_func_para */ - case 308: /* predicate */ - case 311: /* in_predicate_value */ - case 312: /* boolean_value_expression */ - case 313: /* boolean_primary */ - case 314: /* common_expression */ - case 315: /* from_clause */ - case 316: /* table_reference_list */ - case 317: /* table_reference */ - case 318: /* table_primary */ - case 319: /* joined_table */ - case 321: /* parenthesized_joined_table */ - case 323: /* search_condition */ - case 324: /* query_specification */ - case 327: /* where_clause_opt */ - case 329: /* twindow_clause_opt */ - case 331: /* having_clause_opt */ - case 333: /* select_item */ - case 334: /* fill_opt */ - case 337: /* query_expression_body */ - case 339: /* slimit_clause_opt */ - case 340: /* limit_clause_opt */ - case 341: /* query_primary */ - case 343: /* sort_specification */ + case 296: /* literal_func */ + case 299: /* expression */ + case 300: /* pseudo_column */ + case 301: /* column_reference */ + case 302: /* function_expression */ + case 303: /* subquery */ + case 308: /* star_func_para */ + case 309: /* predicate */ + case 312: /* in_predicate_value */ + case 313: /* boolean_value_expression */ + case 314: /* boolean_primary */ + case 315: /* common_expression */ + case 316: /* from_clause */ + case 317: /* table_reference_list */ + case 318: /* table_reference */ + case 319: /* table_primary */ + case 320: /* joined_table */ + case 322: /* parenthesized_joined_table */ + case 324: /* search_condition */ + case 325: /* query_specification */ + case 328: /* where_clause_opt */ + case 330: /* twindow_clause_opt */ + case 332: /* having_clause_opt */ + case 334: /* select_item */ + case 335: /* fill_opt */ + case 338: /* query_expression_body */ + case 340: /* slimit_clause_opt */ + case 341: /* limit_clause_opt */ + case 342: /* query_primary */ + case 344: /* sort_specification */ { - nodesDestroyNode((yypminor->yy456)); + nodesDestroyNode((yypminor->yy662)); } break; case 232: /* account_options */ @@ -2086,11 +2119,11 @@ static void yy_destructor( case 276: /* index_name */ case 283: /* topic_name */ case 290: /* stream_name */ - case 296: /* table_alias */ - case 297: /* column_alias */ - case 303: /* star_func */ - case 305: /* noarg_func */ - case 320: /* alias_opt */ + case 297: /* table_alias */ + case 298: /* column_alias */ + case 304: /* star_func */ + case 306: /* noarg_func */ + case 321: /* alias_opt */ { } @@ -2099,7 +2132,7 @@ static void yy_destructor( case 242: /* exists_opt */ case 286: /* analyze_opt */ case 288: /* agg_func_opt */ - case 325: /* set_quantifier_opt */ + case 326: /* set_quantifier_opt */ { } @@ -2119,17 +2152,17 @@ static void yy_destructor( case 278: /* func_list */ case 282: /* expression_list */ case 293: /* dnode_list */ - case 304: /* star_func_para_list */ - case 306: /* other_para_list */ - case 326: /* select_list */ - case 328: /* partition_by_clause_opt */ - case 330: /* group_by_clause_opt */ - case 332: /* select_sublist */ - case 336: /* group_by_list */ - case 338: /* order_by_clause_opt */ - case 342: /* sort_specification_list */ + case 305: /* star_func_para_list */ + case 307: /* other_para_list */ + case 327: /* select_list */ + case 329: /* partition_by_clause_opt */ + case 331: /* group_by_clause_opt */ + case 333: /* select_sublist */ + case 337: /* group_by_list */ + case 339: /* order_by_clause_opt */ + case 343: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy652)); + nodesDestroyList((yypminor->yy568)); } break; case 247: /* alter_db_option */ @@ -2143,28 +2176,28 @@ static void yy_destructor( } break; - case 309: /* compare_op */ - case 310: /* in_op */ + case 310: /* compare_op */ + case 311: /* in_op */ { } break; - case 322: /* join_type */ + case 323: /* join_type */ { } break; - case 335: /* fill_mode */ + case 336: /* fill_mode */ { } break; - case 344: /* ordering_specification_opt */ + case 345: /* ordering_specification_opt */ { } break; - case 345: /* null_ordering_opt */ + case 346: /* null_ordering_opt */ { } @@ -2742,168 +2775,171 @@ static const struct { { 295, -2 }, /* (276) signed_literal ::= TIMESTAMP NK_STRING */ { 295, -1 }, /* (277) signed_literal ::= duration_literal */ { 295, -1 }, /* (278) signed_literal ::= NULL */ - { 262, -1 }, /* (279) literal_list ::= signed_literal */ - { 262, -3 }, /* (280) literal_list ::= literal_list NK_COMMA signed_literal */ - { 240, -1 }, /* (281) db_name ::= NK_ID */ - { 265, -1 }, /* (282) table_name ::= NK_ID */ - { 258, -1 }, /* (283) column_name ::= NK_ID */ - { 275, -1 }, /* (284) function_name ::= NK_ID */ - { 296, -1 }, /* (285) table_alias ::= NK_ID */ - { 297, -1 }, /* (286) column_alias ::= NK_ID */ - { 236, -1 }, /* (287) user_name ::= NK_ID */ - { 276, -1 }, /* (288) index_name ::= NK_ID */ - { 283, -1 }, /* (289) topic_name ::= NK_ID */ - { 290, -1 }, /* (290) stream_name ::= NK_ID */ - { 298, -1 }, /* (291) expression ::= literal */ - { 298, -1 }, /* (292) expression ::= pseudo_column */ - { 298, -1 }, /* (293) expression ::= column_reference */ - { 298, -1 }, /* (294) expression ::= function_expression */ - { 298, -1 }, /* (295) expression ::= subquery */ - { 298, -3 }, /* (296) expression ::= NK_LP expression NK_RP */ - { 298, -2 }, /* (297) expression ::= NK_PLUS expression */ - { 298, -2 }, /* (298) expression ::= NK_MINUS expression */ - { 298, -3 }, /* (299) expression ::= expression NK_PLUS expression */ - { 298, -3 }, /* (300) expression ::= expression NK_MINUS expression */ - { 298, -3 }, /* (301) expression ::= expression NK_STAR expression */ - { 298, -3 }, /* (302) expression ::= expression NK_SLASH expression */ - { 298, -3 }, /* (303) expression ::= expression NK_REM expression */ - { 298, -3 }, /* (304) expression ::= column_reference NK_ARROW NK_STRING */ - { 282, -1 }, /* (305) expression_list ::= expression */ - { 282, -3 }, /* (306) expression_list ::= expression_list NK_COMMA expression */ - { 300, -1 }, /* (307) column_reference ::= column_name */ - { 300, -3 }, /* (308) column_reference ::= table_name NK_DOT column_name */ - { 299, -1 }, /* (309) pseudo_column ::= ROWTS */ - { 299, -1 }, /* (310) pseudo_column ::= TBNAME */ - { 299, -1 }, /* (311) pseudo_column ::= QSTARTTS */ - { 299, -1 }, /* (312) pseudo_column ::= QENDTS */ - { 299, -1 }, /* (313) pseudo_column ::= WSTARTTS */ - { 299, -1 }, /* (314) pseudo_column ::= WENDTS */ - { 299, -1 }, /* (315) pseudo_column ::= WDURATION */ - { 301, -4 }, /* (316) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 301, -4 }, /* (317) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 301, -6 }, /* (318) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ - { 301, -3 }, /* (319) function_expression ::= noarg_func NK_LP NK_RP */ - { 305, -1 }, /* (320) noarg_func ::= NOW */ - { 305, -1 }, /* (321) noarg_func ::= TODAY */ - { 305, -1 }, /* (322) noarg_func ::= TIMEZONE */ - { 303, -1 }, /* (323) star_func ::= COUNT */ - { 303, -1 }, /* (324) star_func ::= FIRST */ - { 303, -1 }, /* (325) star_func ::= LAST */ - { 303, -1 }, /* (326) star_func ::= LAST_ROW */ - { 304, -1 }, /* (327) star_func_para_list ::= NK_STAR */ - { 304, -1 }, /* (328) star_func_para_list ::= other_para_list */ - { 306, -1 }, /* (329) other_para_list ::= star_func_para */ - { 306, -3 }, /* (330) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 307, -1 }, /* (331) star_func_para ::= expression */ - { 307, -3 }, /* (332) star_func_para ::= table_name NK_DOT NK_STAR */ - { 308, -3 }, /* (333) predicate ::= expression compare_op expression */ - { 308, -5 }, /* (334) predicate ::= expression BETWEEN expression AND expression */ - { 308, -6 }, /* (335) predicate ::= expression NOT BETWEEN expression AND expression */ - { 308, -3 }, /* (336) predicate ::= expression IS NULL */ - { 308, -4 }, /* (337) predicate ::= expression IS NOT NULL */ - { 308, -3 }, /* (338) predicate ::= expression in_op in_predicate_value */ - { 309, -1 }, /* (339) compare_op ::= NK_LT */ - { 309, -1 }, /* (340) compare_op ::= NK_GT */ - { 309, -1 }, /* (341) compare_op ::= NK_LE */ - { 309, -1 }, /* (342) compare_op ::= NK_GE */ - { 309, -1 }, /* (343) compare_op ::= NK_NE */ - { 309, -1 }, /* (344) compare_op ::= NK_EQ */ - { 309, -1 }, /* (345) compare_op ::= LIKE */ - { 309, -2 }, /* (346) compare_op ::= NOT LIKE */ - { 309, -1 }, /* (347) compare_op ::= MATCH */ - { 309, -1 }, /* (348) compare_op ::= NMATCH */ - { 309, -1 }, /* (349) compare_op ::= CONTAINS */ - { 310, -1 }, /* (350) in_op ::= IN */ - { 310, -2 }, /* (351) in_op ::= NOT IN */ - { 311, -3 }, /* (352) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 312, -1 }, /* (353) boolean_value_expression ::= boolean_primary */ - { 312, -2 }, /* (354) boolean_value_expression ::= NOT boolean_primary */ - { 312, -3 }, /* (355) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 312, -3 }, /* (356) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 313, -1 }, /* (357) boolean_primary ::= predicate */ - { 313, -3 }, /* (358) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 314, -1 }, /* (359) common_expression ::= expression */ - { 314, -1 }, /* (360) common_expression ::= boolean_value_expression */ - { 315, -2 }, /* (361) from_clause ::= FROM table_reference_list */ - { 316, -1 }, /* (362) table_reference_list ::= table_reference */ - { 316, -3 }, /* (363) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 317, -1 }, /* (364) table_reference ::= table_primary */ - { 317, -1 }, /* (365) table_reference ::= joined_table */ - { 318, -2 }, /* (366) table_primary ::= table_name alias_opt */ - { 318, -4 }, /* (367) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 318, -2 }, /* (368) table_primary ::= subquery alias_opt */ - { 318, -1 }, /* (369) table_primary ::= parenthesized_joined_table */ - { 320, 0 }, /* (370) alias_opt ::= */ - { 320, -1 }, /* (371) alias_opt ::= table_alias */ - { 320, -2 }, /* (372) alias_opt ::= AS table_alias */ - { 321, -3 }, /* (373) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 321, -3 }, /* (374) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 319, -6 }, /* (375) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 322, 0 }, /* (376) join_type ::= */ - { 322, -1 }, /* (377) join_type ::= INNER */ - { 324, -9 }, /* (378) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 325, 0 }, /* (379) set_quantifier_opt ::= */ - { 325, -1 }, /* (380) set_quantifier_opt ::= DISTINCT */ - { 325, -1 }, /* (381) set_quantifier_opt ::= ALL */ - { 326, -1 }, /* (382) select_list ::= NK_STAR */ - { 326, -1 }, /* (383) select_list ::= select_sublist */ - { 332, -1 }, /* (384) select_sublist ::= select_item */ - { 332, -3 }, /* (385) select_sublist ::= select_sublist NK_COMMA select_item */ - { 333, -1 }, /* (386) select_item ::= common_expression */ - { 333, -2 }, /* (387) select_item ::= common_expression column_alias */ - { 333, -3 }, /* (388) select_item ::= common_expression AS column_alias */ - { 333, -3 }, /* (389) select_item ::= table_name NK_DOT NK_STAR */ - { 327, 0 }, /* (390) where_clause_opt ::= */ - { 327, -2 }, /* (391) where_clause_opt ::= WHERE search_condition */ - { 328, 0 }, /* (392) partition_by_clause_opt ::= */ - { 328, -3 }, /* (393) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 329, 0 }, /* (394) twindow_clause_opt ::= */ - { 329, -6 }, /* (395) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 329, -4 }, /* (396) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ - { 329, -6 }, /* (397) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 329, -8 }, /* (398) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 280, 0 }, /* (399) sliding_opt ::= */ - { 280, -4 }, /* (400) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 334, 0 }, /* (401) fill_opt ::= */ - { 334, -4 }, /* (402) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 334, -6 }, /* (403) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 335, -1 }, /* (404) fill_mode ::= NONE */ - { 335, -1 }, /* (405) fill_mode ::= PREV */ - { 335, -1 }, /* (406) fill_mode ::= NULL */ - { 335, -1 }, /* (407) fill_mode ::= LINEAR */ - { 335, -1 }, /* (408) fill_mode ::= NEXT */ - { 330, 0 }, /* (409) group_by_clause_opt ::= */ - { 330, -3 }, /* (410) group_by_clause_opt ::= GROUP BY group_by_list */ - { 336, -1 }, /* (411) group_by_list ::= expression */ - { 336, -3 }, /* (412) group_by_list ::= group_by_list NK_COMMA expression */ - { 331, 0 }, /* (413) having_clause_opt ::= */ - { 331, -2 }, /* (414) having_clause_opt ::= HAVING search_condition */ - { 285, -4 }, /* (415) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 337, -1 }, /* (416) query_expression_body ::= query_primary */ - { 337, -4 }, /* (417) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 337, -3 }, /* (418) query_expression_body ::= query_expression_body UNION query_expression_body */ - { 341, -1 }, /* (419) query_primary ::= query_specification */ - { 338, 0 }, /* (420) order_by_clause_opt ::= */ - { 338, -3 }, /* (421) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 339, 0 }, /* (422) slimit_clause_opt ::= */ - { 339, -2 }, /* (423) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 339, -4 }, /* (424) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 339, -4 }, /* (425) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 340, 0 }, /* (426) limit_clause_opt ::= */ - { 340, -2 }, /* (427) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 340, -4 }, /* (428) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 340, -4 }, /* (429) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 302, -3 }, /* (430) subquery ::= NK_LP query_expression NK_RP */ - { 323, -1 }, /* (431) search_condition ::= common_expression */ - { 342, -1 }, /* (432) sort_specification_list ::= sort_specification */ - { 342, -3 }, /* (433) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 343, -3 }, /* (434) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 344, 0 }, /* (435) ordering_specification_opt ::= */ - { 344, -1 }, /* (436) ordering_specification_opt ::= ASC */ - { 344, -1 }, /* (437) ordering_specification_opt ::= DESC */ - { 345, 0 }, /* (438) null_ordering_opt ::= */ - { 345, -2 }, /* (439) null_ordering_opt ::= NULLS FIRST */ - { 345, -2 }, /* (440) null_ordering_opt ::= NULLS LAST */ + { 295, -1 }, /* (279) signed_literal ::= literal_func */ + { 262, -1 }, /* (280) literal_list ::= signed_literal */ + { 262, -3 }, /* (281) literal_list ::= literal_list NK_COMMA signed_literal */ + { 240, -1 }, /* (282) db_name ::= NK_ID */ + { 265, -1 }, /* (283) table_name ::= NK_ID */ + { 258, -1 }, /* (284) column_name ::= NK_ID */ + { 275, -1 }, /* (285) function_name ::= NK_ID */ + { 297, -1 }, /* (286) table_alias ::= NK_ID */ + { 298, -1 }, /* (287) column_alias ::= NK_ID */ + { 236, -1 }, /* (288) user_name ::= NK_ID */ + { 276, -1 }, /* (289) index_name ::= NK_ID */ + { 283, -1 }, /* (290) topic_name ::= NK_ID */ + { 290, -1 }, /* (291) stream_name ::= NK_ID */ + { 299, -1 }, /* (292) expression ::= literal */ + { 299, -1 }, /* (293) expression ::= pseudo_column */ + { 299, -1 }, /* (294) expression ::= column_reference */ + { 299, -1 }, /* (295) expression ::= function_expression */ + { 299, -1 }, /* (296) expression ::= subquery */ + { 299, -3 }, /* (297) expression ::= NK_LP expression NK_RP */ + { 299, -2 }, /* (298) expression ::= NK_PLUS expression */ + { 299, -2 }, /* (299) expression ::= NK_MINUS expression */ + { 299, -3 }, /* (300) expression ::= expression NK_PLUS expression */ + { 299, -3 }, /* (301) expression ::= expression NK_MINUS expression */ + { 299, -3 }, /* (302) expression ::= expression NK_STAR expression */ + { 299, -3 }, /* (303) expression ::= expression NK_SLASH expression */ + { 299, -3 }, /* (304) expression ::= expression NK_REM expression */ + { 299, -3 }, /* (305) expression ::= column_reference NK_ARROW NK_STRING */ + { 282, -1 }, /* (306) expression_list ::= expression */ + { 282, -3 }, /* (307) expression_list ::= expression_list NK_COMMA expression */ + { 301, -1 }, /* (308) column_reference ::= column_name */ + { 301, -3 }, /* (309) column_reference ::= table_name NK_DOT column_name */ + { 300, -1 }, /* (310) pseudo_column ::= ROWTS */ + { 300, -1 }, /* (311) pseudo_column ::= TBNAME */ + { 300, -1 }, /* (312) pseudo_column ::= QSTARTTS */ + { 300, -1 }, /* (313) pseudo_column ::= QENDTS */ + { 300, -1 }, /* (314) pseudo_column ::= WSTARTTS */ + { 300, -1 }, /* (315) pseudo_column ::= WENDTS */ + { 300, -1 }, /* (316) pseudo_column ::= WDURATION */ + { 302, -4 }, /* (317) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 302, -4 }, /* (318) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 302, -6 }, /* (319) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ + { 302, -1 }, /* (320) function_expression ::= literal_func */ + { 296, -3 }, /* (321) literal_func ::= noarg_func NK_LP NK_RP */ + { 296, -1 }, /* (322) literal_func ::= NOW */ + { 306, -1 }, /* (323) noarg_func ::= NOW */ + { 306, -1 }, /* (324) noarg_func ::= TODAY */ + { 306, -1 }, /* (325) noarg_func ::= TIMEZONE */ + { 304, -1 }, /* (326) star_func ::= COUNT */ + { 304, -1 }, /* (327) star_func ::= FIRST */ + { 304, -1 }, /* (328) star_func ::= LAST */ + { 304, -1 }, /* (329) star_func ::= LAST_ROW */ + { 305, -1 }, /* (330) star_func_para_list ::= NK_STAR */ + { 305, -1 }, /* (331) star_func_para_list ::= other_para_list */ + { 307, -1 }, /* (332) other_para_list ::= star_func_para */ + { 307, -3 }, /* (333) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 308, -1 }, /* (334) star_func_para ::= expression */ + { 308, -3 }, /* (335) star_func_para ::= table_name NK_DOT NK_STAR */ + { 309, -3 }, /* (336) predicate ::= expression compare_op expression */ + { 309, -5 }, /* (337) predicate ::= expression BETWEEN expression AND expression */ + { 309, -6 }, /* (338) predicate ::= expression NOT BETWEEN expression AND expression */ + { 309, -3 }, /* (339) predicate ::= expression IS NULL */ + { 309, -4 }, /* (340) predicate ::= expression IS NOT NULL */ + { 309, -3 }, /* (341) predicate ::= expression in_op in_predicate_value */ + { 310, -1 }, /* (342) compare_op ::= NK_LT */ + { 310, -1 }, /* (343) compare_op ::= NK_GT */ + { 310, -1 }, /* (344) compare_op ::= NK_LE */ + { 310, -1 }, /* (345) compare_op ::= NK_GE */ + { 310, -1 }, /* (346) compare_op ::= NK_NE */ + { 310, -1 }, /* (347) compare_op ::= NK_EQ */ + { 310, -1 }, /* (348) compare_op ::= LIKE */ + { 310, -2 }, /* (349) compare_op ::= NOT LIKE */ + { 310, -1 }, /* (350) compare_op ::= MATCH */ + { 310, -1 }, /* (351) compare_op ::= NMATCH */ + { 310, -1 }, /* (352) compare_op ::= CONTAINS */ + { 311, -1 }, /* (353) in_op ::= IN */ + { 311, -2 }, /* (354) in_op ::= NOT IN */ + { 312, -3 }, /* (355) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 313, -1 }, /* (356) boolean_value_expression ::= boolean_primary */ + { 313, -2 }, /* (357) boolean_value_expression ::= NOT boolean_primary */ + { 313, -3 }, /* (358) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 313, -3 }, /* (359) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 314, -1 }, /* (360) boolean_primary ::= predicate */ + { 314, -3 }, /* (361) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 315, -1 }, /* (362) common_expression ::= expression */ + { 315, -1 }, /* (363) common_expression ::= boolean_value_expression */ + { 316, -2 }, /* (364) from_clause ::= FROM table_reference_list */ + { 317, -1 }, /* (365) table_reference_list ::= table_reference */ + { 317, -3 }, /* (366) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 318, -1 }, /* (367) table_reference ::= table_primary */ + { 318, -1 }, /* (368) table_reference ::= joined_table */ + { 319, -2 }, /* (369) table_primary ::= table_name alias_opt */ + { 319, -4 }, /* (370) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 319, -2 }, /* (371) table_primary ::= subquery alias_opt */ + { 319, -1 }, /* (372) table_primary ::= parenthesized_joined_table */ + { 321, 0 }, /* (373) alias_opt ::= */ + { 321, -1 }, /* (374) alias_opt ::= table_alias */ + { 321, -2 }, /* (375) alias_opt ::= AS table_alias */ + { 322, -3 }, /* (376) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 322, -3 }, /* (377) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 320, -6 }, /* (378) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 323, 0 }, /* (379) join_type ::= */ + { 323, -1 }, /* (380) join_type ::= INNER */ + { 325, -9 }, /* (381) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 326, 0 }, /* (382) set_quantifier_opt ::= */ + { 326, -1 }, /* (383) set_quantifier_opt ::= DISTINCT */ + { 326, -1 }, /* (384) set_quantifier_opt ::= ALL */ + { 327, -1 }, /* (385) select_list ::= NK_STAR */ + { 327, -1 }, /* (386) select_list ::= select_sublist */ + { 333, -1 }, /* (387) select_sublist ::= select_item */ + { 333, -3 }, /* (388) select_sublist ::= select_sublist NK_COMMA select_item */ + { 334, -1 }, /* (389) select_item ::= common_expression */ + { 334, -2 }, /* (390) select_item ::= common_expression column_alias */ + { 334, -3 }, /* (391) select_item ::= common_expression AS column_alias */ + { 334, -3 }, /* (392) select_item ::= table_name NK_DOT NK_STAR */ + { 328, 0 }, /* (393) where_clause_opt ::= */ + { 328, -2 }, /* (394) where_clause_opt ::= WHERE search_condition */ + { 329, 0 }, /* (395) partition_by_clause_opt ::= */ + { 329, -3 }, /* (396) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 330, 0 }, /* (397) twindow_clause_opt ::= */ + { 330, -6 }, /* (398) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 330, -4 }, /* (399) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ + { 330, -6 }, /* (400) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 330, -8 }, /* (401) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 280, 0 }, /* (402) sliding_opt ::= */ + { 280, -4 }, /* (403) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 335, 0 }, /* (404) fill_opt ::= */ + { 335, -4 }, /* (405) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 335, -6 }, /* (406) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 336, -1 }, /* (407) fill_mode ::= NONE */ + { 336, -1 }, /* (408) fill_mode ::= PREV */ + { 336, -1 }, /* (409) fill_mode ::= NULL */ + { 336, -1 }, /* (410) fill_mode ::= LINEAR */ + { 336, -1 }, /* (411) fill_mode ::= NEXT */ + { 331, 0 }, /* (412) group_by_clause_opt ::= */ + { 331, -3 }, /* (413) group_by_clause_opt ::= GROUP BY group_by_list */ + { 337, -1 }, /* (414) group_by_list ::= expression */ + { 337, -3 }, /* (415) group_by_list ::= group_by_list NK_COMMA expression */ + { 332, 0 }, /* (416) having_clause_opt ::= */ + { 332, -2 }, /* (417) having_clause_opt ::= HAVING search_condition */ + { 285, -4 }, /* (418) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 338, -1 }, /* (419) query_expression_body ::= query_primary */ + { 338, -4 }, /* (420) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 338, -3 }, /* (421) query_expression_body ::= query_expression_body UNION query_expression_body */ + { 342, -1 }, /* (422) query_primary ::= query_specification */ + { 339, 0 }, /* (423) order_by_clause_opt ::= */ + { 339, -3 }, /* (424) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 340, 0 }, /* (425) slimit_clause_opt ::= */ + { 340, -2 }, /* (426) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 340, -4 }, /* (427) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 340, -4 }, /* (428) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 341, 0 }, /* (429) limit_clause_opt ::= */ + { 341, -2 }, /* (430) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 341, -4 }, /* (431) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 341, -4 }, /* (432) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 303, -3 }, /* (433) subquery ::= NK_LP query_expression NK_RP */ + { 324, -1 }, /* (434) search_condition ::= common_expression */ + { 343, -1 }, /* (435) sort_specification_list ::= sort_specification */ + { 343, -3 }, /* (436) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 344, -3 }, /* (437) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 345, 0 }, /* (438) ordering_specification_opt ::= */ + { 345, -1 }, /* (439) ordering_specification_opt ::= ASC */ + { 345, -1 }, /* (440) ordering_specification_opt ::= DESC */ + { 346, 0 }, /* (441) null_ordering_opt ::= */ + { 346, -2 }, /* (442) null_ordering_opt ::= NULLS FIRST */ + { 346, -2 }, /* (443) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3040,28 +3076,28 @@ static YYACTIONTYPE yy_reduce( yy_destructor(yypParser,234,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy0); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy517, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy555, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy517, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy555, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy555); } break; case 28: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy517, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy555, NULL); } break; case 29: /* cmd ::= CREATE DNODE dnode_host_name PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy0); } break; case 30: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; case 31: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy555); } break; case 32: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -3078,25 +3114,25 @@ static YYACTIONTYPE yy_reduce( case 36: /* dnode_endpoint ::= NK_STRING */ case 37: /* dnode_host_name ::= NK_ID */ yytestcase(yyruleno==37); case 38: /* dnode_host_name ::= NK_IPTOKEN */ yytestcase(yyruleno==38); - case 281: /* db_name ::= NK_ID */ yytestcase(yyruleno==281); - case 282: /* table_name ::= NK_ID */ yytestcase(yyruleno==282); - case 283: /* column_name ::= NK_ID */ yytestcase(yyruleno==283); - case 284: /* function_name ::= NK_ID */ yytestcase(yyruleno==284); - case 285: /* table_alias ::= NK_ID */ yytestcase(yyruleno==285); - case 286: /* column_alias ::= NK_ID */ yytestcase(yyruleno==286); - case 287: /* user_name ::= NK_ID */ yytestcase(yyruleno==287); - case 288: /* index_name ::= NK_ID */ yytestcase(yyruleno==288); - case 289: /* topic_name ::= NK_ID */ yytestcase(yyruleno==289); - case 290: /* stream_name ::= NK_ID */ yytestcase(yyruleno==290); - case 320: /* noarg_func ::= NOW */ yytestcase(yyruleno==320); - case 321: /* noarg_func ::= TODAY */ yytestcase(yyruleno==321); - case 322: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==322); - case 323: /* star_func ::= COUNT */ yytestcase(yyruleno==323); - case 324: /* star_func ::= FIRST */ yytestcase(yyruleno==324); - case 325: /* star_func ::= LAST */ yytestcase(yyruleno==325); - case 326: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==326); -{ yylhsminor.yy517 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy517 = yylhsminor.yy517; + case 282: /* db_name ::= NK_ID */ yytestcase(yyruleno==282); + case 283: /* table_name ::= NK_ID */ yytestcase(yyruleno==283); + case 284: /* column_name ::= NK_ID */ yytestcase(yyruleno==284); + case 285: /* function_name ::= NK_ID */ yytestcase(yyruleno==285); + case 286: /* table_alias ::= NK_ID */ yytestcase(yyruleno==286); + case 287: /* column_alias ::= NK_ID */ yytestcase(yyruleno==287); + case 288: /* user_name ::= NK_ID */ yytestcase(yyruleno==288); + case 289: /* index_name ::= NK_ID */ yytestcase(yyruleno==289); + case 290: /* topic_name ::= NK_ID */ yytestcase(yyruleno==290); + case 291: /* stream_name ::= NK_ID */ yytestcase(yyruleno==291); + case 323: /* noarg_func ::= NOW */ yytestcase(yyruleno==323); + case 324: /* noarg_func ::= TODAY */ yytestcase(yyruleno==324); + case 325: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==325); + case 326: /* star_func ::= COUNT */ yytestcase(yyruleno==326); + case 327: /* star_func ::= FIRST */ yytestcase(yyruleno==327); + case 328: /* star_func ::= LAST */ yytestcase(yyruleno==328); + case 329: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==329); +{ yylhsminor.yy555 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy555 = yylhsminor.yy555; break; case 39: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -3129,163 +3165,163 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 49: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy673, &yymsp[-1].minor.yy517, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy617, &yymsp[-1].minor.yy555, yymsp[0].minor.yy662); } break; case 50: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy673, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy555); } break; case 51: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy555); } break; case 52: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy517, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy555, yymsp[0].minor.yy662); } break; case 53: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy673 = true; } +{ yymsp[-2].minor.yy617 = true; } break; case 54: /* not_exists_opt ::= */ case 56: /* exists_opt ::= */ yytestcase(yyruleno==56); case 229: /* analyze_opt ::= */ yytestcase(yyruleno==229); case 237: /* agg_func_opt ::= */ yytestcase(yyruleno==237); - case 379: /* set_quantifier_opt ::= */ yytestcase(yyruleno==379); -{ yymsp[1].minor.yy673 = false; } + case 382: /* set_quantifier_opt ::= */ yytestcase(yyruleno==382); +{ yymsp[1].minor.yy617 = false; } break; case 55: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy673 = true; } +{ yymsp[-1].minor.yy617 = true; } break; case 57: /* db_options ::= */ -{ yymsp[1].minor.yy456 = createDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy662 = createDatabaseOptions(pCxt); } break; case 58: /* db_options ::= db_options BLOCKS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pNumOfBlocks = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 59: /* db_options ::= db_options CACHE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pCacheBlockSize = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 60: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pCachelast = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 61: /* db_options ::= db_options COMP NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pCompressionLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 62: /* db_options ::= db_options DAYS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pDaysPerFile = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 63: /* db_options ::= db_options DAYS NK_VARIABLE */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 64: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pFsyncPeriod = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 65: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pMaxRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 66: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pMinRowsPerBlock = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 67: /* db_options ::= db_options KEEP integer_list */ case 68: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==68); -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pKeep = yymsp[0].minor.yy652; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pKeep = yymsp[0].minor.yy568; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 69: /* db_options ::= db_options PRECISION NK_STRING */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pPrecision = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 70: /* db_options ::= db_options QUORUM NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pQuorum = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 71: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pReplica = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 72: /* db_options ::= db_options TTL NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 73: /* db_options ::= db_options WAL NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pWalLevel = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 74: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pNumOfVgroups = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 75: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pSingleStable = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 76: /* db_options ::= db_options STREAM_MODE NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pStreamMode = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 77: /* db_options ::= db_options RETENTIONS retention_list */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pRetentions = yymsp[0].minor.yy652; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pRetentions = yymsp[0].minor.yy568; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 78: /* db_options ::= db_options STRICT NK_INTEGER */ -{ ((SDatabaseOptions*)yymsp[-2].minor.yy456)->pStrict = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SDatabaseOptions*)yymsp[-2].minor.yy662)->pStrict = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 79: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy456 = createDatabaseOptions(pCxt); yylhsminor.yy456 = setDatabaseAlterOption(pCxt, yylhsminor.yy456, &yymsp[0].minor.yy145); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createDatabaseOptions(pCxt); yylhsminor.yy662 = setDatabaseAlterOption(pCxt, yylhsminor.yy662, &yymsp[0].minor.yy475); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 80: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy456 = setDatabaseAlterOption(pCxt, yymsp[-1].minor.yy456, &yymsp[0].minor.yy145); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = setDatabaseAlterOption(pCxt, yymsp[-1].minor.yy662, &yymsp[0].minor.yy475); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 81: /* alter_db_option ::= BLOCKS NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = DB_OPTION_BLOCKS; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 82: /* alter_db_option ::= FSYNC NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 83: /* alter_db_option ::= KEEP integer_list */ case 84: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==84); -{ yymsp[-1].minor.yy145.type = DB_OPTION_KEEP; yymsp[-1].minor.yy145.pList = yymsp[0].minor.yy652; } +{ yymsp[-1].minor.yy475.type = DB_OPTION_KEEP; yymsp[-1].minor.yy475.pList = yymsp[0].minor.yy568; } break; case 85: /* alter_db_option ::= WAL NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = DB_OPTION_WAL; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = DB_OPTION_WAL; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 86: /* alter_db_option ::= QUORUM NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = DB_OPTION_QUORUM; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 87: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 88: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 89: /* alter_db_option ::= STRICT NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = DB_OPTION_STRICT; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = DB_OPTION_STRICT; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 90: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy652 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy652 = yylhsminor.yy652; +{ yylhsminor.yy568 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy568 = yylhsminor.yy568; break; case 91: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ case 255: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==255); -{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy652 = yylhsminor.yy652; +{ yylhsminor.yy568 = addNodeToList(pCxt, yymsp[-2].minor.yy568, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy568 = yylhsminor.yy568; break; case 92: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy652 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy652 = yylhsminor.yy652; +{ yylhsminor.yy568 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy568 = yylhsminor.yy568; break; case 93: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy652 = yylhsminor.yy652; +{ yylhsminor.yy568 = addNodeToList(pCxt, yymsp[-2].minor.yy568, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy568 = yylhsminor.yy568; break; case 94: /* retention_list ::= retention */ case 114: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==114); @@ -3294,255 +3330,255 @@ static YYACTIONTYPE yy_reduce( case 169: /* col_name_list ::= col_name */ yytestcase(yyruleno==169); case 206: /* func_name_list ::= func_name */ yytestcase(yyruleno==206); case 215: /* func_list ::= func */ yytestcase(yyruleno==215); - case 279: /* literal_list ::= signed_literal */ yytestcase(yyruleno==279); - case 329: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==329); - case 384: /* select_sublist ::= select_item */ yytestcase(yyruleno==384); - case 432: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==432); -{ yylhsminor.yy652 = createNodeList(pCxt, yymsp[0].minor.yy456); } - yymsp[0].minor.yy652 = yylhsminor.yy652; + case 280: /* literal_list ::= signed_literal */ yytestcase(yyruleno==280); + case 332: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==332); + case 387: /* select_sublist ::= select_item */ yytestcase(yyruleno==387); + case 435: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==435); +{ yylhsminor.yy568 = createNodeList(pCxt, yymsp[0].minor.yy662); } + yymsp[0].minor.yy568 = yylhsminor.yy568; break; case 95: /* retention_list ::= retention_list NK_COMMA retention */ case 125: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==125); case 170: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==170); case 207: /* func_name_list ::= func_name_list NK_COMMA func_name */ yytestcase(yyruleno==207); case 216: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==216); - case 280: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==280); - case 330: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==330); - case 385: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==385); - case 433: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==433); -{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, yymsp[0].minor.yy456); } - yymsp[-2].minor.yy652 = yylhsminor.yy652; + case 281: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==281); + case 333: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==333); + case 388: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==388); + case 436: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==436); +{ yylhsminor.yy568 = addNodeToList(pCxt, yymsp[-2].minor.yy568, yymsp[0].minor.yy662); } + yymsp[-2].minor.yy568 = yylhsminor.yy568; break; case 96: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy456 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 97: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ case 99: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==99); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy673, yymsp[-5].minor.yy456, yymsp[-3].minor.yy652, yymsp[-1].minor.yy652, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy617, yymsp[-5].minor.yy662, yymsp[-3].minor.yy568, yymsp[-1].minor.yy568, yymsp[0].minor.yy662); } break; case 98: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy652); } +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy568); } break; case 100: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy652); } +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy568); } break; case 101: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy673, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy617, yymsp[0].minor.yy662); } break; case 102: /* cmd ::= ALTER TABLE alter_table_clause */ case 103: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==103); case 257: /* cmd ::= query_expression */ yytestcase(yyruleno==257); -{ pCxt->pRootNode = yymsp[0].minor.yy456; } +{ pCxt->pRootNode = yymsp[0].minor.yy662; } break; case 104: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy456 = createAlterTableOption(pCxt, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableOption(pCxt, yymsp[-1].minor.yy662, yymsp[0].minor.yy662); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 105: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy662, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy555, yymsp[0].minor.yy156); } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 106: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy456 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy456, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy517); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy662, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy555); } + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; case 107: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy662, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy555, yymsp[0].minor.yy156); } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 108: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy456 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy662, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy555, &yymsp[0].minor.yy555); } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 109: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy662, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy555, yymsp[0].minor.yy156); } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 110: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy456 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy456, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy517); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy662, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy555); } + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; case 111: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy456 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy662, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy555, yymsp[0].minor.yy156); } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 112: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy456 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy456, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy662, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy555, &yymsp[0].minor.yy555); } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 113: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ literal */ -{ yylhsminor.yy456 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy456, &yymsp[-2].minor.yy517, yymsp[0].minor.yy456); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy662, &yymsp[-2].minor.yy555, yymsp[0].minor.yy662); } + yymsp[-5].minor.yy662 = yylhsminor.yy662; break; case 115: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ case 118: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==118); -{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-1].minor.yy652, yymsp[0].minor.yy456); } - yymsp[-1].minor.yy652 = yylhsminor.yy652; +{ yylhsminor.yy568 = addNodeToList(pCxt, yymsp[-1].minor.yy568, yymsp[0].minor.yy662); } + yymsp[-1].minor.yy568 = yylhsminor.yy568; break; case 116: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP */ -{ yylhsminor.yy456 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy673, yymsp[-7].minor.yy456, yymsp[-5].minor.yy456, yymsp[-4].minor.yy652, yymsp[-1].minor.yy652); } - yymsp[-8].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createCreateSubTableClause(pCxt, yymsp[-8].minor.yy617, yymsp[-7].minor.yy662, yymsp[-5].minor.yy662, yymsp[-4].minor.yy568, yymsp[-1].minor.yy568); } + yymsp[-8].minor.yy662 = yylhsminor.yy662; break; case 119: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy456 = createDropTableClause(pCxt, yymsp[-1].minor.yy673, yymsp[0].minor.yy456); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createDropTableClause(pCxt, yymsp[-1].minor.yy617, yymsp[0].minor.yy662); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 120: /* specific_tags_opt ::= */ case 151: /* tags_def_opt ::= */ yytestcase(yyruleno==151); - case 392: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==392); - case 409: /* group_by_clause_opt ::= */ yytestcase(yyruleno==409); - case 420: /* order_by_clause_opt ::= */ yytestcase(yyruleno==420); -{ yymsp[1].minor.yy652 = NULL; } + case 395: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==395); + case 412: /* group_by_clause_opt ::= */ yytestcase(yyruleno==412); + case 423: /* order_by_clause_opt ::= */ yytestcase(yyruleno==423); +{ yymsp[1].minor.yy568 = NULL; } break; case 121: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy652 = yymsp[-1].minor.yy652; } +{ yymsp[-2].minor.yy568 = yymsp[-1].minor.yy568; } break; case 122: /* full_table_name ::= table_name */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy517, NULL); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy555, NULL); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 123: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517, NULL); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRealTableNode(pCxt, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy555, NULL); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 126: /* column_def ::= column_name type_name */ -{ yylhsminor.yy456 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy517, yymsp[0].minor.yy380, NULL); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy555, yymsp[0].minor.yy156, NULL); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 127: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy456 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy517, yymsp[-2].minor.yy380, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy555, yymsp[-2].minor.yy156, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; case 128: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_BOOL); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_BOOL); } break; case 129: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_TINYINT); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; case 130: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_SMALLINT); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; case 131: /* type_name ::= INT */ case 132: /* type_name ::= INTEGER */ yytestcase(yyruleno==132); -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_INT); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_INT); } break; case 133: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_BIGINT); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; case 134: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_FLOAT); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; case 135: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_DOUBLE); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; case 136: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy156 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; case 137: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; case 138: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy156 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; case 139: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_UTINYINT); } +{ yymsp[-1].minor.yy156 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; case 140: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_USMALLINT); } +{ yymsp[-1].minor.yy156 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; case 141: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_UINT); } +{ yymsp[-1].minor.yy156 = createDataType(TSDB_DATA_TYPE_UINT); } break; case 142: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy380 = createDataType(TSDB_DATA_TYPE_UBIGINT); } +{ yymsp[-1].minor.yy156 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; case 143: /* type_name ::= JSON */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_JSON); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_JSON); } break; case 144: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy156 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; case 145: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; case 146: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_BLOB); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_BLOB); } break; case 147: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy380 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } +{ yymsp[-3].minor.yy156 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; case 148: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy380 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[0].minor.yy156 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 149: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy380 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-3].minor.yy156 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 150: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy380 = createDataType(TSDB_DATA_TYPE_DECIMAL); } +{ yymsp[-5].minor.yy156 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; case 152: /* tags_def_opt ::= tags_def */ - case 328: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==328); - case 383: /* select_list ::= select_sublist */ yytestcase(yyruleno==383); -{ yylhsminor.yy652 = yymsp[0].minor.yy652; } - yymsp[0].minor.yy652 = yylhsminor.yy652; + case 331: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==331); + case 386: /* select_list ::= select_sublist */ yytestcase(yyruleno==386); +{ yylhsminor.yy568 = yymsp[0].minor.yy568; } + yymsp[0].minor.yy568 = yylhsminor.yy568; break; case 153: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy652 = yymsp[-1].minor.yy652; } +{ yymsp[-3].minor.yy568 = yymsp[-1].minor.yy568; } break; case 154: /* table_options ::= */ -{ yymsp[1].minor.yy456 = createTableOptions(pCxt); } +{ yymsp[1].minor.yy662 = createTableOptions(pCxt); } break; case 155: /* table_options ::= table_options COMMENT NK_STRING */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STableOptions*)yymsp[-2].minor.yy662)->pComments = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 156: /* table_options ::= table_options KEEP integer_list */ case 157: /* table_options ::= table_options KEEP variable_list */ yytestcase(yyruleno==157); -{ ((STableOptions*)yymsp[-2].minor.yy456)->pKeep = yymsp[0].minor.yy652; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STableOptions*)yymsp[-2].minor.yy662)->pKeep = yymsp[0].minor.yy568; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 158: /* table_options ::= table_options TTL NK_INTEGER */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STableOptions*)yymsp[-2].minor.yy662)->pTtl = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 159: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ ((STableOptions*)yymsp[-4].minor.yy456)->pSma = yymsp[-1].minor.yy652; yylhsminor.yy456 = yymsp[-4].minor.yy456; } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ ((STableOptions*)yymsp[-4].minor.yy662)->pSma = yymsp[-1].minor.yy568; yylhsminor.yy662 = yymsp[-4].minor.yy662; } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 160: /* table_options ::= table_options ROLLUP NK_LP func_name_list NK_RP */ -{ ((STableOptions*)yymsp[-4].minor.yy456)->pFuncs = yymsp[-1].minor.yy652; yylhsminor.yy456 = yymsp[-4].minor.yy456; } - yymsp[-4].minor.yy456 = yylhsminor.yy456; +{ ((STableOptions*)yymsp[-4].minor.yy662)->pFuncs = yymsp[-1].minor.yy568; yylhsminor.yy662 = yymsp[-4].minor.yy662; } + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; case 161: /* table_options ::= table_options FILE_FACTOR NK_FLOAT */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STableOptions*)yymsp[-2].minor.yy662)->pFilesFactor = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 162: /* table_options ::= table_options DELAY NK_INTEGER */ -{ ((STableOptions*)yymsp[-2].minor.yy456)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STableOptions*)yymsp[-2].minor.yy662)->pDelay = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 163: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy456 = createTableOptions(pCxt); yylhsminor.yy456 = setTableAlterOption(pCxt, yylhsminor.yy456, &yymsp[0].minor.yy145); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createTableOptions(pCxt); yylhsminor.yy662 = setTableAlterOption(pCxt, yylhsminor.yy662, &yymsp[0].minor.yy475); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 164: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy456 = setTableAlterOption(pCxt, yymsp[-1].minor.yy456, &yymsp[0].minor.yy145); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = setTableAlterOption(pCxt, yymsp[-1].minor.yy662, &yymsp[0].minor.yy475); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 165: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy145.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; case 166: /* alter_table_option ::= KEEP integer_list */ case 167: /* alter_table_option ::= KEEP variable_list */ yytestcase(yyruleno==167); -{ yymsp[-1].minor.yy145.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy145.pList = yymsp[0].minor.yy652; } +{ yymsp[-1].minor.yy475.type = TABLE_OPTION_KEEP; yymsp[-1].minor.yy475.pList = yymsp[0].minor.yy568; } break; case 168: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy145.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy145.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy475.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy475.pVal = (SValueNode*)createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 171: /* col_name ::= column_name */ -{ yylhsminor.yy456 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy517); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy555); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 172: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT, NULL, NULL); } @@ -3554,13 +3590,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT, NULL, NULL); } break; case 175: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy662, yymsp[0].minor.yy662); } break; case 176: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy662, yymsp[0].minor.yy662); } break; case 177: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy456, NULL); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy662, NULL); } break; case 178: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT, NULL, NULL); } @@ -3575,7 +3611,7 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT, NULL, NULL); } break; case 182: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[-1].minor.yy662, yymsp[0].minor.yy662); } break; case 183: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT, NULL, NULL); } @@ -3594,13 +3630,13 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT, NULL, NULL); } break; case 189: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy555); } break; case 190: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy662); } break; case 191: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy662); } break; case 192: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT, NULL, NULL); } @@ -3625,148 +3661,148 @@ static YYACTIONTYPE yy_reduce( break; case 199: /* db_name_cond_opt ::= */ case 204: /* from_db_opt ::= */ yytestcase(yyruleno==204); -{ yymsp[1].minor.yy456 = createDefaultDatabaseCondValue(pCxt); } +{ yymsp[1].minor.yy662 = createDefaultDatabaseCondValue(pCxt); } break; case 200: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy517); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy555); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 201: /* like_pattern_opt ::= */ case 212: /* index_options ::= */ yytestcase(yyruleno==212); case 243: /* into_opt ::= */ yytestcase(yyruleno==243); - case 390: /* where_clause_opt ::= */ yytestcase(yyruleno==390); - case 394: /* twindow_clause_opt ::= */ yytestcase(yyruleno==394); - case 399: /* sliding_opt ::= */ yytestcase(yyruleno==399); - case 401: /* fill_opt ::= */ yytestcase(yyruleno==401); - case 413: /* having_clause_opt ::= */ yytestcase(yyruleno==413); - case 422: /* slimit_clause_opt ::= */ yytestcase(yyruleno==422); - case 426: /* limit_clause_opt ::= */ yytestcase(yyruleno==426); -{ yymsp[1].minor.yy456 = NULL; } + case 393: /* where_clause_opt ::= */ yytestcase(yyruleno==393); + case 397: /* twindow_clause_opt ::= */ yytestcase(yyruleno==397); + case 402: /* sliding_opt ::= */ yytestcase(yyruleno==402); + case 404: /* fill_opt ::= */ yytestcase(yyruleno==404); + case 416: /* having_clause_opt ::= */ yytestcase(yyruleno==416); + case 425: /* slimit_clause_opt ::= */ yytestcase(yyruleno==425); + case 429: /* limit_clause_opt ::= */ yytestcase(yyruleno==429); +{ yymsp[1].minor.yy662 = NULL; } break; case 202: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; case 203: /* table_name_cond ::= table_name */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy517); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy555); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 205: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy517); } +{ yymsp[-1].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy555); } break; case 208: /* func_name ::= function_name */ -{ yylhsminor.yy456 = createFunctionNode(pCxt, &yymsp[0].minor.yy517, NULL); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createFunctionNode(pCxt, &yymsp[0].minor.yy555, NULL); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 209: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy673, &yymsp[-3].minor.yy517, &yymsp[-1].minor.yy517, NULL, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy617, &yymsp[-3].minor.yy555, &yymsp[-1].minor.yy555, NULL, yymsp[0].minor.yy662); } break; case 210: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy673, &yymsp[-5].minor.yy517, &yymsp[-3].minor.yy517, yymsp[-1].minor.yy652, NULL); } +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy617, &yymsp[-5].minor.yy555, &yymsp[-3].minor.yy555, yymsp[-1].minor.yy568, NULL); } break; case 211: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy673, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy617, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy555); } break; case 213: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ -{ yymsp[-8].minor.yy456 = createIndexOption(pCxt, yymsp[-6].minor.yy652, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), NULL, yymsp[0].minor.yy456); } +{ yymsp[-8].minor.yy662 = createIndexOption(pCxt, yymsp[-6].minor.yy568, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), NULL, yymsp[0].minor.yy662); } break; case 214: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ -{ yymsp[-10].minor.yy456 = createIndexOption(pCxt, yymsp[-8].minor.yy652, releaseRawExprNode(pCxt, yymsp[-4].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), yymsp[0].minor.yy456); } +{ yymsp[-10].minor.yy662 = createIndexOption(pCxt, yymsp[-8].minor.yy568, releaseRawExprNode(pCxt, yymsp[-4].minor.yy662), releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), yymsp[0].minor.yy662); } break; case 217: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy456 = createFunctionNode(pCxt, &yymsp[-3].minor.yy517, yymsp[-1].minor.yy652); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createFunctionNode(pCxt, &yymsp[-3].minor.yy555, yymsp[-1].minor.yy568); } + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; case 218: /* cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy673, &yymsp[-3].minor.yy517, yymsp[0].minor.yy456, NULL, yymsp[-2].minor.yy456); } +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy617, &yymsp[-3].minor.yy555, yymsp[0].minor.yy662, NULL, yymsp[-2].minor.yy662); } break; case 219: /* cmd ::= CREATE TOPIC not_exists_opt topic_name topic_options AS db_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy673, &yymsp[-3].minor.yy517, NULL, &yymsp[0].minor.yy517, yymsp[-2].minor.yy456); } +{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy617, &yymsp[-3].minor.yy555, NULL, &yymsp[0].minor.yy555, yymsp[-2].minor.yy662); } break; case 220: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy673, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy555); } break; case 221: /* topic_options ::= */ -{ yymsp[1].minor.yy456 = createTopicOptions(pCxt); } +{ yymsp[1].minor.yy662 = createTopicOptions(pCxt); } break; case 222: /* topic_options ::= topic_options WITH TABLE */ -{ ((STopicOptions*)yymsp[-2].minor.yy456)->withTable = true; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STopicOptions*)yymsp[-2].minor.yy662)->withTable = true; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 223: /* topic_options ::= topic_options WITH SCHEMA */ -{ ((STopicOptions*)yymsp[-2].minor.yy456)->withSchema = true; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STopicOptions*)yymsp[-2].minor.yy662)->withSchema = true; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 224: /* topic_options ::= topic_options WITH TAG */ -{ ((STopicOptions*)yymsp[-2].minor.yy456)->withTag = true; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((STopicOptions*)yymsp[-2].minor.yy662)->withTag = true; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 225: /* cmd ::= DESC full_table_name */ case 226: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==226); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy662); } break; case 227: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; case 228: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy673, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy617, yymsp[-1].minor.yy662, yymsp[0].minor.yy662); } break; case 230: /* analyze_opt ::= ANALYZE */ case 238: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==238); - case 380: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==380); -{ yymsp[0].minor.yy673 = true; } + case 383: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==383); +{ yymsp[0].minor.yy617 = true; } break; case 231: /* explain_options ::= */ -{ yymsp[1].minor.yy456 = createDefaultExplainOptions(pCxt); } +{ yymsp[1].minor.yy662 = createDefaultExplainOptions(pCxt); } break; case 232: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy456 = setExplainVerbose(pCxt, yymsp[-2].minor.yy456, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = setExplainVerbose(pCxt, yymsp[-2].minor.yy662, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 233: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy456 = setExplainRatio(pCxt, yymsp[-2].minor.yy456, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = setExplainRatio(pCxt, yymsp[-2].minor.yy662, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 234: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ -{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy652); } +{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy568); } break; case 235: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy673, yymsp[-8].minor.yy673, &yymsp[-5].minor.yy517, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy380, yymsp[0].minor.yy376); } +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy617, yymsp[-8].minor.yy617, &yymsp[-5].minor.yy555, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy156, yymsp[0].minor.yy610); } break; case 236: /* cmd ::= DROP FUNCTION function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, &yymsp[0].minor.yy555); } break; case 239: /* bufsize_opt ::= */ -{ yymsp[1].minor.yy376 = 0; } +{ yymsp[1].minor.yy610 = 0; } break; case 240: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy376 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy610 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; case 241: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy673, &yymsp[-4].minor.yy517, yymsp[-2].minor.yy456, yymsp[-3].minor.yy456, yymsp[0].minor.yy456); } +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy617, &yymsp[-4].minor.yy555, yymsp[-2].minor.yy662, yymsp[-3].minor.yy662, yymsp[0].minor.yy662); } break; case 242: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy673, &yymsp[0].minor.yy517); } +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy617, &yymsp[0].minor.yy555); } break; case 244: /* into_opt ::= INTO full_table_name */ - case 361: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==361); - case 391: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==391); - case 414: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==414); -{ yymsp[-1].minor.yy456 = yymsp[0].minor.yy456; } + case 364: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==364); + case 394: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==394); + case 417: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==417); +{ yymsp[-1].minor.yy662 = yymsp[0].minor.yy662; } break; case 245: /* stream_options ::= */ -{ yymsp[1].minor.yy456 = createStreamOptions(pCxt); } +{ yymsp[1].minor.yy662 = createStreamOptions(pCxt); } break; case 246: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy456)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SStreamOptions*)yymsp[-2].minor.yy662)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 247: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy456)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SStreamOptions*)yymsp[-2].minor.yy662)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 248: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy456)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy456); yylhsminor.yy456 = yymsp[-2].minor.yy456; } - yymsp[-2].minor.yy456 = yylhsminor.yy456; +{ ((SStreamOptions*)yymsp[-2].minor.yy662)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy662); yylhsminor.yy662 = yymsp[-2].minor.yy662; } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; case 249: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } @@ -3778,505 +3814,508 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 252: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy652); } +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy568); } break; case 253: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; case 254: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy652 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } +{ yymsp[-1].minor.yy568 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } break; case 256: /* cmd ::= SYNCDB db_name REPLICA */ -{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy517); } +{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy555); } break; case 258: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 259: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 260: /* literal ::= NK_STRING */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 261: /* literal ::= NK_BOOL */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 262: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 263: /* literal ::= duration_literal */ case 273: /* signed_literal ::= signed */ yytestcase(yyruleno==273); - case 291: /* expression ::= literal */ yytestcase(yyruleno==291); - case 292: /* expression ::= pseudo_column */ yytestcase(yyruleno==292); - case 293: /* expression ::= column_reference */ yytestcase(yyruleno==293); - case 294: /* expression ::= function_expression */ yytestcase(yyruleno==294); - case 295: /* expression ::= subquery */ yytestcase(yyruleno==295); - case 353: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==353); - case 357: /* boolean_primary ::= predicate */ yytestcase(yyruleno==357); - case 359: /* common_expression ::= expression */ yytestcase(yyruleno==359); - case 360: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==360); - case 362: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==362); - case 364: /* table_reference ::= table_primary */ yytestcase(yyruleno==364); - case 365: /* table_reference ::= joined_table */ yytestcase(yyruleno==365); - case 369: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==369); - case 416: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==416); - case 419: /* query_primary ::= query_specification */ yytestcase(yyruleno==419); -{ yylhsminor.yy456 = yymsp[0].minor.yy456; } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 292: /* expression ::= literal */ yytestcase(yyruleno==292); + case 293: /* expression ::= pseudo_column */ yytestcase(yyruleno==293); + case 294: /* expression ::= column_reference */ yytestcase(yyruleno==294); + case 295: /* expression ::= function_expression */ yytestcase(yyruleno==295); + case 296: /* expression ::= subquery */ yytestcase(yyruleno==296); + case 320: /* function_expression ::= literal_func */ yytestcase(yyruleno==320); + case 356: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==356); + case 360: /* boolean_primary ::= predicate */ yytestcase(yyruleno==360); + case 362: /* common_expression ::= expression */ yytestcase(yyruleno==362); + case 363: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==363); + case 365: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==365); + case 367: /* table_reference ::= table_primary */ yytestcase(yyruleno==367); + case 368: /* table_reference ::= joined_table */ yytestcase(yyruleno==368); + case 372: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==372); + case 419: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==419); + case 422: /* query_primary ::= query_specification */ yytestcase(yyruleno==422); +{ yylhsminor.yy662 = yymsp[0].minor.yy662; } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 264: /* literal ::= NULL */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 265: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 266: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 267: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 268: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } break; case 269: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 270: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 271: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; case 272: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; case 274: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 275: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy456 = yylhsminor.yy456; +{ yylhsminor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 276: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } +{ yymsp[-1].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; case 277: /* signed_literal ::= duration_literal */ - case 331: /* star_func_para ::= expression */ yytestcase(yyruleno==331); - case 386: /* select_item ::= common_expression */ yytestcase(yyruleno==386); - case 431: /* search_condition ::= common_expression */ yytestcase(yyruleno==431); -{ yylhsminor.yy456 = releaseRawExprNode(pCxt, yymsp[0].minor.yy456); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 279: /* signed_literal ::= literal_func */ yytestcase(yyruleno==279); + case 334: /* star_func_para ::= expression */ yytestcase(yyruleno==334); + case 389: /* select_item ::= common_expression */ yytestcase(yyruleno==389); + case 434: /* search_condition ::= common_expression */ yytestcase(yyruleno==434); +{ yylhsminor.yy662 = releaseRawExprNode(pCxt, yymsp[0].minor.yy662); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; case 278: /* signed_literal ::= NULL */ -{ yymsp[0].minor.yy456 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } +{ yymsp[0].minor.yy662 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, NULL); } break; - case 296: /* expression ::= NK_LP expression NK_RP */ - case 358: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==358); -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 297: /* expression ::= NK_LP expression NK_RP */ + case 361: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==361); +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy662)); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 297: /* expression ::= NK_PLUS expression */ + case 298: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy662)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; - case 298: /* expression ::= NK_MINUS expression */ + case 299: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy456), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy662), NULL)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; - case 299: /* expression ::= expression NK_PLUS expression */ + case 300: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 300: /* expression ::= expression NK_MINUS expression */ + case 301: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 301: /* expression ::= expression NK_STAR expression */ + case 302: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 302: /* expression ::= expression NK_SLASH expression */ + case 303: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 303: /* expression ::= expression NK_REM expression */ + case 304: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 304: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 305: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 305: /* expression_list ::= expression */ -{ yylhsminor.yy652 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); } - yymsp[0].minor.yy652 = yylhsminor.yy652; + case 306: /* expression_list ::= expression */ +{ yylhsminor.yy568 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy662)); } + yymsp[0].minor.yy568 = yylhsminor.yy568; break; - case 306: /* expression_list ::= expression_list NK_COMMA expression */ -{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, releaseRawExprNode(pCxt, yymsp[0].minor.yy456)); } - yymsp[-2].minor.yy652 = yylhsminor.yy652; + case 307: /* expression_list ::= expression_list NK_COMMA expression */ +{ yylhsminor.yy568 = addNodeToList(pCxt, yymsp[-2].minor.yy568, releaseRawExprNode(pCxt, yymsp[0].minor.yy662)); } + yymsp[-2].minor.yy568 = yylhsminor.yy568; break; - case 307: /* column_reference ::= column_name */ -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy517, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy517)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 308: /* column_reference ::= column_name */ +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy555, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy555)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; - case 308: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517, createColumnNode(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy517)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 309: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy555, createColumnNode(pCxt, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy555)); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 309: /* pseudo_column ::= ROWTS */ - case 310: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==310); - case 311: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==311); - case 312: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==312); - case 313: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==313); - case 314: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==314); - case 315: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==315); -{ yylhsminor.yy456 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy456 = yylhsminor.yy456; + case 310: /* pseudo_column ::= ROWTS */ + case 311: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==311); + case 312: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==312); + case 313: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==313); + case 314: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==314); + case 315: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==315); + case 316: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==316); + case 322: /* literal_func ::= NOW */ yytestcase(yyruleno==322); +{ yylhsminor.yy662 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy662 = yylhsminor.yy662; break; - case 316: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 317: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==317); -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy517, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy517, yymsp[-1].minor.yy652)); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 317: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 318: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==318); +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy555, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy555, yymsp[-1].minor.yy568)); } + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; - case 318: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), yymsp[-1].minor.yy380)); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; + case 319: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy662), yymsp[-1].minor.yy156)); } + yymsp[-5].minor.yy662 = yylhsminor.yy662; break; - case 319: /* function_expression ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0, createFunctionNodeNoArg(pCxt, &yymsp[-2].minor.yy517)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 321: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy555, NULL)); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 327: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy652 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy652 = yylhsminor.yy652; + case 330: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy568 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy568 = yylhsminor.yy568; break; - case 332: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 389: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==389); -{ yylhsminor.yy456 = createColumnNode(pCxt, &yymsp[-2].minor.yy517, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 335: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 392: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==392); +{ yylhsminor.yy662 = createColumnNode(pCxt, &yymsp[-2].minor.yy555, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 333: /* predicate ::= expression compare_op expression */ - case 338: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==338); + case 336: /* predicate ::= expression compare_op expression */ + case 341: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==341); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy348, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy304, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 334: /* predicate ::= expression BETWEEN expression AND expression */ + case 337: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy662), releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-4].minor.yy456 = yylhsminor.yy456; + yymsp[-4].minor.yy662 = yylhsminor.yy662; break; - case 335: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 338: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy662), releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; + yymsp[-5].minor.yy662 = yylhsminor.yy662; break; - case 336: /* predicate ::= expression IS NULL */ + case 339: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), NULL)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 337: /* predicate ::= expression IS NOT NULL */ + case 340: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy662), NULL)); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; - case 339: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy348 = OP_TYPE_LOWER_THAN; } + case 342: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy304 = OP_TYPE_LOWER_THAN; } break; - case 340: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy348 = OP_TYPE_GREATER_THAN; } + case 343: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy304 = OP_TYPE_GREATER_THAN; } break; - case 341: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy348 = OP_TYPE_LOWER_EQUAL; } + case 344: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy304 = OP_TYPE_LOWER_EQUAL; } break; - case 342: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy348 = OP_TYPE_GREATER_EQUAL; } + case 345: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy304 = OP_TYPE_GREATER_EQUAL; } break; - case 343: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy348 = OP_TYPE_NOT_EQUAL; } + case 346: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy304 = OP_TYPE_NOT_EQUAL; } break; - case 344: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy348 = OP_TYPE_EQUAL; } + case 347: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy304 = OP_TYPE_EQUAL; } break; - case 345: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy348 = OP_TYPE_LIKE; } + case 348: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy304 = OP_TYPE_LIKE; } break; - case 346: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy348 = OP_TYPE_NOT_LIKE; } + case 349: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy304 = OP_TYPE_NOT_LIKE; } break; - case 347: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy348 = OP_TYPE_MATCH; } + case 350: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy304 = OP_TYPE_MATCH; } break; - case 348: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy348 = OP_TYPE_NMATCH; } + case 351: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy304 = OP_TYPE_NMATCH; } break; - case 349: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy348 = OP_TYPE_JSON_CONTAINS; } + case 352: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy304 = OP_TYPE_JSON_CONTAINS; } break; - case 350: /* in_op ::= IN */ -{ yymsp[0].minor.yy348 = OP_TYPE_IN; } + case 353: /* in_op ::= IN */ +{ yymsp[0].minor.yy304 = OP_TYPE_IN; } break; - case 351: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy348 = OP_TYPE_NOT_IN; } + case 354: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy304 = OP_TYPE_NOT_IN; } break; - case 352: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy652)); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 355: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy568)); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 354: /* boolean_value_expression ::= NOT boolean_primary */ + case 357: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy456), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy662), NULL)); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; - case 355: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 358: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 356: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 359: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy456); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy456); - yylhsminor.yy456 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy662); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy662); + yylhsminor.yy662 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 363: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy456 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy456, yymsp[0].minor.yy456, NULL); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 366: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy662 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy662, yymsp[0].minor.yy662, NULL); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 366: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 369: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy662 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy555, &yymsp[0].minor.yy555); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; - case 367: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy456 = createRealTableNode(pCxt, &yymsp[-3].minor.yy517, &yymsp[-1].minor.yy517, &yymsp[0].minor.yy517); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 370: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy662 = createRealTableNode(pCxt, &yymsp[-3].minor.yy555, &yymsp[-1].minor.yy555, &yymsp[0].minor.yy555); } + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; - case 368: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy456 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456), &yymsp[0].minor.yy517); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 371: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy662 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy662), &yymsp[0].minor.yy555); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; - case 370: /* alias_opt ::= */ -{ yymsp[1].minor.yy517 = nil_token; } + case 373: /* alias_opt ::= */ +{ yymsp[1].minor.yy555 = nil_token; } break; - case 371: /* alias_opt ::= table_alias */ -{ yylhsminor.yy517 = yymsp[0].minor.yy517; } - yymsp[0].minor.yy517 = yylhsminor.yy517; + case 374: /* alias_opt ::= table_alias */ +{ yylhsminor.yy555 = yymsp[0].minor.yy555; } + yymsp[0].minor.yy555 = yylhsminor.yy555; break; - case 372: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy517 = yymsp[0].minor.yy517; } + case 375: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy555 = yymsp[0].minor.yy555; } break; - case 373: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 374: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==374); -{ yymsp[-2].minor.yy456 = yymsp[-1].minor.yy456; } + case 376: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 377: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==377); +{ yymsp[-2].minor.yy662 = yymsp[-1].minor.yy662; } break; - case 375: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy456 = createJoinTableNode(pCxt, yymsp[-4].minor.yy684, yymsp[-5].minor.yy456, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } - yymsp[-5].minor.yy456 = yylhsminor.yy456; + case 378: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy662 = createJoinTableNode(pCxt, yymsp[-4].minor.yy84, yymsp[-5].minor.yy662, yymsp[-2].minor.yy662, yymsp[0].minor.yy662); } + yymsp[-5].minor.yy662 = yylhsminor.yy662; break; - case 376: /* join_type ::= */ -{ yymsp[1].minor.yy684 = JOIN_TYPE_INNER; } + case 379: /* join_type ::= */ +{ yymsp[1].minor.yy84 = JOIN_TYPE_INNER; } break; - case 377: /* join_type ::= INNER */ -{ yymsp[0].minor.yy684 = JOIN_TYPE_INNER; } + case 380: /* join_type ::= INNER */ +{ yymsp[0].minor.yy84 = JOIN_TYPE_INNER; } break; - case 378: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 381: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-8].minor.yy456 = createSelectStmt(pCxt, yymsp[-7].minor.yy673, yymsp[-6].minor.yy652, yymsp[-5].minor.yy456); - yymsp[-8].minor.yy456 = addWhereClause(pCxt, yymsp[-8].minor.yy456, yymsp[-4].minor.yy456); - yymsp[-8].minor.yy456 = addPartitionByClause(pCxt, yymsp[-8].minor.yy456, yymsp[-3].minor.yy652); - yymsp[-8].minor.yy456 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy456, yymsp[-2].minor.yy456); - yymsp[-8].minor.yy456 = addGroupByClause(pCxt, yymsp[-8].minor.yy456, yymsp[-1].minor.yy652); - yymsp[-8].minor.yy456 = addHavingClause(pCxt, yymsp[-8].minor.yy456, yymsp[0].minor.yy456); + yymsp[-8].minor.yy662 = createSelectStmt(pCxt, yymsp[-7].minor.yy617, yymsp[-6].minor.yy568, yymsp[-5].minor.yy662); + yymsp[-8].minor.yy662 = addWhereClause(pCxt, yymsp[-8].minor.yy662, yymsp[-4].minor.yy662); + yymsp[-8].minor.yy662 = addPartitionByClause(pCxt, yymsp[-8].minor.yy662, yymsp[-3].minor.yy568); + yymsp[-8].minor.yy662 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy662, yymsp[-2].minor.yy662); + yymsp[-8].minor.yy662 = addGroupByClause(pCxt, yymsp[-8].minor.yy662, yymsp[-1].minor.yy568); + yymsp[-8].minor.yy662 = addHavingClause(pCxt, yymsp[-8].minor.yy662, yymsp[0].minor.yy662); } break; - case 381: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy673 = false; } + case 384: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy617 = false; } break; - case 382: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy652 = NULL; } + case 385: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy568 = NULL; } break; - case 387: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy456 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456), &yymsp[0].minor.yy517); } - yymsp[-1].minor.yy456 = yylhsminor.yy456; + case 390: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy662 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy662), &yymsp[0].minor.yy555); } + yymsp[-1].minor.yy662 = yylhsminor.yy662; break; - case 388: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy456 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), &yymsp[0].minor.yy517); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 391: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy662 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), &yymsp[0].minor.yy555); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 393: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 410: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==410); - case 421: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==421); -{ yymsp[-2].minor.yy652 = yymsp[0].minor.yy652; } + case 396: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 413: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==413); + case 424: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==424); +{ yymsp[-2].minor.yy568 = yymsp[0].minor.yy568; } break; - case 395: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy456 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } + case 398: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy662 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy662), releaseRawExprNode(pCxt, yymsp[-1].minor.yy662)); } break; - case 396: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ -{ yymsp[-3].minor.yy456 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy456)); } + case 399: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ +{ yymsp[-3].minor.yy662 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy662)); } break; - case 397: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy456 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), NULL, yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } + case 400: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy662 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy662), NULL, yymsp[-1].minor.yy662, yymsp[0].minor.yy662); } break; - case 398: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy456 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy456), releaseRawExprNode(pCxt, yymsp[-3].minor.yy456), yymsp[-1].minor.yy456, yymsp[0].minor.yy456); } + case 401: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy662 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy662), releaseRawExprNode(pCxt, yymsp[-3].minor.yy662), yymsp[-1].minor.yy662, yymsp[0].minor.yy662); } break; - case 400: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ yymsp[-3].minor.yy456 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy456); } + case 403: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ yymsp[-3].minor.yy662 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy662); } break; - case 402: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy456 = createFillNode(pCxt, yymsp[-1].minor.yy534, NULL); } + case 405: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy662 = createFillNode(pCxt, yymsp[-1].minor.yy284, NULL); } break; - case 403: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy456 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy652)); } + case 406: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy662 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy568)); } break; - case 404: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy534 = FILL_MODE_NONE; } + case 407: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy284 = FILL_MODE_NONE; } break; - case 405: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy534 = FILL_MODE_PREV; } + case 408: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy284 = FILL_MODE_PREV; } break; - case 406: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy534 = FILL_MODE_NULL; } + case 409: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy284 = FILL_MODE_NULL; } break; - case 407: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy534 = FILL_MODE_LINEAR; } + case 410: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy284 = FILL_MODE_LINEAR; } break; - case 408: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy534 = FILL_MODE_NEXT; } + case 411: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy284 = FILL_MODE_NEXT; } break; - case 411: /* group_by_list ::= expression */ -{ yylhsminor.yy652 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[0].minor.yy652 = yylhsminor.yy652; + case 414: /* group_by_list ::= expression */ +{ yylhsminor.yy568 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } + yymsp[0].minor.yy568 = yylhsminor.yy568; break; - case 412: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy652 = addNodeToList(pCxt, yymsp[-2].minor.yy652, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy456))); } - yymsp[-2].minor.yy652 = yylhsminor.yy652; + case 415: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy568 = addNodeToList(pCxt, yymsp[-2].minor.yy568, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy662))); } + yymsp[-2].minor.yy568 = yylhsminor.yy568; break; - case 415: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 418: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy456 = addOrderByClause(pCxt, yymsp[-3].minor.yy456, yymsp[-2].minor.yy652); - yylhsminor.yy456 = addSlimitClause(pCxt, yylhsminor.yy456, yymsp[-1].minor.yy456); - yylhsminor.yy456 = addLimitClause(pCxt, yylhsminor.yy456, yymsp[0].minor.yy456); + yylhsminor.yy662 = addOrderByClause(pCxt, yymsp[-3].minor.yy662, yymsp[-2].minor.yy568); + yylhsminor.yy662 = addSlimitClause(pCxt, yylhsminor.yy662, yymsp[-1].minor.yy662); + yylhsminor.yy662 = addLimitClause(pCxt, yylhsminor.yy662, yymsp[0].minor.yy662); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; - case 417: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy456 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy456, yymsp[0].minor.yy456); } - yymsp[-3].minor.yy456 = yylhsminor.yy456; + case 420: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy662 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy662, yymsp[0].minor.yy662); } + yymsp[-3].minor.yy662 = yylhsminor.yy662; break; - case 418: /* query_expression_body ::= query_expression_body UNION query_expression_body */ -{ yylhsminor.yy456 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy456, yymsp[0].minor.yy456); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 421: /* query_expression_body ::= query_expression_body UNION query_expression_body */ +{ yylhsminor.yy662 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy662, yymsp[0].minor.yy662); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 423: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 427: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==427); -{ yymsp[-1].minor.yy456 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 426: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 430: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==430); +{ yymsp[-1].minor.yy662 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 424: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 428: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==428); -{ yymsp[-3].minor.yy456 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 427: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 431: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==431); +{ yymsp[-3].minor.yy662 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 425: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 429: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==429); -{ yymsp[-3].minor.yy456 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 428: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 432: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==432); +{ yymsp[-3].minor.yy662 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 430: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy456 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy456); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 433: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy662 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy662); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 434: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy456 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy456), yymsp[-1].minor.yy250, yymsp[0].minor.yy645); } - yymsp[-2].minor.yy456 = yylhsminor.yy456; + case 437: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy662 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy662), yymsp[-1].minor.yy272, yymsp[0].minor.yy181); } + yymsp[-2].minor.yy662 = yylhsminor.yy662; break; - case 435: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy250 = ORDER_ASC; } + case 438: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy272 = ORDER_ASC; } break; - case 436: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy250 = ORDER_ASC; } + case 439: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy272 = ORDER_ASC; } break; - case 437: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy250 = ORDER_DESC; } + case 440: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy272 = ORDER_DESC; } break; - case 438: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy645 = NULL_ORDER_DEFAULT; } + case 441: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy181 = NULL_ORDER_DEFAULT; } break; - case 439: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy645 = NULL_ORDER_FIRST; } + case 442: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy181 = NULL_ORDER_FIRST; } break; - case 440: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy645 = NULL_ORDER_LAST; } + case 443: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy181 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/parserAstTest.cpp b/source/libs/parser/test/parserAstTest.cpp index 5fc4e777c1..491a923457 100644 --- a/source/libs/parser/test/parserAstTest.cpp +++ b/source/libs/parser/test/parserAstTest.cpp @@ -353,7 +353,7 @@ TEST_F(ParserTest, selectSemanticError) { // TSDB_CODE_PAR_INVALID_FUNTION bind("SELECT cnt(*) FROM t1"); - ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_PAR_INVALID_FUNTION)); + ASSERT_TRUE(run(TSDB_CODE_SUCCESS, TSDB_CODE_FUNC_INVALID_FUNTION)); // TSDB_CODE_PAR_FUNTION_PARA_NUM // TSDB_CODE_PAR_FUNTION_PARA_TYPE diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2c7ad27927..ac5188170a 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -199,7 +199,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); - pScan->scanFlag = MAIN_SCAN; + pScan->scanSeq[0] = 1; + pScan->scanSeq[1] = 0; pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.acctId = pCxt->pPlanCxt->acctId; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 832135e90e..21e55fc34a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -20,11 +20,14 @@ #define OPTIMIZE_FLAG_MASK(n) (1 << n) #define OPTIMIZE_FLAG_OSD OPTIMIZE_FLAG_MASK(0) +#define OPTIMIZE_FLAG_CPD OPTIMIZE_FLAG_MASK(1) +#define OPTIMIZE_FLAG_OPK OPTIMIZE_FLAG_MASK(2) #define OPTIMIZE_FLAG_SET_MASK(val, mask) (val) |= (mask) #define OPTIMIZE_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) typedef struct SOptimizeContext { + SPlanContext* pPlanCxt; bool optimized; } SOptimizeContext; @@ -57,7 +60,23 @@ typedef enum ECondAction { // after supporting outer join, there are other possibilities } ECondAction; -EDealRes haveNormalColImpl(SNode* pNode, void* pContext) { +typedef bool (*FMayBeOptimized)(SLogicNode* pNode); + +static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) { + if (func(pNode)) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pScanNode = optFindPossibleNode((SLogicNode*)pChild, func); + if (NULL != pScanNode) { + return pScanNode; + } + } + return NULL; +} + +EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType); return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; @@ -65,9 +84,9 @@ EDealRes haveNormalColImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool haveNormalCol(SNodeList* pList) { +static bool osdHaveNormalCol(SNodeList* pList) { bool res = false; - nodesWalkExprsPostOrder(pList, haveNormalColImpl, &res); + nodesWalkExprsPostOrder(pList, osdHaveNormalColImpl, &res); return res; } @@ -89,21 +108,7 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) { return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType); } - return !haveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); -} - -static SLogicNode* osdFindPossibleScanNode(SLogicNode* pNode) { - if (osdMayBeOptimized(pNode)) { - return pNode; - } - SNode* pChild; - FOREACH(pChild, pNode->pChildren) { - SLogicNode* pScanNode = osdFindPossibleScanNode((SLogicNode*)pChild); - if (NULL != pScanNode) { - return pScanNode; - } - } - return NULL; + return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } static SNodeList* osdGetAllFuncs(SLogicNode* pNode) { @@ -138,7 +143,7 @@ static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, } static int32_t osdMatch(SOptimizeContext* pCxt, SLogicNode* pLogicNode, SOsdInfo* pInfo) { - pInfo->pScan = (SScanLogicNode*)osdFindPossibleScanNode(pLogicNode); + pInfo->pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, osdMayBeOptimized); if (NULL == pInfo->pScan) { return TSDB_CODE_SUCCESS; } @@ -345,7 +350,7 @@ static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, } static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan) { - if (NULL == pScan->node.pConditions) { + if (NULL == pScan->node.pConditions || OPTIMIZE_FLAG_TEST_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) { return TSDB_CODE_SUCCESS; } @@ -359,7 +364,10 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* pScan->node.pConditions = pOtherCond; } - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS == code) { + OPTIMIZE_FLAG_SET_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + pCxt->optimized = true; + } else { nodesDestroyNode(pPrimaryKeyCond); nodesDestroyNode(pOtherCond); } @@ -367,7 +375,7 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* return code; } -static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) { +static bool cpdBelongThisTable(SNode* pCondCol, SNodeList* pTableCols) { SNode* pTableCol = NULL; FOREACH(pTableCol, pTableCols) { if (nodesEqualNode(pCondCol, pTableCol)) { @@ -380,9 +388,9 @@ static bool belongThisTable(SNode* pCondCol, SNodeList* pTableCols) { static EDealRes cpdIsMultiTableCondImpl(SNode* pNode, void* pContext) { SCpdIsMultiTableCondCxt* pCxt = pContext; if (QUERY_NODE_COLUMN == nodeType(pNode)) { - if (belongThisTable(pNode, pCxt->pLeftCols)) { + if (cpdBelongThisTable(pNode, pCxt->pLeftCols)) { pCxt->havaLeftCol = true; - } else if (belongThisTable(pNode, pCxt->pRightCols)) { + } else if (cpdBelongThisTable(pNode, pCxt->pRightCols)) { pCxt->haveRightCol = true; } return pCxt->havaLeftCol && pCxt->haveRightCol ? DEAL_RES_END : DEAL_RES_CONTINUE; @@ -508,11 +516,76 @@ static int32_t cpdPushCondToChild(SOptimizeContext* pCxt, SLogicNode* pChild, SN return TSDB_CODE_PLAN_INTERNAL_ERROR; } +static bool cpdIsPrimaryKey(SNode* pNode, SNodeList* pTableCols) { + if (QUERY_NODE_COLUMN != nodeType(pNode)) { + return false; + } + SColumnNode* pCol = (SColumnNode*)pNode; + if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { + return false; + } + return cpdBelongThisTable(pNode, pTableCols); +} + +static bool cpdIsPrimaryKeyEqualCond(SJoinLogicNode* pJoin, SNode* pCond) { + if (QUERY_NODE_OPERATOR != nodeType(pCond)) { + return false; + } + SNodeList* pLeftCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 0))->pTargets; + SNodeList* pRightCols = ((SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1))->pTargets; + SOperatorNode* pOper = (SOperatorNode*)pJoin->pOnConditions; + if (cpdIsPrimaryKey(pOper->pLeft, pLeftCols)) { + return cpdIsPrimaryKey(pOper->pRight, pRightCols); + } else if (cpdIsPrimaryKey(pOper->pLeft, pRightCols)) { + return cpdIsPrimaryKey(pOper->pRight, pLeftCols); + } + return false; +} + +static int32_t cpdCheckOpCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SNode* pOnCond) { + if (!cpdIsPrimaryKeyEqualCond(pJoin, pOnCond)) { + snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t cpdCheckLogicCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin, SLogicConditionNode* pOnCond) { + if (LOGIC_COND_TYPE_AND != pOnCond->condType) { + snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); + return TSDB_CODE_FAILED; + } + SNode* pCond = NULL; + FOREACH(pCond, pOnCond->pParameterList) { + if (!cpdIsPrimaryKeyEqualCond(pJoin, pCond)) { + snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "l.ts = r.ts is expected in join expression"); + return TSDB_CODE_FAILED; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t cpdCheckJoinOnCond(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { + if (NULL == pJoin->pOnConditions) { + snprintf(pCxt->pPlanCxt->pMsg, pCxt->pPlanCxt->msgLen, "not support cross join"); + return TSDB_CODE_FAILED; + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pJoin->pOnConditions)) { + return cpdCheckLogicCond(pCxt, pJoin, (SLogicConditionNode*)pJoin->pOnConditions); + } else { + return cpdCheckOpCond(pCxt, pJoin, pJoin->pOnConditions); + } +} + static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoin) { - if (NULL == pJoin->node.pConditions) { + if (OPTIMIZE_FLAG_TEST_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD)) { return TSDB_CODE_SUCCESS; } + if (NULL == pJoin->node.pConditions) { + return cpdCheckJoinOnCond(pCxt, pJoin); + } + SNode* pOnCond = NULL; SNode* pLeftChildCond = NULL; SNode* pRightChildCond = NULL; @@ -527,7 +600,11 @@ static int32_t cpdPushJoinCondition(SOptimizeContext* pCxt, SJoinLogicNode* pJoi code = cpdPushCondToChild(pCxt, (SLogicNode*)nodesListGetNode(pJoin->node.pChildren, 1), &pRightChildCond); } - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS == code) { + OPTIMIZE_FLAG_SET_MASK(pJoin->node.optimizedFlag, OPTIMIZE_FLAG_CPD); + pCxt->optimized = true; + code = cpdCheckJoinOnCond(pCxt, pJoin); + } else { nodesDestroyNode(pOnCond); nodesDestroyNode(pLeftChildCond); nodesDestroyNode(pRightChildCond); @@ -572,15 +649,129 @@ static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { return cpdPushCondition(pCxt, pLogicNode); } +static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { + if (1 != LIST_LENGTH(pSortKeys)) { + return false; + } + SNode* pNode = ((SOrderByExprNode*)nodesListGetNode(pSortKeys, 0))->pExpr; + return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false); +} + +static bool opkSortMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SORT != nodeType(pNode)) { + return false; + } + if (OPTIMIZE_FLAG_TEST_MASK(pNode->optimizedFlag, OPTIMIZE_FLAG_OPK)) { + return false; + } + return true; +} + +static int32_t opkGetScanNodesImpl(SLogicNode* pNode, bool* pNotOptimize, SNodeList** pScanNodes) { + int32_t code = TSDB_CODE_SUCCESS; + + switch (nodeType(pNode)) { + case QUERY_NODE_LOGIC_PLAN_SCAN: + return nodesListMakeAppend(pScanNodes, pNode); + case QUERY_NODE_LOGIC_PLAN_JOIN: + code = opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); + if (TSDB_CODE_SUCCESS == code) { + code = opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 1), pNotOptimize, pScanNodes); + } + return code; + case QUERY_NODE_LOGIC_PLAN_AGG: + *pNotOptimize = true; + return code; + default: + break; + } + + if (1 != LIST_LENGTH(pNode->pChildren)) { + *pNotOptimize = true; + } + + return opkGetScanNodesImpl(nodesListGetNode(pNode->pChildren, 0), pNotOptimize, pScanNodes); +} + +static int32_t opkGetScanNodes(SLogicNode* pNode, SNodeList** pScanNodes) { + bool notOptimize = false; + int32_t code = opkGetScanNodesImpl(pNode, ¬Optimize, pScanNodes); + if (TSDB_CODE_SUCCESS != code || notOptimize) { + nodesClearList(*pScanNodes); + } + return code; +} + +static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { + return ((SOrderByExprNode*)nodesListGetNode(pSort->pSortKeys, 0))->order; +} + +static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { + SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); + // todo + pSort->node.pChildren = NULL; + return pDownNode; +} + +static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) { + EOrder order = opkGetPrimaryKeyOrder(pSort); + if (ORDER_DESC == order) { + SNode* pScan = NULL; + FOREACH(pScan, pScanNodes) { + ((SScanLogicNode*)pScan)->scanSeq[0] = 0; + ((SScanLogicNode*)pScan)->scanSeq[1] = 1; + } + } + + if (NULL == pSort->node.pParent) { + // todo + return TSDB_CODE_SUCCESS; + } + + SNode* pDownNode = opkRewriteDownNode(pSort); + SNode* pNode; + FOREACH(pNode, pSort->node.pParent->pChildren) { + if (nodesEqualNode(pNode, pSort)) { + REPLACE_NODE(pDownNode); + break; + } + } + nodesDestroyNode(pSort); + return TSDB_CODE_SUCCESS; +} + +static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) { + OPTIMIZE_FLAG_SET_MASK(pSort->node.optimizedFlag, OPTIMIZE_FLAG_OPK); + if (!opkIsPrimaryKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { + return TSDB_CODE_SUCCESS; + } + SNodeList* pScanNodes = NULL; + int32_t code = opkGetScanNodes(nodesListGetNode(pSort->node.pChildren, 0), &pScanNodes); + if (TSDB_CODE_SUCCESS == code && NULL != pScanNodes) { + code = opkDoOptimized(pCxt, pSort, pScanNodes); + } + nodesClearList(pScanNodes); + return code; +} + +static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { + SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized); + if (NULL == pSort) { + return TSDB_CODE_SUCCESS; + } + return opkOptimizeImpl(pCxt, pSort); +} + static const SOptimizeRule optimizeRuleSet[] = { { .pName = "OptimizeScanData", .optimizeFunc = osdOptimize }, - { .pName = "ConditionPushDown", .optimizeFunc = cpdOptimize } + { .pName = "ConditionPushDown", .optimizeFunc = cpdOptimize }, + { .pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize } }; static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule)); -static int32_t applyOptimizeRule(SLogicNode* pLogicNode) { - SOptimizeContext cxt = { .optimized = false }; +static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { + SOptimizeContext cxt = { .pPlanCxt = pCxt, .optimized = false }; do { cxt.optimized = false; for (int32_t i = 0; i < optimizeRuleNum; ++i) { @@ -594,5 +785,5 @@ static int32_t applyOptimizeRule(SLogicNode* pLogicNode) { } int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode) { - return applyOptimizeRule(pLogicNode); + return applyOptimizeRule(pCxt, pLogicNode); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index d5e12608ac..28635ef940 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -398,9 +398,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; - pScanPhysiNode->order = TSDB_ORDER_ASC; - pScanPhysiNode->count = 1; - pScanPhysiNode->reverse = 0; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); } @@ -432,7 +429,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp return TSDB_CODE_OUT_OF_MEMORY; } - pTableScan->scanFlag = pScanLogicNode->scanFlag; + memcpy(pTableScan->scanSeq, pScanLogicNode->scanSeq, sizeof(pScanLogicNode->scanSeq)); pTableScan->scanRange = pScanLogicNode->scanRange; pTableScan->ratio = pScanLogicNode->ratio; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); diff --git a/source/libs/planner/test/planOptTest.cpp b/source/libs/planner/test/planOptTest.cpp new file mode 100644 index 0000000000..4a682a5e17 --- /dev/null +++ b/source/libs/planner/test/planOptTest.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanOptimizeTest : public PlannerTestBase { + +}; + +TEST_F(PlanOptimizeTest, orderByPrimaryKey) { + useDb("root", "test"); + + run("select * from t1 order by ts"); + run("select * from t1 order by ts desc"); + run("select c1 from t1 order by ts"); + run("select c1 from t1 order by ts desc"); +} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index de805d2c77..fd7c831399 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -342,28 +342,20 @@ PROCESS_META_OVER: int32_t queryProcessQnodeListRsp(void *output, char *msg, int32_t msgSize) { SQnodeListRsp out = {0}; - int32_t code = -1; + int32_t code = 0; if (NULL == output || NULL == msg || msgSize <= 0) { code = TSDB_CODE_TSC_INVALID_INPUT; - goto PROCESS_QLIST_OVER; + return code; } + out.addrsList = (SArray *)output; if (tDeserializeSQnodeListRsp(msg, msgSize, &out) != 0) { qError("invalid qnode list rsp msg, msgSize:%d", msgSize); code = TSDB_CODE_INVALID_MSG; - goto PROCESS_QLIST_OVER; + return code; } -PROCESS_QLIST_OVER: - - if (code != 0) { - tFreeSQnodeListRsp(&out); - out.addrsList = NULL; - } - - *(SArray **)output = out.addrsList; - return code; } diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index cc82349fee..de73f1db0b 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -25,10 +25,10 @@ extern "C" { #include "ttimer.h" #define QW_DEFAULT_SCHEDULER_NUMBER 10000 -#define QW_DEFAULT_TASK_NUMBER 10000 -#define QW_DEFAULT_SCH_TASK_NUMBER 10000 -#define QW_DEFAULT_SHORT_RUN_TIMES 2 -#define QW_DEFAULT_HEARTBEAT_MSEC 3000 +#define QW_DEFAULT_TASK_NUMBER 10000 +#define QW_DEFAULT_SCH_TASK_NUMBER 10000 +#define QW_DEFAULT_SHORT_RUN_TIMES 2 +#define QW_DEFAULT_HEARTBEAT_MSEC 3000 enum { QW_PHASE_PRE_QUERY = 1, @@ -60,7 +60,6 @@ enum { QW_WRITE, }; - enum { QW_NOT_EXIST_RET_ERR = 1, QW_NOT_EXIST_ADD, @@ -73,25 +72,26 @@ typedef struct SQWDebug { } SQWDebug; typedef struct SQWConnInfo { - void *handle; - void *ahandle; + void * handle; + void * ahandle; + int64_t refId; } SQWConnInfo; typedef struct SQWMsg { - void *node; - int32_t code; - char *msg; - int32_t msgLen; - SQWConnInfo connInfo; + void * node; + int32_t code; + char * msg; + int32_t msgLen; + SQWConnInfo connInfo; } SQWMsg; typedef struct SQWHbInfo { - SSchedulerHbRsp rsp; - SQWConnInfo connInfo; + SSchedulerHbRsp rsp; + SQWConnInfo connInfo; } SQWHbInfo; typedef struct SQWPhaseInput { - int32_t code; + int32_t code; } SQWPhaseInput; typedef struct SQWPhaseOutput { @@ -100,41 +100,40 @@ typedef struct SQWPhaseOutput { #endif } SQWPhaseOutput; - -typedef struct SQWTaskStatus { - int64_t refId; // job's refId - int32_t code; - int8_t status; +typedef struct SQWTaskStatus { + int64_t refId; // job's refId + int32_t code; + int8_t status; } SQWTaskStatus; typedef struct SQWTaskCtx { - SRWLatch lock; - int8_t phase; - int8_t taskType; - int8_t explain; - - bool queryFetched; - bool queryEnd; - bool queryContinue; - bool queryInQueue; - int32_t rspCode; + SRWLatch lock; + int8_t phase; + int8_t taskType; + int8_t explain; - SQWConnInfo ctrlConnInfo; - SQWConnInfo dataConnInfo; + bool queryFetched; + bool queryEnd; + bool queryContinue; + bool queryInQueue; + int32_t rspCode; - int8_t events[QW_EVENT_MAX]; - - qTaskInfo_t taskHandle; - DataSinkHandle sinkHandle; + SQWConnInfo ctrlConnInfo; + SQWConnInfo dataConnInfo; + + int8_t events[QW_EVENT_MAX]; + + void *taskHandle; + void *sinkHandle; } SQWTaskCtx; typedef struct SQWSchStatus { - int32_t lastAccessTs; // timestamp in second + int32_t lastAccessTs; // timestamp in second SRWLatch hbConnLock; SQWConnInfo hbConnInfo; - SQueryNodeEpId hbEpId; + SQueryNodeEpId hbEpId; SRWLatch tasksLock; - SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus + SHashObj * tasksHash; // key:queryId+taskId, value: SQWTaskStatus } SQWSchStatus; // Qnode/Vnode level task management @@ -142,100 +141,146 @@ typedef struct SQWorkerMgmt { SQWorkerCfg cfg; int8_t nodeType; int32_t nodeId; - void *timer; + void * timer; tmr_h hbTimer; SRWLatch schLock; // SRWLatch ctxLock; - SHashObj *schHash; // key: schedulerId, value: SQWSchStatus - SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx - SMsgCb msgCb; + SHashObj *schHash; // key: schedulerId, value: SQWSchStatus + SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx + SMsgCb msgCb; } SQWorkerMgmt; #define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId -#define QW_IDS() sId, qId, tId, rId -#define QW_FPARAMS() mgmt, QW_IDS() +#define QW_IDS() sId, qId, tId, rId +#define QW_FPARAMS() mgmt, QW_IDS() #define QW_GET_EVENT_VALUE(ctx, event) atomic_load_8(&(ctx)->events[event]) -#define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED) -#define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED) -#define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) +#define QW_IS_EVENT_RECEIVED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_RECEIVED) +#define QW_IS_EVENT_PROCESSED(ctx, event) (atomic_load_8(&(ctx)->events[event]) == QW_EVENT_PROCESSED) +#define QW_SET_EVENT_RECEIVED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_RECEIVED) #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) -#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) +#define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) #define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) #define QW_IS_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY) -#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) +#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) -#define QW_TASK_READY(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED) -#define QW_SET_QTID(id, qId, tId) do { *(uint64_t *)(id) = (qId); *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); } while (0) -#define QW_GET_QTID(id, qId, tId) do { (qId) = *(uint64_t *)(id); (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); } while (0) +#define QW_TASK_READY(status) \ + (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || \ + status == JOB_TASK_STATUS_PARTIAL_SUCCEED) +#define QW_SET_QTID(id, qId, tId) \ + do { \ + *(uint64_t *)(id) = (qId); \ + *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); \ + } while (0) +#define QW_GET_QTID(id, qId, tId) \ + do { \ + (qId) = *(uint64_t *)(id); \ + (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); \ + } while (0) -#define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) -#define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) -#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +#define QW_ERR_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + terrno = _code; \ + return _code; \ + } \ + } while (0) +#define QW_RET(c) \ + do { \ + int32_t _code = c; \ + if (_code != TSDB_CODE_SUCCESS) { \ + terrno = _code; \ + } \ + return _code; \ + } while (0) +#define QW_ERR_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + terrno = code; \ + goto _return; \ + } \ + } while (0) #define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) #define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) -#define QW_DUMP(param, ...) do { if (gQWDebug.dumpEnable) { qDebug("QW:%p " param, mgmt, __VA_ARGS__); } } while (0) +#define QW_DUMP(param, ...) \ + do { \ + if (gQWDebug.dumpEnable) { \ + qDebug("QW:%p " param, mgmt, __VA_ARGS__); \ + } \ + } while (0) +#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) +#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%" PRIx64 " " param, mgmt, sId, __VA_ARGS__) -#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) -#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOGL(param, ...) \ + qDebugL("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_DLOGL(param, ...) qDebugL("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId) +#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId) +#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, qId, tId) -#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) +#define QW_SCH_TASK_ELOG(param, ...) \ + qError("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) \ + qWarn("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) \ + qDebug("QW:%p SID:0x%" PRIx64 ",QID:0x%" PRIx64 ",TID:0x%" PRIx64 " " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:0x%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) - -#define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockEnable) { qDebug(__VA_ARGS__); } } while (0) +#define QW_LOCK_DEBUG(...) \ + do { \ + if (gQWDebug.lockEnable) { \ + qDebug(__VA_ARGS__); \ + } \ + } while (0) #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define QW_LOCK(type, _lock) do { \ - if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) >= 0); \ - QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) > 0); \ - } else { \ - assert(atomic_load_32((_lock)) >= 0); \ - QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ - } \ -} while (0) - -#define QW_UNLOCK(type, _lock) do { \ - if (QW_READ == (type)) { \ - assert(atomic_load_32((_lock)) > 0); \ - QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ - } else { \ - assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ - QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - assert(atomic_load_32((_lock)) >= 0); \ - } \ -} while (0) +#define QW_LOCK(type, _lock) \ + do { \ + if (QW_READ == (type)) { \ + assert(atomic_load_32((_lock)) >= 0); \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + QW_LOCK_DEBUG("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ + } else { \ + assert(atomic_load_32((_lock)) >= 0); \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + QW_LOCK_DEBUG("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + } \ + } while (0) + +#define QW_UNLOCK(type, _lock) \ + do { \ + if (QW_READ == (type)) { \ + assert(atomic_load_32((_lock)) > 0); \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + QW_LOCK_DEBUG("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + } else { \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + QW_LOCK_DEBUG("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + } \ + } while (0) #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7329dc76a3..b51bc5083e 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,13 +1,13 @@ #include "qworker.h" -#include "tcommon.h" +#include "dataSinkMgt.h" #include "executor.h" #include "planner.h" #include "query.h" #include "qworkerInt.h" #include "qworkerMsg.h" +#include "tcommon.h" #include "tmsg.h" #include "tname.h" -#include "dataSinkMgt.h" SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = true}; @@ -26,47 +26,40 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + switch (oriStatus) { case JOB_TASK_STATUS_NULL: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_NOT_START) { + if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_FAILED && + newStatus != JOB_TASK_STATUS_NOT_START) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_NOT_START: if (newStatus != JOB_TASK_STATUS_CANCELLED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED - && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_CANCELLING - && newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED && newStatus != JOB_TASK_STATUS_SUCCEED && + newStatus != JOB_TASK_STATUS_FAILED && newStatus != JOB_TASK_STATUS_CANCELLING && + newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_DROPPING) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_DROPPING) { + if (newStatus != JOB_TASK_STATUS_EXECUTING && newStatus != JOB_TASK_STATUS_SUCCEED && + newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_FAILED && + newStatus != JOB_TASK_STATUS_DROPPING) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_SUCCEED: - if (newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING - && newStatus != JOB_TASK_STATUS_FAILED) { + if (newStatus != JOB_TASK_STATUS_CANCELLED && newStatus != JOB_TASK_STATUS_DROPPING && + newStatus != JOB_TASK_STATUS_FAILED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -81,7 +74,7 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, if (newStatus != JOB_TASK_STATUS_CANCELLED) { QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - + break; case JOB_TASK_STATUS_CANCELLED: case JOB_TASK_STATUS_DROPPING: @@ -89,7 +82,7 @@ int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } break; - + default: QW_TASK_ELOG("invalid task origStatus:%s", jobTaskStatusStr(oriStatus)); return TSDB_CODE_QRY_APP_ERROR; @@ -103,9 +96,7 @@ _return: QW_RET(code); } -void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) { - -} +void qwDbgDumpSchInfo(SQWSchStatus *sch, int32_t i) {} void qwDbgDumpMgmtInfo(SQWorkerMgmt *mgmt) { if (!gQWDebug.dumpEnable) { @@ -113,12 +104,12 @@ void qwDbgDumpMgmtInfo(SQWorkerMgmt *mgmt) { } QW_LOCK(QW_READ, &mgmt->schLock); - + QW_DUMP("total remain schduler num:%d", taosHashGetSize(mgmt->schHash)); - void *key = NULL; - size_t keyLen = 0; - int32_t i = 0; + void * key = NULL; + size_t keyLen = 0; + int32_t i = 0; SQWSchStatus *sch = NULL; void *pIter = taosHashIterate(mgmt->schHash, NULL); @@ -172,33 +163,33 @@ char *qwBufStatusStr(int32_t bufStatus) { int32_t qwSetTaskStatus(QW_FPARAMS_DEF, SQWTaskStatus *task, int8_t status) { int32_t code = 0; - int8_t origStatus = 0; - bool ignore = false; + int8_t origStatus = 0; + bool ignore = false; while (true) { origStatus = atomic_load_8(&task->status); - + QW_ERR_RET(qwDbgValidateStatus(QW_FPARAMS(), origStatus, status, &ignore)); if (ignore) { break; } - + if (origStatus != atomic_val_compare_exchange_8(&task->status, origStatus, status)) { continue; } - + QW_TASK_DLOG("task status updated from %s to %s", jobTaskStatusStr(origStatus), jobTaskStatusStr(status)); break; } - + return TSDB_CODE_SUCCESS; } - int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { SQWSchStatus newSch = {0}; - newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + newSch.tasksHash = + taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { QW_SCH_ELOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -209,7 +200,7 @@ int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { if (0 != code) { if (!HASH_NODE_EXIST(code)) { QW_UNLOCK(QW_WRITE, &mgmt->schLock); - + QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); taosHashCleanup(newSch.tasksHash); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -219,7 +210,7 @@ int32_t qwAddSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType) { } QW_UNLOCK(QW_WRITE, &mgmt->schLock); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch, int32_t nOpt) { @@ -228,12 +219,12 @@ int32_t qwAcquireSchedulerImpl(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); if (NULL == (*sch)) { QW_UNLOCK(rwType, &mgmt->schLock); - + if (QW_NOT_EXIST_ADD == nOpt) { QW_ERR_RET(qwAddSchedulerImpl(mgmt, sId, rwType)); nOpt = QW_NOT_EXIST_RET_ERR; - + continue; } else if (QW_NOT_EXIST_RET_ERR == nOpt) { QW_RET(TSDB_CODE_QRY_SCH_NOT_EXIST); @@ -257,10 +248,7 @@ int32_t qwAcquireScheduler(SQWorkerMgmt *mgmt, uint64_t sId, int32_t rwType, SQW return qwAcquireSchedulerImpl(mgmt, sId, rwType, sch, QW_NOT_EXIST_RET_ERR); } -void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { - QW_UNLOCK(rwType, &mgmt->schLock); -} - +void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, SQWTaskStatus **task) { char id[sizeof(qId) + sizeof(tId)] = {0}; @@ -276,8 +264,6 @@ int32_t qwAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, S return TSDB_CODE_SUCCESS; } - - int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, int32_t status, SQWTaskStatus **task) { int32_t code = 0; @@ -317,7 +303,7 @@ int32_t qwAddTaskStatusImpl(QW_FPARAMS_DEF, SQWSchStatus *sch, int32_t rwType, i int32_t qwAddTaskStatus(QW_FPARAMS_DEF, int32_t status) { SQWSchStatus *tsch = NULL; - int32_t code = 0; + int32_t code = 0; QW_ERR_RET(qwAcquireAddScheduler(mgmt, sId, QW_READ, &tsch)); QW_ERR_JRET(qwAddTaskStatusImpl(QW_FPARAMS(), tsch, 0, status, NULL)); @@ -325,20 +311,16 @@ int32_t qwAddTaskStatus(QW_FPARAMS_DEF, int32_t status) { _return: qwReleaseScheduler(QW_READ, mgmt); - + QW_RET(code); } - -int32_t qwAddAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, int32_t status, SQWTaskStatus **task) { +int32_t qwAddAcquireTaskStatus(QW_FPARAMS_DEF, int32_t rwType, SQWSchStatus *sch, int32_t status, + SQWTaskStatus **task) { return qwAddTaskStatusImpl(QW_FPARAMS(), sch, rwType, status, task); } - -void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { - QW_UNLOCK(rwType, &sch->tasksLock); -} - +void qwReleaseTaskStatus(int32_t rwType, SQWSchStatus *sch) { QW_UNLOCK(rwType, &sch->tasksLock); } int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; @@ -356,7 +338,7 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - + *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*ctx)) { QW_TASK_DLOG_E("task ctx not exist, may be dropped"); @@ -398,19 +380,13 @@ int32_t qwAddTaskCtxImpl(QW_FPARAMS_DEF, bool acquire, SQWTaskCtx **ctx) { return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskCtx(QW_FPARAMS_DEF) { - QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), false, NULL)); -} +int32_t qwAddTaskCtx(QW_FPARAMS_DEF) { QW_RET(qwAddTaskCtxImpl(QW_FPARAMS(), false, NULL)); } -int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { - return qwAddTaskCtxImpl(QW_FPARAMS(), true, ctx); -} +int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTaskCtxImpl(QW_FPARAMS(), true, ctx); } -void qwReleaseTaskCtx(SQWorkerMgmt *mgmt, void *ctx) { - taosHashRelease(mgmt->ctxHash, ctx); -} +void qwReleaseTaskCtx(SQWorkerMgmt *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } -void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { +void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { @@ -430,22 +406,21 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { QW_RET(code); } - void qwFreeTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { tmsgReleaseHandle(ctx->ctrlConnInfo.handle, TAOS_CONN_SERVER); ctx->ctrlConnInfo.handle = NULL; + ctx->ctrlConnInfo.refId = -1; // NO need to release dataConnInfo qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); - + if (ctx->sinkHandle) { dsDestroyDataSinker(ctx->sinkHandle); ctx->sinkHandle = NULL; } } - int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -464,22 +439,22 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_DROP); if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { - QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); + QW_TASK_ELOG_E("taosHashRemove from ctx hash failed"); QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } qwFreeTask(QW_FPARAMS(), &octx); QW_TASK_DLOG_E("task ctx dropped"); - + return TSDB_CODE_SUCCESS; } int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { - SQWSchStatus *sch = NULL; + SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; - int32_t code = 0; - + int32_t code = 0; + char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); @@ -490,7 +465,7 @@ int32_t qwDropTaskStatus(QW_FPARAMS_DEF) { if (qwAcquireTaskStatus(QW_FPARAMS(), QW_WRITE, sch, &task)) { qwReleaseScheduler(QW_WRITE, mgmt); - + QW_TASK_WLOG_E("task does not exist"); return TSDB_CODE_SUCCESS; } @@ -508,20 +483,20 @@ _return: qwReleaseTaskStatus(QW_WRITE, sch); } qwReleaseScheduler(QW_WRITE, mgmt); - + QW_RET(code); } int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status) { - SQWSchStatus *sch = NULL; + SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; - int32_t code = 0; + int32_t code = 0; QW_ERR_RET(qwAcquireScheduler(mgmt, sId, QW_READ, &sch)); QW_ERR_JRET(qwAcquireTaskStatus(QW_FPARAMS(), QW_READ, sch, &task)); QW_ERR_JRET(qwSetTaskStatus(QW_FPARAMS(), task, status)); - + _return: if (task) { @@ -541,39 +516,38 @@ int32_t qwDropTask(QW_FPARAMS_DEF) { return TSDB_CODE_SUCCESS; } - int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { - qTaskInfo_t *taskHandle = &ctx->taskHandle; + qTaskInfo_t *taskHandle = &ctx->taskHandle; if (TASK_TYPE_TEMP == ctx->taskType) { if (ctx->explain) { SExplainExecInfo *execInfo = NULL; - int32_t resNum = 0; + int32_t resNum = 0; QW_ERR_RET(qGetExplainExecInfo(ctx->taskHandle, &resNum, &execInfo)); SQWConnInfo connInfo = {0}; connInfo.handle = ctx->ctrlConnInfo.handle; - + connInfo.refId = ctx->ctrlConnInfo.refId; + QW_ERR_RET(qwBuildAndSendExplainRsp(&connInfo, execInfo, resNum)); } - + qwFreeTaskHandle(QW_FPARAMS(), taskHandle); } return TSDB_CODE_SUCCESS; } - int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { - int32_t code = 0; - bool qcontinue = true; - SSDataBlock* pRes = NULL; - uint64_t useconds = 0; - int32_t i = 0; - int32_t execNum = 0; - qTaskInfo_t *taskHandle = &ctx->taskHandle; + int32_t code = 0; + bool qcontinue = true; + SSDataBlock * pRes = NULL; + uint64_t useconds = 0; + int32_t i = 0; + int32_t execNum = 0; + qTaskInfo_t * taskHandle = &ctx->taskHandle; DataSinkHandle sinkHandle = ctx->sinkHandle; - + while (true) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); @@ -586,7 +560,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { ++execNum; if (NULL == pRes) { - QW_TASK_DLOG("qExecTask end with empty res, useconds:%"PRIu64, useconds); + QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); dsEndPut(sinkHandle, useconds); @@ -595,7 +569,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { if (queryEnd) { *queryEnd = true; } - + break; } @@ -611,7 +585,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { } QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", rows, qcontinue); - + if (!qcontinue) { break; } @@ -639,7 +613,7 @@ int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbI hbInfo->rsp.epId = sch->hbEpId; QW_LOCK(QW_READ, &sch->tasksLock); - + taskNum = taosHashGetSize(sch->tasksHash); hbInfo->rsp.taskStatus = taosArrayInit(taskNum, sizeof(STaskStatus)); @@ -649,9 +623,9 @@ int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbI return TSDB_CODE_QRY_OUT_OF_MEMORY; } - void *key = NULL; - size_t keyLen = 0; - int32_t i = 0; + void * key = NULL; + size_t keyLen = 0; + int32_t i = 0; STaskStatus status = {0}; void *pIter = taosHashIterate(sch->tasksHash, NULL); @@ -659,29 +633,28 @@ int32_t qwGenerateSchHbRsp(SQWorkerMgmt *mgmt, SQWSchStatus *sch, SQWHbInfo *hbI SQWTaskStatus *taskStatus = (SQWTaskStatus *)pIter; key = taosHashGetKey(pIter, &keyLen); - //TODO GET EXECUTOR API TO GET MORE INFO + // TODO GET EXECUTOR API TO GET MORE INFO QW_GET_QTID(key, status.queryId, status.taskId); status.status = taskStatus->status; status.refId = taskStatus->refId; - + taosArrayPush(hbInfo->rsp.taskStatus, &status); - + ++i; pIter = taosHashIterate(sch->tasksHash, pIter); - } + } QW_UNLOCK(QW_READ, &sch->tasksLock); return TSDB_CODE_SUCCESS; } - int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void **rspMsg, SOutputData *pOutput) { - int32_t len = 0; + int32_t len = 0; SRetrieveTableRsp *rsp = NULL; - bool queryEnd = false; - int32_t code = 0; + bool queryEnd = false; + int32_t code = 0; dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); @@ -697,7 +670,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); QW_ERR_RET(code); } - + QW_TASK_DLOG_E("no data in sink and query end"); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED); @@ -717,10 +690,10 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_TASK_DLOG("there are data in sink, dataLength:%d", len); *dataLen = len; - + QW_ERR_RET(qwMallocFetchRsp(len, &rsp)); *rspMsg = rsp; - + pOutput->pData = rsp->data; code = dsGetDataBlock(ctx->sinkHandle, pOutput); if (code) { @@ -737,8 +710,8 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void } int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { - int32_t code = 0; - SQWTaskCtx *ctx = NULL; + int32_t code = 0; + SQWTaskCtx * ctx = NULL; SQWConnInfo *dropConnection = NULL; SQWConnInfo *cancelConnection = NULL; @@ -749,16 +722,16 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } else { QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); } - + QW_LOCK(QW_WRITE, &ctx->lock); if (QW_PHASE_PRE_FETCH == phase) { - atomic_store_8((int8_t*)&ctx->queryFetched, true); + atomic_store_8((int8_t *)&ctx->queryFetched, true); } else { atomic_store_8(&ctx->phase, phase); } - if (atomic_load_8((int8_t*)&ctx->queryEnd)) { + if (atomic_load_8((int8_t *)&ctx->queryEnd)) { QW_TASK_ELOG_E("query already end"); QW_ERR_JRET(TSDB_CODE_QW_MSG_ERROR); } @@ -802,7 +775,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_ERR_JRET(TSDB_CODE_QRY_TASK_MSG_ERROR); } break; - } + } case QW_PHASE_PRE_CQUERY: { if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { QW_TASK_WLOG("task already dropped, phase:%s", qwPhaseStr(phase)); @@ -828,7 +801,8 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } if (ctx->rspCode) { - QW_TASK_ELOG("task already failed at phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, tstrerror(ctx->rspCode)); + QW_TASK_ELOG("task already failed at phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, + tstrerror(ctx->rspCode)); QW_ERR_JRET(ctx->rspCode); } @@ -836,7 +810,7 @@ _return: if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); - + QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } @@ -856,17 +830,16 @@ _return: QW_RET(code); } - int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { - int32_t code = 0; - SQWTaskCtx *ctx = NULL; - SQWConnInfo connInfo = {0}; + int32_t code = 0; + SQWTaskCtx * ctx = NULL; + SQWConnInfo connInfo = {0}; SQWConnInfo *readyConnection = NULL; QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); - + QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); - + QW_LOCK(QW_WRITE, &ctx->lock); if (QW_IS_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) { @@ -882,8 +855,9 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp } #else connInfo.handle = ctx->ctrlConnInfo.handle; + connInfo.refId = ctx->ctrlConnInfo.refId; readyConnection = &connInfo; - + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); #endif } @@ -902,9 +876,10 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp } if (ctx->rspCode) { - QW_TASK_ELOG("task already failed, phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, tstrerror(ctx->rspCode)); + QW_TASK_ELOG("task already failed, phase %s, error:%x - %s", qwPhaseStr(phase), ctx->rspCode, + tstrerror(ctx->rspCode)); QW_ERR_JRET(ctx->rspCode); - } + } QW_ERR_JRET(input->code); @@ -920,13 +895,13 @@ _return: if (QW_PHASE_POST_FETCH != phase) { atomic_store_8(&ctx->phase, phase); } - + QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); } if (readyConnection) { - qwBuildAndSendReadyRsp(readyConnection, code); + qwBuildAndSendReadyRsp(readyConnection, code); QW_TASK_DLOG("ready msg rsped, handle:%p, code:%x - %s", readyConnection->handle, code, tstrerror(code)); } @@ -940,25 +915,26 @@ _return: } int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t explain) { - int32_t code = 0; - bool queryRsped = false; + int32_t code = 0; + bool queryRsped = false; struct SSubplan *plan = NULL; - SQWPhaseInput input = {0}; - qTaskInfo_t pTaskInfo = NULL; - DataSinkHandle sinkHandle = NULL; - SQWTaskCtx *ctx = NULL; + SQWPhaseInput input = {0}; + qTaskInfo_t pTaskInfo = NULL; + DataSinkHandle sinkHandle = NULL; + SQWTaskCtx * ctx = NULL; QW_ERR_JRET(qwRegisterQueryBrokenLinkArg(QW_FPARAMS(), &qwMsg->connInfo)); QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_QUERY, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - + atomic_store_8(&ctx->taskType, taskType); atomic_store_8(&ctx->explain, explain); atomic_store_ptr(&ctx->ctrlConnInfo.handle, qwMsg->connInfo.handle); atomic_store_ptr(&ctx->ctrlConnInfo.ahandle, qwMsg->connInfo.ahandle); + atomic_store_64(&ctx->ctrlConnInfo.refId, qwMsg->connInfo.refId); QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); @@ -967,7 +943,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex QW_TASK_ELOG("task string to subplan failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); } - + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -979,10 +955,10 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - //QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code)); - //QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); + // QW_ERR_JRET(qwBuildAndSendQueryRsp(&qwMsg->connInfo, code)); + // QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); - //queryRsped = true; + // queryRsped = true; atomic_store_ptr(&ctx->taskHandle, pTaskInfo); atomic_store_ptr(&ctx->sinkHandle, sinkHandle); @@ -990,13 +966,13 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType, int8_t ex if (pTaskInfo && sinkHandle) { QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL)); } - + _return: input.code = code; code = qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_QUERY, &input, NULL); - //if (!queryRsped) { + // if (!queryRsped) { // qwBuildAndSendQueryRsp(&qwMsg->connInfo, code); // QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); //} @@ -1005,10 +981,10 @@ _return: } int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - int32_t code = 0; + int32_t code = 0; SQWTaskCtx *ctx = NULL; - int8_t phase = 0; - bool needRsp = true; + int8_t phase = 0; + bool needRsp = true; QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx)); @@ -1018,7 +994,7 @@ int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_TASK_WLOG_E("task is dropping or already dropped"); QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - + if (ctx->phase == QW_PHASE_PRE_QUERY) { ctx->ctrlConnInfo.handle = qwMsg->connInfo.handle; ctx->ctrlConnInfo.ahandle = qwMsg->connInfo.ahandle; @@ -1030,8 +1006,9 @@ int32_t qwProcessReady(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_READY); - if (atomic_load_8((int8_t*)&ctx->queryEnd) || atomic_load_8((int8_t*)&ctx->queryFetched)) { - QW_TASK_ELOG("got ready msg at wrong status, queryEnd:%d, queryFetched:%d", atomic_load_8((int8_t*)&ctx->queryEnd), atomic_load_8((int8_t*)&ctx->queryFetched)); + if (atomic_load_8((int8_t *)&ctx->queryEnd) || atomic_load_8((int8_t *)&ctx->queryFetched)) { + QW_TASK_ELOG("got ready msg at wrong status, queryEnd:%d, queryFetched:%d", atomic_load_8((int8_t *)&ctx->queryEnd), + atomic_load_8((int8_t *)&ctx->queryFetched)); QW_ERR_JRET(TSDB_CODE_QW_MSG_ERROR); } @@ -1067,116 +1044,116 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } - int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - SQWTaskCtx *ctx = NULL; - int32_t code = 0; + SQWTaskCtx * ctx = NULL; + int32_t code = 0; SQWPhaseInput input = {0}; - void *rsp = NULL; - int32_t dataLen = 0; - bool queryEnd = false; - + void * rsp = NULL; + int32_t dataLen = 0; + bool queryEnd = false; + do { QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - atomic_store_8((int8_t*)&ctx->queryInQueue, 0); - atomic_store_8((int8_t*)&ctx->queryContinue, 0); + atomic_store_8((int8_t *)&ctx->queryInQueue, 0); + atomic_store_8((int8_t *)&ctx->queryContinue, 0); QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd)); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); - - if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { + + if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { QW_TASK_DLOG("task not end and buf is %s, need to continue query", qwBufStatusStr(sOutput.bufStatus)); - - atomic_store_8((int8_t*)&ctx->queryContinue, 1); + + atomic_store_8((int8_t *)&ctx->queryContinue, 1); } - + if (rsp) { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); if (qComplete) { - atomic_store_8((int8_t*)&ctx->queryEnd, true); + atomic_store_8((int8_t *)&ctx->queryEnd, true); } qwMsg->connInfo = ctx->dataConnInfo; - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - - qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + + qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, + tstrerror(code), dataLen); } else { - atomic_store_8((int8_t*)&ctx->queryContinue, 1); + atomic_store_8((int8_t *)&ctx->queryContinue, 1); } } -_return: + _return: if (NULL == ctx) { break; } if (code && QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { - QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); + QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); qwFreeFetchRsp(rsp); rsp = NULL; - + qwMsg->connInfo = ctx->dataConnInfo; qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, 0, code); - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), 0); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), + 0); } QW_LOCK(QW_WRITE, &ctx->lock); - if (queryEnd || code || 0 == atomic_load_8((int8_t*)&ctx->queryContinue)) { + if (queryEnd || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { // Note: if necessary, fetch need to put cquery to queue again atomic_store_8(&ctx->phase, 0); - QW_UNLOCK(QW_WRITE,&ctx->lock); + QW_UNLOCK(QW_WRITE, &ctx->lock); break; } - QW_UNLOCK(QW_WRITE,&ctx->lock); + QW_UNLOCK(QW_WRITE, &ctx->lock); } while (true); input.code = code; - qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL); + qwHandlePostPhaseEvents(QW_FPARAMS(), QW_PHASE_POST_CQUERY, &input, NULL); - QW_RET(TSDB_CODE_SUCCESS); + QW_RET(TSDB_CODE_SUCCESS); } - int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - int32_t code = 0; - int32_t dataLen = 0; - bool locked = false; - SQWTaskCtx *ctx = NULL; - void *rsp = NULL; + int32_t code = 0; + int32_t dataLen = 0; + bool locked = false; + SQWTaskCtx * ctx = NULL; + void * rsp = NULL; SQWPhaseInput input = {0}; QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - + SOutputData sOutput = {0}; QW_ERR_JRET(qwGetResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); if (NULL == rsp) { atomic_store_ptr(&ctx->dataConnInfo.handle, qwMsg->connInfo.handle); atomic_store_ptr(&ctx->dataConnInfo.ahandle, qwMsg->connInfo.ahandle); - + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); if (qComplete) { - atomic_store_8((int8_t*)&ctx->queryEnd, true); + atomic_store_8((int8_t *)&ctx->queryEnd, true); } } - if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { + if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { QW_TASK_DLOG("task not end and buf is %s, need to continue query", qwBufStatusStr(sOutput.bufStatus)); QW_LOCK(QW_WRITE, &ctx->lock); @@ -1184,16 +1161,16 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { // RC WARNING if (QW_IS_QUERY_RUNNING(ctx)) { - atomic_store_8((int8_t*)&ctx->queryContinue, 1); - } else if (0 == atomic_load_8((int8_t*)&ctx->queryInQueue)) { + atomic_store_8((int8_t *)&ctx->queryContinue, 1); + } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXECUTING); - atomic_store_8((int8_t*)&ctx->queryInQueue, 1); - + atomic_store_8((int8_t *)&ctx->queryInQueue, 1); + QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo)); } } - + _return: if (locked) { @@ -1211,23 +1188,23 @@ _return: if (code || rsp) { qwBuildAndSendFetchRsp(&qwMsg->connInfo, rsp, dataLen, code); - QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); } QW_RET(TSDB_CODE_SUCCESS); } - int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - int32_t code = 0; - bool rsped = false; + int32_t code = 0; + bool rsped = false; SQWTaskCtx *ctx = NULL; - bool locked = false; + bool locked = false; // TODO : TASK ALREADY REMOVED AND A NEW DROP MSG RECEIVED QW_ERR_JRET(qwAddAcquireTaskCtx(QW_FPARAMS(), &ctx)); - + QW_LOCK(QW_WRITE, &ctx->lock); locked = true; @@ -1255,7 +1232,7 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (!rsped) { ctx->ctrlConnInfo.handle = qwMsg->connInfo.handle; ctx->ctrlConnInfo.ahandle = qwMsg->connInfo.ahandle; - + QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } @@ -1286,9 +1263,9 @@ _return: } int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { - int32_t code = 0; + int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus *sch = NULL; + SQWSchStatus * sch = NULL; QW_ERR_RET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); @@ -1303,23 +1280,23 @@ int32_t qwProcessHbLinkBroken(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq } else { QW_DLOG("ignore hb connection broken, handle:%p, currentHandle:%p", qwMsg->connInfo.handle, sch->hbConnInfo.handle); } - + QW_UNLOCK(QW_WRITE, &sch->hbConnLock); - + qwReleaseScheduler(QW_READ, mgmt); - + QW_RET(TSDB_CODE_SUCCESS); } int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { - int32_t code = 0; + int32_t code = 0; SSchedulerHbRsp rsp = {0}; - SQWSchStatus *sch = NULL; + SQWSchStatus * sch = NULL; if (qwMsg->code) { QW_RET(qwProcessHbLinkBroken(mgmt, qwMsg, req)); } - + QW_ERR_JRET(qwAcquireAddScheduler(mgmt, req->sId, QW_READ, &sch)); QW_ERR_JRET(qwRegisterHbBrokenLinkArg(mgmt, req->sId, &qwMsg->connInfo)); @@ -1332,11 +1309,11 @@ int32_t qwProcessHb(SQWorkerMgmt *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { memcpy(&sch->hbConnInfo, &qwMsg->connInfo, sizeof(qwMsg->connInfo)); memcpy(&sch->hbEpId, &req->epId, sizeof(req->epId)); - + QW_UNLOCK(QW_WRITE, &sch->hbConnLock); - - QW_DLOG("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, handle:%p, ahandle:%p", - req->sId, req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connInfo.handle, qwMsg->connInfo.ahandle); + + QW_DLOG("hb connection updated, sId:%" PRIx64 ", nodeId:%d, fqdn:%s, port:%d, handle:%p, ahandle:%p", req->sId, + req->epId.nodeId, req->epId.ep.fqdn, req->epId.ep.port, qwMsg->connInfo.handle, qwMsg->connInfo.ahandle); qwReleaseScheduler(QW_READ, mgmt); @@ -1349,19 +1326,18 @@ _return: if (code) { tmsgReleaseHandle(qwMsg->connInfo.handle, TAOS_CONN_SERVER); } - + QW_DLOG("hb rsp send, handle:%p, code:%x - %s", qwMsg->connInfo.handle, code, tstrerror(code)); - + QW_RET(TSDB_CODE_SUCCESS); } - void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWorkerMgmt *mgmt = (SQWorkerMgmt *)param; SQWSchStatus *sch = NULL; - int32_t taskNum = 0; - SQWHbInfo *rspList = NULL; - int32_t code = 0; + int32_t taskNum = 0; + SQWHbInfo * rspList = NULL; + int32_t code = 0; qwDbgDumpMgmtInfo(mgmt); @@ -1382,8 +1358,8 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - void *key = NULL; - size_t keyLen = 0; + void * key = NULL; + size_t keyLen = 0; int32_t i = 0; void *pIter = taosHashIterate(mgmt->schHash, NULL); @@ -1395,7 +1371,7 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { pIter = taosHashIterate(mgmt->schHash, pIter); continue; } - + code = qwGenerateSchHbRsp(mgmt, (SQWSchStatus *)pIter, &rspList[i]); if (code) { taosHashCancelIterate(mgmt->schHash, pIter); @@ -1419,7 +1395,7 @@ _return: taosMemoryFreeClear(rspList); - taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); + taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); } int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb) { @@ -1428,7 +1404,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; + int32_t code = 0; SQWorkerMgmt *mgmt = taosMemoryCalloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); @@ -1452,14 +1428,16 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->cfg.maxSchTaskNum = QW_DEFAULT_SCH_TASK_NUMBER; } - mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); + mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, + HASH_ENTRY_LOCK); if (NULL == mgmt->schHash) { taosMemoryFreeClear(mgmt); qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + mgmt->ctxHash = + taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (NULL == mgmt->ctxHash) { qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); QW_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -1493,7 +1471,7 @@ _return: taosHashCleanup(mgmt->ctxHash); taosTmrCleanUp(mgmt->timer); - + taosMemoryFreeClear(mgmt); QW_RET(code); @@ -1508,10 +1486,10 @@ void qWorkerDestroy(void **qWorkerMgmt) { taosTmrStopA(&mgmt->hbTimer); taosTmrCleanUp(mgmt->timer); - - //TODO STOP ALL QUERY - //TODO FREE ALL + // TODO STOP ALL QUERY + + // TODO FREE ALL taosHashCleanup(mgmt->ctxHash); taosHashCleanup(mgmt->schHash); @@ -1520,152 +1498,145 @@ void qWorkerDestroy(void **qWorkerMgmt) { } int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRsp **rsp) { -/* - SQWSchStatus *sch = NULL; - int32_t taskNum = 0; + /* + SQWSchStatus *sch = NULL; + int32_t taskNum = 0; - QW_ERR_RET(qwAcquireScheduler(mgmt, sId, QW_READ, &sch)); - - sch->lastAccessTs = taosGetTimestampSec(); + QW_ERR_RET(qwAcquireScheduler(mgmt, sId, QW_READ, &sch)); + + sch->lastAccessTs = taosGetTimestampSec(); + + QW_LOCK(QW_READ, &sch->tasksLock); + + taskNum = taosHashGetSize(sch->tasksHash); + + int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; + *rsp = taosMemoryCalloc(1, size); + if (NULL == *rsp) { + QW_SCH_ELOG("calloc %d failed", size); + QW_UNLOCK(QW_READ, &sch->tasksLock); + qwReleaseScheduler(QW_READ, mgmt); + + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + void *key = NULL; + size_t keyLen = 0; + int32_t i = 0; + + void *pIter = taosHashIterate(sch->tasksHash, NULL); + while (pIter) { + SQWTaskStatus *taskStatus = (SQWTaskStatus *)pIter; + taosHashGetKey(pIter, &key, &keyLen); + + QW_GET_QTID(key, (*rsp)->status[i].queryId, (*rsp)->status[i].taskId); + (*rsp)->status[i].status = taskStatus->status; + + ++i; + pIter = taosHashIterate(sch->tasksHash, pIter); + } - QW_LOCK(QW_READ, &sch->tasksLock); - - taskNum = taosHashGetSize(sch->tasksHash); - - int32_t size = sizeof(SSchedulerStatusRsp) + sizeof((*rsp)->status[0]) * taskNum; - *rsp = taosMemoryCalloc(1, size); - if (NULL == *rsp) { - QW_SCH_ELOG("calloc %d failed", size); QW_UNLOCK(QW_READ, &sch->tasksLock); qwReleaseScheduler(QW_READ, mgmt); - - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - void *key = NULL; - size_t keyLen = 0; - int32_t i = 0; - - void *pIter = taosHashIterate(sch->tasksHash, NULL); - while (pIter) { - SQWTaskStatus *taskStatus = (SQWTaskStatus *)pIter; - taosHashGetKey(pIter, &key, &keyLen); - - QW_GET_QTID(key, (*rsp)->status[i].queryId, (*rsp)->status[i].taskId); - (*rsp)->status[i].status = taskStatus->status; - - ++i; - pIter = taosHashIterate(sch->tasksHash, pIter); - } - - QW_UNLOCK(QW_READ, &sch->tasksLock); - qwReleaseScheduler(QW_READ, mgmt); - - (*rsp)->num = taskNum; -*/ + (*rsp)->num = taskNum; + */ return TSDB_CODE_SUCCESS; } - - int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; -/* - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); + /* + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - sch->lastAccessTs = taosGetTimestampSec(); + sch->lastAccessTs = taosGetTimestampSec(); - qwReleaseScheduler(QW_READ, mgmt); -*/ + qwReleaseScheduler(QW_READ, mgmt); + */ return TSDB_CODE_SUCCESS; } - int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t *taskStatus) { - SQWSchStatus *sch = NULL; + SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; - int32_t code = 0; + int32_t code = 0; -/* - if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) { - *taskStatus = JOB_TASK_STATUS_NULL; - return TSDB_CODE_SUCCESS; - } + /* + if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) { + *taskStatus = JOB_TASK_STATUS_NULL; + return TSDB_CODE_SUCCESS; + } - if (qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)) { + if (qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)) { + qwReleaseScheduler(QW_READ, mgmt); + + *taskStatus = JOB_TASK_STATUS_NULL; + return TSDB_CODE_SUCCESS; + } + + *taskStatus = task->status; + + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - - *taskStatus = JOB_TASK_STATUS_NULL; - return TSDB_CODE_SUCCESS; - } - - *taskStatus = task->status; - - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); -*/ + */ QW_RET(code); } - int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { - SQWSchStatus *sch = NULL; + SQWSchStatus * sch = NULL; SQWTaskStatus *task = NULL; - int32_t code = 0; + int32_t code = 0; -/* - QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch)); + /* + QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task)); + QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task)); - QW_LOCK(QW_WRITE, &task->lock); + QW_LOCK(QW_WRITE, &task->lock); + + task->cancel = true; + + int8_t oriStatus = task->status; + int8_t newStatus = 0; + + if (task->status == JOB_TASK_STATUS_CANCELLED || task->status == JOB_TASK_STATUS_NOT_START || task->status == + JOB_TASK_STATUS_CANCELLING || task->status == JOB_TASK_STATUS_DROPPING) { QW_UNLOCK(QW_WRITE, &task->lock); + + qwReleaseTask(QW_READ, sch); + qwReleaseScheduler(QW_READ, mgmt); + + return TSDB_CODE_SUCCESS; + } else if (task->status == JOB_TASK_STATUS_FAILED || task->status == JOB_TASK_STATUS_SUCCEED || task->status == + JOB_TASK_STATUS_PARTIAL_SUCCEED) { QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); } else { + QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLING)); + } - task->cancel = true; - - int8_t oriStatus = task->status; - int8_t newStatus = 0; - - if (task->status == JOB_TASK_STATUS_CANCELLED || task->status == JOB_TASK_STATUS_NOT_START || task->status == JOB_TASK_STATUS_CANCELLING || task->status == JOB_TASK_STATUS_DROPPING) { QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - + + if (oriStatus == JOB_TASK_STATUS_EXECUTING) { + //TODO call executer to cancel subquery async + } + return TSDB_CODE_SUCCESS; - } else if (task->status == JOB_TASK_STATUS_FAILED || task->status == JOB_TASK_STATUS_SUCCEED || task->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLED)); - } else { - QW_ERR_JRET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_CANCELLING)); - } - QW_UNLOCK(QW_WRITE, &task->lock); - - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); + _return: - if (oriStatus == JOB_TASK_STATUS_EXECUTING) { - //TODO call executer to cancel subquery async - } - - return TSDB_CODE_SUCCESS; + if (task) { + QW_UNLOCK(QW_WRITE, &task->lock); -_return: + qwReleaseTask(QW_READ, sch); + } - if (task) { - QW_UNLOCK(QW_WRITE, &task->lock); - - qwReleaseTask(QW_READ, sch); - } - - if (sch) { - qwReleaseScheduler(QW_READ, mgmt); - } -*/ + if (sch) { + qwReleaseScheduler(QW_READ, mgmt); + } + */ QW_RET(code); } - - diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 6723eb21c1..64df4f02bf 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -1,18 +1,17 @@ -#include "qworker.h" -#include "tcommon.h" +#include "qworkerMsg.h" +#include "dataSinkMgt.h" #include "executor.h" #include "planner.h" #include "query.h" +#include "qworker.h" #include "qworkerInt.h" -#include "qworkerMsg.h" +#include "tcommon.h" #include "tmsg.h" #include "tname.h" -#include "dataSinkMgt.h" - int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { int32_t msgSize = sizeof(SRetrieveTableRsp) + length; - + SRetrieveTableRsp *pRsp = (SRetrieveTableRsp *)rpcMallocCont(msgSize); if (NULL == pRsp) { qError("rpcMallocCont %d failed", msgSize); @@ -26,11 +25,9 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { return TSDB_CODE_SUCCESS; } - - void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - + rsp->useconds = htobe64(input->useconds); rsp->completed = qComplete; rsp->precision = input->precision; @@ -39,7 +36,6 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) rsp->numOfRows = htonl(input->numOfRows); } - void qwFreeFetchRsp(void *msg) { if (msg) { rpcFreeCont(msg); @@ -48,18 +44,19 @@ void qwFreeFetchRsp(void *msg) { int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) { SQueryTableRsp rsp = {.code = code}; - + int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp); - void *msg = rpcMallocCont(contLen); + void *msg = rpcMallocCont(contLen); tSerializeSQueryTableRsp(msg, contLen, &rsp); SRpcMsg rpcRsp = { - .msgType = TDMT_VND_QUERY_RSP, - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .pCont = msg, - .contLen = contLen, - .code = code, + .msgType = TDMT_VND_QUERY_RSP, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .pCont = msg, + .contLen = contLen, + .code = code, }; tmsgSendRsp(&rpcRsp); @@ -72,12 +69,13 @@ int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { - .msgType = TDMT_VND_RES_READY_RSP, - .handle = pConn->handle, - .ahandle = NULL, - .pCont = pRsp, - .contLen = sizeof(*pRsp), - .code = code, + .msgType = TDMT_VND_RES_READY_RSP, + .handle = pConn->handle, + .refId = pConn->refId, + .ahandle = NULL, + .pCont = pRsp, + .contLen = sizeof(*pRsp), + .code = code, }; tmsgSendRsp(&rpcRsp); @@ -85,20 +83,21 @@ int32_t qwBuildAndSendReadyRsp(SQWConnInfo *pConn, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num) { +int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t num) { SExplainRsp rsp = {.numOfPlans = num, .subplanInfo = execInfo}; int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); + void *pRsp = rpcMallocCont(contLen); tSerializeSExplainRsp(pRsp, contLen, &rsp); SRpcMsg rpcRsp = { - .msgType = TDMT_VND_EXPLAIN_RSP, - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .pCont = pRsp, - .contLen = contLen, - .code = 0, + .msgType = TDMT_VND_EXPLAIN_RSP, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .pCont = pRsp, + .contLen = contLen, + .code = 0, }; tmsgSendRsp(&rpcRsp); @@ -108,16 +107,17 @@ int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) { int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus); - void *pRsp = rpcMallocCont(contLen); + void *pRsp = rpcMallocCont(contLen); tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus); SRpcMsg rpcRsp = { - .msgType = TDMT_VND_QUERY_HEARTBEAT_RSP, - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .pCont = pRsp, - .contLen = contLen, - .code = code, + .msgType = TDMT_VND_QUERY_HEARTBEAT_RSP, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .pCont = pRsp, + .contLen = contLen, + .code = code, }; tmsgSendRsp(&rpcRsp); @@ -133,12 +133,13 @@ int32_t qwBuildAndSendFetchRsp(SQWConnInfo *pConn, SRetrieveTableRsp *pRsp, int3 } SRpcMsg rpcRsp = { - .msgType = TDMT_VND_FETCH_RSP, - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .pCont = pRsp, - .contLen = sizeof(*pRsp) + dataLength, - .code = code, + .msgType = TDMT_VND_FETCH_RSP, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .pCont = pRsp, + .contLen = sizeof(*pRsp) + dataLength, + .code = code, }; tmsgSendRsp(&rpcRsp); @@ -151,12 +152,13 @@ int32_t qwBuildAndSendCancelRsp(SQWConnInfo *pConn, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { - .msgType = TDMT_VND_CANCEL_TASK_RSP, - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .pCont = pRsp, - .contLen = sizeof(*pRsp), - .code = code, + .msgType = TDMT_VND_CANCEL_TASK_RSP, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .pCont = pRsp, + .contLen = sizeof(*pRsp), + .code = code, }; tmsgSendRsp(&rpcRsp); @@ -168,12 +170,13 @@ int32_t qwBuildAndSendDropRsp(SQWConnInfo *pConn, int32_t code) { pRsp->code = code; SRpcMsg rpcRsp = { - .msgType = TDMT_VND_DROP_TASK_RSP, - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .pCont = pRsp, - .contLen = sizeof(*pRsp), - .code = code, + .msgType = TDMT_VND_DROP_TASK_RSP, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .pCont = pRsp, + .contLen = sizeof(*pRsp), + .code = code, }; tmsgSendRsp(&rpcRsp); @@ -191,7 +194,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { return -1; } - col_id_t cols = 0; + col_id_t cols = 0; SSchema *pSchema = showRsp.tableMeta.pSchemas; const SSchema *s = tGetTbnameColumnSchema(); @@ -226,6 +229,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { SRpcMsg rpcMsg = { .handle = pMsg->handle, .ahandle = pMsg->ahandle, + .refId = pMsg->refId, .pCont = pBuf, .contLen = bufLen, .code = code, @@ -235,17 +239,18 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { return TSDB_CODE_SUCCESS; } -int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchReq) { +int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq *pFetchReq) { SVShowTablesFetchRsp *pRsp = (SVShowTablesFetchRsp *)rpcMallocCont(sizeof(SVShowTablesFetchRsp)); - int32_t handle = htonl(pFetchReq->id); + int32_t handle = htonl(pFetchReq->id); pRsp->numOfRows = 0; SRpcMsg rpcMsg = { - .handle = pMsg->handle, + .handle = pMsg->handle, .ahandle = pMsg->ahandle, - .pCont = pRsp, + .refId = pMsg->refId, + .pCont = pRsp, .contLen = sizeof(*pRsp), - .code = 0, + .code = 0, }; tmsgSendRsp(&rpcMsg); @@ -253,7 +258,7 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe } int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { - SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); + SQueryContinueReq *req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -265,12 +270,13 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { req->taskId = tId; SRpcMsg pNewMsg = { - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .msgType = TDMT_VND_QUERY_CONTINUE, - .pCont = req, - .contLen = sizeof(SQueryContinueReq), - .code = 0, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .msgType = TDMT_VND_QUERY_CONTINUE, + .refId = pConn->refId, + .pCont = req, + .contLen = sizeof(SQueryContinueReq), + .code = 0, }; int32_t code = tmsgPutToQueue(&mgmt->msgCb, QUERY_QUEUE, &pNewMsg); @@ -285,29 +291,29 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { return TSDB_CODE_SUCCESS; } - int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SQWConnInfo *pConn) { - STaskDropReq * req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); + STaskDropReq *req = (STaskDropReq *)rpcMallocCont(sizeof(STaskDropReq)); if (NULL == req) { QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(STaskDropReq)); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } + } req->header.vgId = htonl(mgmt->nodeId); req->sId = htobe64(sId); req->queryId = htobe64(qId); req->taskId = htobe64(tId); req->refId = htobe64(rId); - + SRpcMsg pMsg = { - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .msgType = TDMT_VND_DROP_TASK, - .pCont = req, - .contLen = sizeof(STaskDropReq), - .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .msgType = TDMT_VND_DROP_TASK, + .pCont = req, + .contLen = sizeof(STaskDropReq), + .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, }; - + tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg); return TSDB_CODE_SUCCESS; @@ -333,43 +339,42 @@ int32_t qwRegisterHbBrokenLinkArg(SQWorkerMgmt *mgmt, uint64_t sId, SQWConnInfo taosMemoryFree(msg); QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - + SRpcMsg pMsg = { - .handle = pConn->handle, - .ahandle = pConn->ahandle, - .msgType = TDMT_VND_QUERY_HEARTBEAT, - .pCont = msg, - .contLen = sizeof(SSchedulerHbReq), - .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, + .handle = pConn->handle, + .ahandle = pConn->ahandle, + .refId = pConn->refId, + .msgType = TDMT_VND_QUERY_HEARTBEAT, + .pCont = msg, + .contLen = msgSize, + .code = TSDB_CODE_RPC_NETWORK_UNAVAIL, }; - + tmsgRegisterBrokenLinkArg(&mgmt->msgCb, &pMsg); return TSDB_CODE_SUCCESS; } - - int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - int32_t code = 0; + int32_t code = 0; SSubQueryMsg *msg = pMsg->pCont; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - + if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { QW_ELOG("invalid query msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); + msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->refId = be64toh(msg->refId); - msg->phyLen = ntohl(msg->phyLen); - msg->sqlLen = ntohl(msg->sqlLen); + msg->taskId = be64toh(msg->taskId); + msg->refId = be64toh(msg->refId); + msg->phyLen = ntohl(msg->phyLen); + msg->sqlLen = ntohl(msg->sqlLen); uint64_t sId = msg->sId; uint64_t qId = msg->queryId; @@ -379,8 +384,9 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWMsg qwMsg = {.node = node, .msg = msg->msg + msg->sqlLen, .msgLen = msg->phyLen}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + qwMsg.connInfo.refId = pMsg->refId; - char* sql = strndup(msg->msg, msg->sqlLen); + char *sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, handle:%p, sql:%s", node, pMsg->handle, sql); taosMemoryFreeClear(sql); @@ -388,17 +394,17 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_SCH_TASK_DLOG("processQuery end, node:%p", node); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { - int32_t code = 0; - int8_t status = 0; - bool queryDone = false; + int32_t code = 0; + int8_t status = 0; + bool queryDone = false; SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont; - bool needStop = false; - SQWTaskCtx *handles = NULL; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + bool needStop = false; + SQWTaskCtx *handles = NULL; + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -408,11 +414,12 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; - int64_t rId = 0; + int64_t rId = 0; SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + qwMsg.connInfo.refId = pMsg->refId; QW_SCH_TASK_DLOG("processCQuery start, node:%p, handle:%p", node, pMsg->handle); @@ -420,10 +427,10 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_SCH_TASK_DLOG("processCQuery end, node:%p", node); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } -int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ +int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } @@ -433,7 +440,7 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task ready msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); @@ -442,18 +449,19 @@ int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; - int64_t rId = 0; + int64_t rId = 0; SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + qwMsg.connInfo.refId = pMsg->refId; QW_SCH_TASK_DLOG("processReady start, node:%p, handle:%p", node, pMsg->handle); QW_ERR_RET(qwProcessReady(QW_FPARAMS(), &qwMsg)); QW_SCH_TASK_DLOG("processReady end, node:%p", node); - + return TSDB_CODE_SUCCESS; } @@ -462,24 +470,24 @@ int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - int32_t code = 0; + int32_t code = 0; SSchTasksStatusReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid task status msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; msg->sId = htobe64(msg->sId); uint64_t sId = msg->sId; SSchedulerStatusRsp *sStatus = NULL; - - //QW_ERR_JRET(qwGetSchTasksStatus(qWorkerMgmt, msg->sId, &sStatus)); + + // QW_ERR_JRET(qwGetSchTasksStatus(qWorkerMgmt, msg->sId, &sStatus)); _return: - //QW_ERR_RET(qwBuildAndSendStatusRsp(pMsg, sStatus)); + // QW_ERR_RET(qwBuildAndSendStatusRsp(pMsg, sStatus)); return TSDB_CODE_SUCCESS; } @@ -491,11 +499,11 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SResFetchReq *msg = pMsg->pCont; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); + QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); @@ -504,11 +512,12 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { uint64_t sId = msg->sId; uint64_t qId = msg->queryId; uint64_t tId = msg->taskId; - int64_t rId = 0; + int64_t rId = 0; SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + qwMsg.connInfo.refId = pMsg->refId; QW_SCH_TASK_DLOG("processFetch start, node:%p, handle:%p", node, pMsg->handle); @@ -516,7 +525,7 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_SCH_TASK_DLOG("processFetch end, node:%p", node); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } int32_t qWorkerProcessFetchRsp(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { @@ -529,13 +538,13 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - int32_t code = 0; + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - qError("invalid task cancel msg"); + qError("invalid task cancel msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); @@ -550,8 +559,9 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + qwMsg.connInfo.refId = pMsg->refId; - //QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId)); + // QW_ERR_JRET(qwCancelTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId)); _return: @@ -566,14 +576,14 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - int32_t code = 0; + int32_t code = 0; STaskDropReq *msg = pMsg->pCont; SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - + if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid task drop msg, msg:%p, msgLen:%d", msg, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } msg->sId = be64toh(msg->sId); msg->queryId = be64toh(msg->queryId); @@ -588,9 +598,10 @@ int32_t qWorkerProcessDropMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + qwMsg.connInfo.refId = pMsg->refId; if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) { - QW_SCH_TASK_DLOG("receive drop task due to network broken, error:%s", tstrerror(pMsg->code)); + QW_SCH_TASK_DLOG("receive drop task due to network broken, error:%s", tstrerror(pMsg->code)); } QW_SCH_TASK_DLOG("processDrop start, node:%p, handle:%p", node, pMsg->handle); @@ -607,14 +618,14 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - int32_t code = 0; + int32_t code = 0; SSchedulerHbReq req = {0}; - SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; - + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + if (NULL == pMsg->pCont) { QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } if (tDeserializeSSchedulerHbReq(pMsg->pCont, pMsg->contLen, &req)) { QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen); @@ -623,12 +634,13 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } uint64_t sId = req.sId; - SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code}; + SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .code = pMsg->code}; qwMsg.connInfo.handle = pMsg->handle; qwMsg.connInfo.ahandle = pMsg->ahandle; + qwMsg.connInfo.refId = pMsg->refId; if (TSDB_CODE_RPC_NETWORK_UNAVAIL == pMsg->code) { - QW_SCH_DLOG("receive Hb msg due to network broken, error:%s", tstrerror(pMsg->code)); + QW_SCH_DLOG("receive Hb msg due to network broken, error:%s", tstrerror(pMsg->code)); } QW_SCH_DLOG("processHb start, node:%p, handle:%p", node, pMsg->handle); @@ -645,7 +657,7 @@ int32_t qWorkerProcessShowMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - int32_t code = 0; + int32_t code = 0; SVShowTablesReq *pReq = pMsg->pCont; QW_RET(qwBuildAndSendShowRsp(pMsg, code)); } diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 3d59ffd93d..9142cc41c4 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -32,6 +32,9 @@ typedef struct SScalarCtx { #define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DEFAULT_OP_NUM 10 +#define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) +#define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) + #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) #define sclWarn(...) qWarn(__VA_ARGS__) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 145afbe984..294afc8073 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -244,23 +244,53 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t return TSDB_CODE_SUCCESS; } -int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { +int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *paramNum, int32_t *rowNum) { int32_t code = 0; - SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam)); + if (NULL == pParamList) { + if (ctx->pBlockList) { + SSDataBlock *pBlock = taosArrayGet(ctx->pBlockList, 0); + *rowNum = pBlock->info.rows; + } else { + *rowNum = 1; + } + + *paramNum = 1; + } else { + *paramNum = pParamList->length; + } + + SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam)); if (NULL == paramList) { - sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); + sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SListCell *cell = pParamList->pHead; - for (int32_t i = 0; i < pParamList->length; ++i) { - if (NULL == cell || NULL == cell->pNode) { - sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); - SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + if (pParamList) { + SNode *tnode = NULL; + int32_t i = 0; + if (SCL_IS_CONST_CALC(ctx)) { + WHERE_EACH (tnode, pParamList) { + if (!SCL_IS_CONST_NODE(tnode)) { + WHERE_NEXT; + } else { + SCL_ERR_JRET(sclInitParam(tnode, ¶mList[i], ctx, rowNum)); + ERASE_NODE(pParamList); + } + + ++i; + } + } else { + FOREACH(tnode, pParamList) { + SCL_ERR_JRET(sclInitParam(tnode, ¶mList[i], ctx, rowNum)); + ++i; + } } + } else { + paramList[0].numOfRows = *rowNum; + } - SCL_ERR_JRET(sclInitParam(cell->pNode, ¶mList[i], ctx, rowNum)); - cell = cell->pNext; + if (0 == *rowNum) { + taosMemoryFreeClear(paramList); } *pParams = paramList; @@ -299,37 +329,45 @@ _return: } int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { - if (NULL == node->pParameterList || node->pParameterList->length <= 0) { - sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); - SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SScalarFuncExecFuncs ffpSet = {0}; - int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); - if (code) { - sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); - SCL_ERR_RET(code); - } - SScalarParam *params = NULL; int32_t rowNum = 0; - SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + int32_t paramNum = 0; + int32_t code = 0; + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); - output->columnData = createColumnInfoData(&node->node.resType, rowNum); - if (output->columnData == NULL) { - sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - code = (*ffpSet.process)(params, node->pParameterList->length, output); - if (code) { - sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + if (fmIsUserDefinedFunc(node->funcId)) { +#if 0 + UdfcFuncHandle udfHandle = NULL; + + SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle)); + code = callUdfScalarFunc(udfHandle, params, paramNum, output); + teardownUdf(udfHandle); SCL_ERR_JRET(code); +#endif + } else { + SScalarFuncExecFuncs ffpSet = {0}; + code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); + if (code) { + sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } + + output->columnData = createColumnInfoData(&node->node.resType, rowNum); + if (output->columnData == NULL) { + sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + code = (*ffpSet.process)(params, paramNum, output); + if (code) { + sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } } _return: - for (int32_t i = 0; i < node->pParameterList->length; ++i) { + for (int32_t i = 0; i < paramNum; ++i) { // sclFreeParamNoData(params + i); } @@ -355,8 +393,13 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o SScalarParam *params = NULL; int32_t rowNum = 0; + int32_t paramNum = 0; int32_t code = 0; - SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); + if (NULL == params) { + output->numOfRows = 0; + return TSDB_CODE_SUCCESS; + } int32_t type = node->node.resType.type; output->numOfRows = rowNum; @@ -369,25 +412,41 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o } bool value = false; + bool complete = true; for (int32_t i = 0; i < rowNum; ++i) { - for (int32_t m = 0; m < node->pParameterList->length; ++m) { + complete = true; + for (int32_t m = 0; m < paramNum; ++m) { + if (NULL == params[m].columnData) { + complete = false; + continue; + } char* p = colDataGetData(params[m].columnData, i); GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p); if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { + complete = true; break; } else if (LOGIC_COND_TYPE_OR == node->condType && value) { + complete = true; break; } else if (LOGIC_COND_TYPE_NOT == node->condType) { value = !value; } } - colDataAppend(output->columnData, i, (char*) &value, false); + if (complete) { + colDataAppend(output->columnData, i, (char*) &value, false); + } + } + + if (SCL_IS_CONST_CALC(ctx) && (false == complete)) { + sclFreeParam(output); + output->numOfRows = 0; } _return: - for (int32_t i = 0; i < node->pParameterList->length; ++i) { + + for (int32_t i = 0; i < paramNum; ++i) { // sclFreeParamNoData(params + i); } @@ -426,6 +485,17 @@ _return: EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; + SNode* tnode = NULL; + if (!fmIsScalarFunc(node->funcId)) { + return DEAL_RES_CONTINUE; + } + + FOREACH(tnode, node->pParameterList) { + if (!SCL_IS_CONST_NODE(tnode)) { + return DEAL_RES_CONTINUE; + } + } + SScalarParam output = {0}; ctx->code = sclExecFunction(node, ctx, &output); @@ -441,14 +511,16 @@ EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + res->translate = true; + if (colDataIsNull_s(output.columnData, 0)) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else { res->node.resType = node->node.resType; int32_t type = output.columnData->info.type; if (IS_VAR_DATA_TYPE(type)) { - res->datum.p = output.columnData->pData; - output.columnData->pData = NULL; + res->datum.p = taosMemoryCalloc(res->node.resType.bytes + VARSTR_HEADER_SIZE + 1, 1); + memcpy(res->datum.p, output.columnData->pData, varDataTLen(output.columnData->pData)); } else { memcpy(nodesGetValueFromNode(res), output.columnData->pData, tDataTypes[type].bytes); } @@ -470,6 +542,10 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + if (0 == output.numOfRows) { + return DEAL_RES_CONTINUE; + } + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); @@ -479,6 +555,7 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { } res->node.resType = node->node.resType; + res->translate = true; int32_t type = output.columnData->info.type; if (IS_VAR_DATA_TYPE(type)) { @@ -498,6 +575,14 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; + if (!SCL_IS_CONST_NODE(node->pLeft)) { + return DEAL_RES_CONTINUE; + } + + if (!SCL_IS_CONST_NODE(node->pRight)) { + return DEAL_RES_CONTINUE; + } + SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; ctx->code = sclExecOperator(node, ctx, &output); if (ctx->code) { @@ -513,6 +598,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { } res->node.resType = node->node.resType; + res->translate = true; int32_t type = output.columnData->info.type; if (IS_VAR_DATA_TYPE(type)) { // todo refactor @@ -530,10 +616,6 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { } EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { - return DEAL_RES_CONTINUE; - } - SScalarCtx *ctx = (SScalarCtx *)pContext; if (QUERY_NODE_FUNCTION == nodeType(*pNode)) { @@ -547,10 +629,8 @@ EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { if (QUERY_NODE_OPERATOR == nodeType(*pNode)) { return sclRewriteOperator(pNode, ctx); } - - sclError("invalid node type for calculating constants, type:%d", nodeType(*pNode)); - ctx->code = TSDB_CODE_QRY_INVALID_INPUT; - return DEAL_RES_ERROR; + + return DEAL_RES_CONTINUE; } EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index cca7f1cbff..28514c3605 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -32,8 +32,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu float *in = (float *)pInputData->pData; float *out = (float *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -45,8 +45,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu double *in = (double *)pInputData->pData; double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -58,8 +58,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int8_t *in = (int8_t *)pInputData->pData; int8_t *out = (int8_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -71,8 +71,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int16_t *in = (int16_t *)pInputData->pData; int16_t *out = (int16_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -84,8 +84,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int32_t *in = (int32_t *)pInputData->pData; int32_t *out = (int32_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -97,8 +97,8 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int64_t *in = (int64_t *)pInputData->pData; int64_t *out = (int64_t *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = (in[i] >= 0)? in[i] : -in[i]; @@ -129,8 +129,8 @@ static int32_t doScalarFunctionUnique(SScalarParam *pInput, int32_t inputNum, SS double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = valFn(getValueFn(pInputData->pData, i)); @@ -157,9 +157,9 @@ static int32_t doScalarFunctionUnique2(SScalarParam *pInput, int32_t inputNum, S double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData[0]->nullbitmap, i) || - colDataIsNull_f(pInputData[1]->nullbitmap, 0)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData[0], i) || + colDataIsNull_s(pInputData[1], 0)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = valFn(getValueFn[0](pInputData[0]->pData, i), getValueFn[1](pInputData[1]->pData, 0)); @@ -184,8 +184,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP float *out = (float *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = f1(in[i]); @@ -198,8 +198,8 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP double *out = (double *)pOutputData->pData; for (int32_t i = 0; i < pInput->numOfRows; ++i) { - if (colDataIsNull_f(pInputData->nullbitmap, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); continue; } out[i] = d1(in[i]); @@ -301,7 +301,7 @@ static int32_t doLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarP for (int32_t i = 0; i < pInput->numOfRows; ++i) { if (colDataIsNull_s(pInputData, i)) { - colDataSetNull_f(pOutputData->nullbitmap, i); + colDataAppendNULL(pOutputData, i); continue; } @@ -1247,26 +1247,31 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum != 1) { - return TSDB_CODE_FAILED; + int64_t ts = taosGetTimestamp(TSDB_TIME_PRECISION_MILLI); + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + colDataAppendInt64(pOutput->columnData, i, &ts); } - colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0)); + pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum != 1) { - return TSDB_CODE_FAILED; + int64_t ts = taosGetTimestampToday(TSDB_TIME_PRECISION_MILLI); + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + colDataAppendInt64(pOutput->columnData, i, &ts); } - colDataAppendInt64(pOutput->columnData, pOutput->numOfRows, (int64_t *)colDataGetData(pInput->columnData, 0)); + pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } int32_t timezoneFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - if (inputNum != 1) { - return TSDB_CODE_FAILED; + char output[TD_TIMEZONE_LEN + VARSTR_HEADER_SIZE] = {0}; + memcpy(varDataVal(output), tsTimezoneStr, TD_TIMEZONE_LEN); + varDataSetLen(output, strlen(tsTimezoneStr)); + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + colDataAppend(pOutput->columnData, i, output, false); } - colDataAppend(pOutput->columnData, pOutput->numOfRows, (char *)colDataGetData(pInput->columnData, 0), false); + pOutput->numOfRows = pInput->numOfRows; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 04ab26d8a4..ed52e48f6d 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -137,6 +137,11 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->node.resType.bytes = dataBytes; rnode->dataBlockId = 0; + if (NULL == block) { + *pNode = (SNode *)rnode; + return; + } + if (NULL == *block) { SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; @@ -889,6 +894,8 @@ TEST(constantTest, int_greater_int_is_true2) { } TEST(constantTest, greater_and_lower) { + scltInitLogFile(); + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; bool eRes[5] = {false, false, true, true, true}; int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; @@ -913,6 +920,115 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } +TEST(constantTest, column_and_value1) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *v = (SLogicConditionNode *)res; + ASSERT_EQ(v->condType, LOGIC_COND_TYPE_AND); + ASSERT_EQ(v->pParameterList->length, 1); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value2) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value3) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value4) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *v = (SLogicConditionNode *)res; + ASSERT_EQ(v->condType, LOGIC_COND_TYPE_OR); + ASSERT_EQ(v->pParameterList->length, 1); + nodesDestroyNode(res); +} + + void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ char keyVar[32] = {0}; memcpy(varDataVal(keyVar), key, strlen(key)); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index c0b3ae7055..1aff0951e1 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -369,7 +369,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < childNum; ++n) { - SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); + SSubplan * child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); if (NULL == childTask || NULL == *childTask) { SCH_TASK_ELOG("subplan children relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -401,7 +401,7 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { } for (int32_t n = 0; n < parentNum; ++n) { - SSubplan *parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); + SSubplan * parent = (SSubplan *)nodesListGetNode(pPlan->pParents, n); SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -491,7 +491,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchLevel level = {0}; SNodeListNode *plans = NULL; int32_t taskNum = 0; - SSchLevel *pLevel = NULL; + SSchLevel * pLevel = NULL; level.status = JOB_TASK_STATUS_NOT_START; @@ -1267,7 +1267,7 @@ int32_t schUpdateTaskExecNodeHandle(SSchTask *pTask, void *handle, int32_t rspCo int32_t schHandleCallback(void *param, const SDataBuf *pMsg, int32_t msgType, int32_t rspCode) { int32_t code = 0; SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param; - SSchTask *pTask = NULL; + SSchTask * pTask = NULL; SSchJob *pJob = schAcquireJob(pParam->refId); if (NULL == pJob) { @@ -1617,8 +1617,8 @@ _return: int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx) { int32_t code = 0; SSchHbCallbackParam *param = NULL; - SMsgSendInfo *pMsgSendInfo = NULL; - SQueryNodeAddr *addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); + SMsgSendInfo * pMsgSendInfo = NULL; + SQueryNodeAddr * addr = taosArrayGet(pTask->candidateAddrs, pTask->candidateIdx); SQueryNodeEpId epId = {0}; epId.nodeId = addr->nodeId; @@ -1759,10 +1759,10 @@ int32_t schCloneHbRpcCtx(SRpcCtx *pSrc, SRpcCtx *pDst) { } SRpcCtxVal dst = {0}; - void *pIter = taosHashIterate(pSrc->args, NULL); + void * pIter = taosHashIterate(pSrc->args, NULL); while (pIter) { SRpcCtxVal *pVal = (SRpcCtxVal *)pIter; - int32_t *msgType = taosHashGetKey(pIter, NULL); + int32_t * msgType = taosHashGetKey(pIter, NULL); dst = *pVal; dst.val = NULL; @@ -1916,7 +1916,7 @@ _return: int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType) { uint32_t msgSize = 0; - void *msg = NULL; + void * msg = NULL; int32_t code = 0; bool isCandidateAddr = false; bool persistHandle = false; @@ -2673,7 +2673,7 @@ int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); for (int32_t m = 0; m < pLevel->taskNum; ++m) { - SSchTask *pTask = taosArrayGet(pLevel->subTasks, m); + SSchTask * pTask = taosArrayGet(pLevel->subTasks, m); SQuerySubDesc subDesc = {.tid = pTask->taskId, .status = pTask->status}; taosArrayPush(pSub, &subDesc); @@ -2734,7 +2734,7 @@ void schedulerFreeTaskList(SArray *taskList) { void schedulerDestroy(void) { if (schMgmt.jobRef) { SSchJob *pJob = taosIterateRef(schMgmt.jobRef, 0); - int64_t refId = 0; + int64_t refId = 0; while (pJob) { refId = pJob->refId; @@ -2751,12 +2751,12 @@ void schedulerDestroy(void) { } if (schMgmt.hbConnections) { - void *pIter = taosHashIterate(schMgmt.hbConnections, NULL); + void *pIter = taosHashIterate(schMgmt.hbConnections, NULL); while (pIter != NULL) { SSchHbTrans *hb = pIter; schFreeRpcCtx(&hb->rpcCtx); pIter = taosHashIterate(schMgmt.hbConnections, pIter); - } + } taosHashCleanup(schMgmt.hbConnections); schMgmt.hbConnections = NULL; } diff --git a/source/libs/sync/CMakeLists.txt b/source/libs/sync/CMakeLists.txt index cb196acc02..551849c6f2 100644 --- a/source/libs/sync/CMakeLists.txt +++ b/source/libs/sync/CMakeLists.txt @@ -11,7 +11,7 @@ target_link_libraries( target_include_directories( sync - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" + PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index b1d7af3254..deb158cbae 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -170,7 +170,6 @@ static int32_t syncIOStartInternal(SSyncIO *io) { taosBlockSIGPIPE(); rpcInit(); - tsRpcForceTcp = 1; // cient rpc init { @@ -448,4 +447,4 @@ static void syncIOTickPing(void *param, void *tmrId) { syncPingDestroy(pMsg); taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer); -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a3068d69a9..262c9d5530 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -457,6 +457,20 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { void syncNodeStart(SSyncNode* pSyncNode) { // start raft + if (pSyncNode->replicaNum == 1) { + syncNodeBecomeLeader(pSyncNode); + + syncNodeLog2("==state change become leader immediately==", pSyncNode); + + // Raft 3.6.2 Committing entries from previous terms + + // use this now + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica + return; + } + + syncNodeBecomeFollower(pSyncNode); // for test diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 3319a73fcf..8afe9ff2a7 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -199,197 +199,197 @@ target_sources(syncApplyMsgTest target_include_directories(syncTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEnvTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingTimerTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOTickQTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOTickPingTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOSendMsgTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOClientTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIOServerTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRaftStoreTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEnqTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIndexTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncInitTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncUtilTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncVotesGrantedTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncVotesRespondTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncIndexMgrTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncLogStoreTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEntryTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRequestVoteTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRequestVoteReplyTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncAppendEntriesTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncAppendEntriesReplyTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncClientRequestTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncTimeoutTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingReplyTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRpcMsgTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingTimerTest2 PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncPingSelfTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncElectTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncEncodeTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncWriteTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncReplicateTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRefTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncLogStoreCheck PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRaftCfgTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncRespMgrTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncSnapshotTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) target_include_directories(syncApplyMsgTest PUBLIC - "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) diff --git a/source/libs/sync/test/test.sh b/source/libs/sync/test/test.sh new file mode 100644 index 0000000000..7a693aac0b --- /dev/null +++ b/source/libs/sync/test/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + + diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index e18538cf27..5dab6f0a97 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -327,6 +327,10 @@ void transQueueClear(STransQueue* queue); */ void transQueueDestroy(STransQueue* queue); +/* + * init global func + */ +void transThreadOnce(); #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 9e53811fd3..ef595fb0ec 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -16,6 +16,8 @@ #include "transComm.h" +// static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; + int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { T_MD5_CTX context; int ret = -1; @@ -361,5 +363,10 @@ void transQueueDestroy(STransQueue* queue) { transQueueClear(queue); taosArrayDestroy(queue->q); } - +// int32_t transGetExHandle() { +// static +//} +// void transThreadOnce() { +// taosThreadOnce(&transModuleInit, ); +//} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 59a30051ef..c093bdebce 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -17,6 +17,11 @@ #include "transComm.h" +static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; + +static char* notify = "a"; +static int transSrvInst = 0; + typedef struct { int notifyCount; // int init; // init or not @@ -45,11 +50,12 @@ typedef struct SSrvConn { struct sockaddr_in addr; struct sockaddr_in locaddr; - int spi; - char info[64]; - char user[TSDB_UNI_LEN]; // user ID for the link - char secret[TSDB_PASSWORD_LEN]; - char ckey[TSDB_PASSWORD_LEN]; // ciphering key + int64_t refId; + int spi; + char info[64]; + char user[TSDB_UNI_LEN]; // user ID for the link + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; // ciphering key } SSrvConn; typedef struct SSrvMsg { @@ -89,7 +95,12 @@ typedef struct SServerObj { uv_async_t* pAcceptAsync; // just to quit from from accept thread } SServerObj; -static const char* notify = "a"; +// handle +typedef struct SExHandle { + void* handle; + int64_t refId; + SWorkThrdObj* pThrd; +} SExHandle; static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); @@ -130,6 +141,17 @@ static void uvHandleRegister(SSrvMsg* msg, SWorkThrdObj* thrd); static void (*transAsyncHandle[])(SSrvMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, uvHandleRegister}; +static int32_t exHandlesMgt; + +void uvInitExHandleMgt(); +void uvOpenExHandleMgt(int size); +void uvCloseExHandleMgt(); +int64_t uvAddExHandle(void* p); +int32_t uvRemoveExHandle(int64_t refId); +int32_t uvReleaseExHandle(int64_t refId); +void uvDestoryExHandle(void* handle); +SExHandle* uvAcquireExHandle(int64_t refId); + static void uvDestroyConn(uv_handle_t* handle); // server and worker thread @@ -168,6 +190,40 @@ static bool addHandleToAcceptloop(void* arg); uv_loop_close(loop); \ } while (0); +#define ASYNC_ERR_JRET(thrd) \ + do { \ + if (thrd->quit) { \ + tTrace("worker thread already quit, ignore msg"); \ + goto _return1; \ + } \ + } while (0) + +#define ASYNC_CHECK_HANDLE(exh1, refId) \ + do { \ + if (refId > 0) { \ + tTrace("server handle step1"); \ + SExHandle* exh2 = uvAcquireExHandle(refId); \ + if (exh2 == NULL || refId != exh2->refId) { \ + tTrace("server handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ + exh1->refId, refId); \ + goto _return1; \ + } \ + } else if (refId == 0) { \ + tTrace("server handle step2"); \ + SExHandle* exh2 = uvAcquireExHandle(refId); \ + if (exh2 == NULL || refId != exh2->refId) { \ + tTrace("server handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ + refId, exh2 ? exh2->refId : 0); \ + goto _return1; \ + } else { \ + refId = exh1->refId; \ + } \ + } else if (refId == -1) { \ + tTrace("server handle step3"); \ + goto _return2; \ + } \ + } while (0) + void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SSrvConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; @@ -233,7 +289,15 @@ static void uvHandleReq(SSrvConn* pConn) { // 1. server application should not send resp on handle // 2. once send out data, cli conn released to conn pool immediately // 3. not mixed with persist - transMsg.handle = pConn; + + transMsg.handle = (void*)uvAcquireExHandle(pConn->refId); + tTrace("server handle %p conn: %p translated to app, refId: %" PRIu64 "", transMsg.handle, pConn, pConn->refId); + transMsg.refId = pConn->refId; + assert(transMsg.handle != NULL); + if (pHead->noResp == 1) { + transMsg.refId = -1; + } + uvReleaseExHandle(pConn->refId); STrans* pTransInst = pConn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); @@ -404,6 +468,7 @@ static void destroySmsg(SSrvMsg* smsg) { taosMemoryFree(smsg); } static void destroyAllConn(SWorkThrdObj* pThrd) { + tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); QUEUE_REMOVE(h); @@ -436,7 +501,26 @@ void uvWorkerAsyncCb(uv_async_t* handle) { tError("unexcept occurred, continue"); continue; } - (*transAsyncHandle[msg->type])(msg, pThrd); + // release handle to rpc init + if (msg->type == Quit) { + (*transAsyncHandle[msg->type])(msg, pThrd); + continue; + } else { + STransMsg transMsg = msg->msg; + + SExHandle* exh1 = transMsg.handle; + int64_t refId = transMsg.refId; + SExHandle* exh2 = uvAcquireExHandle(refId); + if (exh2 == NULL || exh1 != exh2) { + tTrace("server handle %p except msg, ignore it", exh1); + uvReleaseExHandle(refId); + destroySmsg(msg); + continue; + } + msg->pConn = exh1->handle; + uvReleaseExHandle(refId); + (*transAsyncHandle[msg->type])(msg, pThrd); + } } } static void uvWalkCb(uv_handle_t* handle, void* arg) { @@ -658,8 +742,15 @@ static SSrvConn* createConn(void* hThrd) { pConn->broken = false; pConn->status = ConnNormal; + SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle)); + exh->handle = pConn; + exh->pThrd = pThrd; + exh->refId = uvAddExHandle(exh); + uvAcquireExHandle(exh->refId); + + pConn->refId = exh->refId; transRefSrvHandle(pConn); - tTrace("server conn %p created", pConn); + tTrace("server handle %p, conn %p created, refId: %" PRId64 "", exh, pConn, pConn->refId); return pConn; } @@ -667,6 +758,7 @@ static void destroyConn(SSrvConn* conn, bool clear) { if (conn == NULL) { return; } + transDestroyBuffer(&conn->readBuf); if (clear) { tTrace("server conn %p to be destroyed", conn); @@ -681,6 +773,9 @@ static void uvDestroyConn(uv_handle_t* handle) { } SWorkThrdObj* thrd = conn->hostThrd; + uvReleaseExHandle(conn->refId); + uvRemoveExHandle(conn->refId); + tDebug("server conn %p destroy", conn); // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); @@ -705,6 +800,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->port = port; uv_loop_init(srv->loop); + taosThreadOnce(&transModuleInit, uvInitExHandleMgt); + transSrvInst++; + // uvOpenExHandleMgt(10000); + for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); thrd->quit = false; @@ -749,6 +848,44 @@ End: transCloseServer(srv); return NULL; } + +void uvInitExHandleMgt() { + // init exhandle mgt + uvOpenExHandleMgt(10000); +} +void uvOpenExHandleMgt(int size) { + // added into once later + exHandlesMgt = taosOpenRef(size, uvDestoryExHandle); +} +void uvCloseExHandleMgt() { + // close ref + taosCloseRef(exHandlesMgt); +} +int64_t uvAddExHandle(void* p) { + // acquire extern handle + return taosAddRef(exHandlesMgt, p); +} +int32_t uvRemoveExHandle(int64_t refId) { + // acquire extern handle + return taosRemoveRef(exHandlesMgt, refId); +} + +SExHandle* uvAcquireExHandle(int64_t refId) { + // acquire extern handle + return (SExHandle*)taosAcquireRef(exHandlesMgt, refId); +} + +int32_t uvReleaseExHandle(int64_t refId) { + // release extern handle + return taosReleaseRef(exHandlesMgt, refId); +} +void uvDestoryExHandle(void* handle) { + if (handle == NULL) { + return; + } + taosMemoryFree(handle); +} + void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { @@ -759,7 +896,6 @@ void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { taosMemoryFree(msg); } void uvHandleRelease(SSrvMsg* msg, SWorkThrdObj* thrd) { - // release handle to rpc init SSrvConn* conn = msg->pConn; if (conn->status == ConnAcquire) { if (!transQueuePush(&conn->srvMsgs, msg)) { @@ -840,6 +976,12 @@ void transCloseServer(void* arg) { taosMemoryFree(srv->pipe); taosMemoryFree(srv); + + transSrvInst--; + if (transSrvInst == 0) { + transModuleInit = PTHREAD_ONCE_INIT; + uvCloseExHandleMgt(); + } } void transRefSrvHandle(void* handle) { @@ -862,57 +1004,98 @@ void transUnrefSrvHandle(void* handle) { } void transReleaseSrvHandle(void* handle) { - if (handle == NULL) { - return; - } - SSrvConn* pConn = handle; - SWorkThrdObj* pThrd = pConn->hostThrd; + SExHandle* exh = handle; + int64_t refId = exh->refId; - STransMsg tmsg = {.code = 0, .handle = handle, .ahandle = NULL}; + ASYNC_CHECK_HANDLE(exh, refId); + + SWorkThrdObj* pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); + + STransMsg tmsg = {.code = 0, .handle = exh, .ahandle = NULL, .refId = refId}; SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); srvMsg->msg = tmsg; srvMsg->type = Release; - srvMsg->pConn = pConn; - tTrace("server conn %p start to release", pConn); + tTrace("server conn %p start to release", exh->handle); transSendAsync(pThrd->asyncPool, &srvMsg->q); + uvReleaseExHandle(refId); + return; +_return1: + tTrace("server handle %p failed to send to release handle", exh); + uvReleaseExHandle(refId); + return; +_return2: + tTrace("server handle %p failed to send to release handle", exh); + return; } -void transSendResponse(const STransMsg* pMsg) { - if (pMsg->handle == NULL) { - return; - } - SSrvConn* pConn = pMsg->handle; - SWorkThrdObj* pThrd = pConn->hostThrd; - if (pThrd->quit) { - return; - } +void transSendResponse(const STransMsg* msg) { + SExHandle* exh = msg->handle; + int64_t refId = msg->refId; + ASYNC_CHECK_HANDLE(exh, refId); + assert(refId != 0); + + STransMsg tmsg = *msg; + tmsg.refId = refId; + + SWorkThrdObj* pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); - srvMsg->pConn = pConn; - srvMsg->msg = *pMsg; + srvMsg->msg = tmsg; srvMsg->type = Normal; - tTrace("server conn %p start to send resp (1/2)", pConn); + tTrace("server conn %p start to send resp (1/2)", exh->handle); transSendAsync(pThrd->asyncPool, &srvMsg->q); + uvReleaseExHandle(refId); + return; +_return1: + tTrace("server handle %p failed to send resp", exh); + rpcFreeCont(msg->pCont); + uvReleaseExHandle(refId); + return; +_return2: + tTrace("server handle %p failed to send resp", exh); + rpcFreeCont(msg->pCont); + return; } void transRegisterMsg(const STransMsg* msg) { - if (msg->handle == NULL) { - return; - } - SSrvConn* pConn = msg->handle; - SWorkThrdObj* pThrd = pConn->hostThrd; + SExHandle* exh = msg->handle; + int64_t refId = msg->refId; + ASYNC_CHECK_HANDLE(exh, refId); + + STransMsg tmsg = *msg; + tmsg.refId = refId; + + SWorkThrdObj* pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); SSrvMsg* srvMsg = taosMemoryCalloc(1, sizeof(SSrvMsg)); - srvMsg->pConn = pConn; - srvMsg->msg = *msg; + srvMsg->msg = tmsg; srvMsg->type = Register; - tTrace("server conn %p start to register brokenlink callback", pConn); + tTrace("server conn %p start to register brokenlink callback", exh->handle); transSendAsync(pThrd->asyncPool, &srvMsg->q); + uvReleaseExHandle(refId); + return; +_return1: + tTrace("server handle %p failed to send to register brokenlink", exh); + rpcFreeCont(msg->pCont); + uvReleaseExHandle(refId); + return; +_return2: + tTrace("server handle %p failed to send to register brokenlink", exh); + rpcFreeCont(msg->pCont); } -int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { - SSrvConn* pConn = thandle; - struct sockaddr_in addr = pConn->addr; +int transGetConnInfo(void* thandle, STransHandleInfo* pInfo) { + if (thandle == NULL) { + tTrace("invalid handle %p, failed to Get Conn info", thandle); + return -1; + } + SExHandle* ex = thandle; + SSrvConn* pConn = ex->handle; + + struct sockaddr_in addr = pConn->addr; pInfo->clientIp = (uint32_t)(addr.sin_addr.s_addr); pInfo->clientPort = ntohs(addr.sin_port); tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 6d598699c5..70a3559cd9 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -138,6 +138,91 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { return 0; } +void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; } + +int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { + int32_t code; + // TODO: valid ver + + if (pRead->curVersion != ver) { + code = walReadSeekVer(pRead, ver); + if (code < 0) return -1; + } + + if (!taosValidFile(pRead->pReadLogTFile)) { + return -1; + } + + code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalHead)); + if (code != sizeof(SWalHead)) { + return -1; + } + + code = walValidHeadCksum(pHead); + + if (code != 0) { + wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } + + return 0; +} + +int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead) { + int32_t code; + + ASSERT(pRead->curVersion == pHead->head.version); + + code = taosLSeekFile(pRead->pReadLogTFile, pHead->head.bodyLen, SEEK_CUR); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + pRead->curVersion = -1; + return -1; + } + + pRead->curVersion++; + + return 0; +} + +int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead) { + SWalReadHead *pReadHead = &((*ppHead)->head); + int64_t ver = pReadHead->version; + + if (pRead->capacity < pReadHead->bodyLen) { + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + pReadHead->bodyLen); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + return -1; + } + *ppHead = ptr; + pRead->capacity = pReadHead->bodyLen; + } + + if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) { + return -1; + } + + if (pReadHead->version != ver) { + wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, + ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } + + if (walValidBodyCksum(*ppHead) != 0) { + wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } + + pRead->curVersion = ver + 1; + return 0; +} + int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead) { taosThreadMutexLock(&pRead->mutex); if (walReadWithHandle(pRead, ver) < 0) { @@ -172,12 +257,14 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { if (code != sizeof(SWalHead)) { return -1; } + code = walValidHeadCksum(pRead->pHead); if (code != 0) { wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } + if (pRead->capacity < pRead->pHead->head.bodyLen) { void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.bodyLen); if (ptr == NULL) { diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 530930c261..207b7d8421 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE - #include "os.h" #include "taoserror.h" #include "tchecksum.h" @@ -298,14 +296,14 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; - // sync info + // sync info for sync module pWal->writeHead.head.syncMeta = syncMeta; pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { - // ftruncate + // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -313,7 +311,7 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog } if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { - // ftruncate + // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index cf3c95e658..148529170c 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -119,7 +119,7 @@ int taosSetConsoleEcho(bool on) { #endif } -void setTerminalMode() { +void taosSetTerminalMode() { #if defined(WINDOWS) #else @@ -152,7 +152,7 @@ void setTerminalMode() { #endif } -int32_t getOldTerminalMode() { +int32_t taosGetOldTerminalMode() { #if defined(WINDOWS) #else @@ -170,7 +170,7 @@ int32_t getOldTerminalMode() { #endif } -void resetTerminalMode() { +void taosResetTerminalMode() { #if defined(WINDOWS) #else diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 1c655fc2bf..2d741b18f6 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -476,6 +476,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t return (void*)buf; } +// todo remove it // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 6f6ae5d699..ee4236dce5 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -18,6 +18,9 @@ #include "taoserror.h" #include "tlog.h" #include "tutil.h" +#include "tenv.h" +#include "cJSON.h" +#include "tjson.h" #define CFG_NAME_PRINT_LEN 24 #define CFG_SRC_PRINT_LEN 12 @@ -25,6 +28,7 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath); int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath); int32_t cfgLoadFromEnvVar(SConfig *pConfig); +int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd); int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url); int32_t cfgSetItem(SConfig *pConfig, const char *name, const char *value, ECfgSrcType stype); @@ -45,7 +49,7 @@ SConfig *cfgInit() { return pCfg; } -int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { +int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr) { switch (cfgType) { case CFG_STYPE_CFG_FILE: return cfgLoadFromCfgFile(pCfg, sourceStr); @@ -55,6 +59,8 @@ int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr) { return cfgLoadFromEnvVar(pCfg); case CFG_STYPE_APOLLO_URL: return cfgLoadFromApollUrl(pCfg, sourceStr); + case CFG_STYPE_ENV_CMD: + return cfgLoadFromEnvCmd(pCfg, (const char **)sourceStr); default: return -1; } @@ -463,6 +469,8 @@ const char *cfgStypeStr(ECfgSrcType type) { return "arg_list"; case CFG_STYPE_TAOS_OPTIONS: return "taos_options"; + case CFG_STYPE_ENV_CMD: + return "env_cmd"; default: return "invalid"; } @@ -513,6 +521,8 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { for (int32_t i = 0; i < size; ++i) { SConfigItem *pItem = taosArrayGet(pCfg->array, i); if (tsc && !pItem->tsc) continue; + if (dump && strcmp(pItem->name, "scriptDir") == 0) continue; + if (dump && strcmp(pItem->name, "simDebugFlag") == 0) continue; tstrncpy(src, cfgStypeStr(pItem->stype), CFG_SRC_PRINT_LEN); for (int32_t i = 0; i < CFG_SRC_PRINT_LEN; ++i) { if (src[i] == 0) src[i] = ' '; @@ -551,10 +561,10 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { break; case CFG_DTYPE_FLOAT: if (dump) { - printf("%s %s %f", src, name, pItem->fval); + printf("%s %s %.2f", src, name, pItem->fval); printf("\n"); } else { - uInfo("%s %s %f", src, name, pItem->fval); + uInfo("%s %s %.2f", src, name, pItem->fval); } break; case CFG_DTYPE_STRING: @@ -582,12 +592,154 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { } int32_t cfgLoadFromEnvVar(SConfig *pConfig) { - uDebug("load from env variables not implemented yet"); + char *line = NULL, *name, *value, *value2, *value3; + int32_t olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; + TdCmdPtr pCmd = taosOpenCmd("set"); + if (pCmd == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + while (!taosEOFCmd(pCmd)) { + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; + + _bytes = taosGetLineCmd(pCmd, &line); + if (_bytes < 0) { + break; + } + if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; + taosEnvToCfg(line, line); + + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_VAR); + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_VAR); + } + } + + taosCloseCmd(&pCmd); + if (line != NULL) taosMemoryFreeClear(line); + + uInfo("load from env variables cfg success"); return 0; } -int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { - uDebug("load from env file not implemented yet"); +int32_t cfgLoadFromEnvCmd(SConfig *pConfig, const char **envCmd) { + char *buf, *name, *value, *value2, *value3; + int32_t olen, vlen, vlen2, vlen3; + int32_t index = 0; + if (envCmd == NULL) return 0; + while (envCmd[index]!=NULL) { + buf = taosMemoryMalloc(strlen(envCmd[index])); + taosEnvToCfg(envCmd[index], buf); + index++; + + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; + + paGetToken(buf, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_CMD); + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_CMD); + } + + taosMemoryFree(buf); + } + + uInfo("load from env cmd cfg success"); + return 0; +} + +int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *envFile) { + char *line = NULL, *name, *value, *value2, *value3; + int32_t olen, vlen, vlen2, vlen3; + ssize_t _bytes = 0; + + const char *filepath = ".env"; + if (envFile != NULL && strlen(envFile)>0) { + if (!taosCheckExistFile(envFile)) { + uError("fial to load env file: %s", envFile); + return -1; + } + filepath = envFile; + }else { + if (!taosCheckExistFile(filepath)) { + uInfo("fial to load env file: %s", filepath); + return 0; + } + } + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + while (!taosEOFFile(pFile)) { + name = value = value2 = value3 = NULL; + olen = vlen = vlen2 = vlen3 = 0; + + _bytes = taosGetLineFile(pFile, &line); + if (_bytes <= 0) { + break; + } + if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; + taosEnvToCfg(line, line); + + paGetToken(line, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + + cfgSetItem(pConfig, name, value, CFG_STYPE_ENV_FILE); + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_ENV_FILE); + } + } + + taosCloseFile(&pFile); + if (line != NULL) taosMemoryFreeClear(line); + + uInfo("load from env cfg file %s success", filepath); return 0; } @@ -615,11 +767,11 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { olen = vlen = vlen2 = vlen3 = 0; _bytes = taosGetLineFile(pFile, &line); - if (_bytes < 0) { + if (_bytes <= 0) { break; } - line[_bytes - 1] = 0; + if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; paGetToken(line, &name, &olen); if (olen == 0) continue; @@ -657,6 +809,192 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { } int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { - uDebug("load from apoll url not implemented yet"); + char *cfgLineBuf = NULL, *name, *value, *value2, *value3; + int32_t olen, vlen, vlen2, vlen3; + if (url == NULL || strlen(url) == 0) { + uInfo("fail to load apoll url"); + return 0; + } + + char *p = strchr(url, ':'); + if (p == NULL) { + uError("fail to load apoll url: %s, unknown format", url); + return -1; + } + p++; + + if (bcmp(url, "jsonFile", 8) == 0) { + char *filepath = p; + if (!taosCheckExistFile(filepath)) { + uError("fial to load json file: %s", filepath); + return -1; + } + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + size_t fileSize = taosLSeekFile(pFile, 0, SEEK_END); + char *buf = taosMemoryMalloc(fileSize); + taosLSeekFile(pFile, 0, SEEK_SET); + if(taosReadFile(pFile, buf, fileSize) <= 0) { + taosCloseFile(&pFile); + uError("load json file error: %s", filepath); + return -1; + } + taosCloseFile(&pFile); + SJson* pJson = tjsonParse(buf); + if (NULL == pJson) { + const char *jsonParseError = tjsonGetError(); + if (jsonParseError != NULL) { + uError("load json file parse error: %s", jsonParseError); + } + return -1; + } + taosMemoryFreeClear(buf); + + int32_t jsonArraySize = tjsonGetArraySize(pJson); + for(int32_t i = 0; i < jsonArraySize; i++) { + cJSON* item = tjsonGetArrayItem(pJson, i); + if (item == NULL) break; + char *itemName = NULL, *itemValueString = NULL; + tjsonGetObjectName(item, &itemName); + tjsonGetObjectName(item, &itemName); + tjsonGetObjectValueString(item, &itemValueString); + if (itemValueString != NULL && itemName != NULL) { + size_t itemNameLen = strlen(itemName); + size_t itemValueStringLen = strlen(itemValueString); + cfgLineBuf = taosMemoryMalloc(itemNameLen + itemValueStringLen + 2); + memcpy(cfgLineBuf, itemName, itemNameLen); + cfgLineBuf[itemNameLen] = ' '; + memcpy(&cfgLineBuf[itemNameLen+1], itemValueString, itemValueStringLen); + cfgLineBuf[itemNameLen + itemValueStringLen + 1] = '\0'; + + paGetToken(cfgLineBuf, &name, &olen); + if (olen == 0) continue; + name[olen] = 0; + + paGetToken(name + olen + 1, &value, &vlen); + if (vlen == 0) continue; + value[vlen] = 0; + + paGetToken(value + vlen + 1, &value2, &vlen2); + if (vlen2 != 0) { + value2[vlen2] = 0; + paGetToken(value2 + vlen2 + 1, &value3, &vlen3); + if (vlen3 != 0) value3[vlen3] = 0; + } + cfgSetItem(pConfig, name, value, CFG_STYPE_APOLLO_URL); + if (value2 != NULL && value3 != NULL && value2[0] != 0 && value3[0] != 0 && strcasecmp(name, "dataDir") == 0) { + cfgSetTfsItem(pConfig, name, value, value2, value3, CFG_STYPE_APOLLO_URL); + } + } + } + tjsonDelete(pJson); + + // } else if (bcmp(url, "jsonUrl", 7) == 0) { + // } else if (bcmp(url, "etcdUrl", 7) == 0) { + } else { + uError("Unsupported url: %s", url); + return -1; + } + + uInfo("load from apoll url not implemented yet"); return 0; } + +int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl) { + int32_t index = 0; + if (envCmd == NULL) return 0; + while (envCmd[index]!=NULL) { + if (bcmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { + char *p = strchr(envCmd[index], '='); + if (p != NULL) { + p++; + if (*p == '\'') { + p++; + p[strlen(p)-1] = '\0'; + } + memcpy(apolloUrl, p, TMIN(strlen(p)+1,PATH_MAX)); + uInfo("get apollo url from env cmd success"); + return 0; + } + } + index++; + } + + char *line = NULL; + ssize_t _bytes = 0; + TdCmdPtr pCmd = taosOpenCmd("set"); + if (pCmd != NULL) { + while (!taosEOFCmd(pCmd)) { + _bytes = taosGetLineCmd(pCmd, &line); + if (_bytes < 0) { + break; + } + if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; + if (bcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + char *p = strchr(line, '='); + if (p != NULL) { + p++; + if (*p == '\'') { + p++; + p[strlen(p)-1] = '\0'; + } + memcpy(apolloUrl, p, TMIN(strlen(p)+1,PATH_MAX)); + uInfo("get apollo url from env variables success, apolloUrl=%s",apolloUrl); + taosCloseCmd(&pCmd); + if (line != NULL) taosMemoryFreeClear(line); + return 0; + } + } + } + taosCloseCmd(&pCmd); + if (line != NULL) taosMemoryFreeClear(line); + } + + const char *filepath = ".env"; + if (envFile != NULL && strlen(envFile)>0) { + if (!taosCheckExistFile(envFile)) { + uError("fial to load env file: %s", envFile); + return -1; + } + filepath = envFile; + }else { + if (!taosCheckExistFile(filepath)) { + uInfo("fial to load env file: %s", filepath); + return 0; + } + } + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_STREAM); + if (pFile != NULL) { + while (!taosEOFFile(pFile)) { + _bytes = taosGetLineFile(pFile, &line); + if (_bytes <= 0) { + break; + } + if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; + if (bcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + char *p = strchr(line, '='); + if (p != NULL) { + p++; + if (*p == '\'') { + p++; + p[strlen(p)-1] = '\0'; + } + memcpy(apolloUrl, p, TMIN(strlen(p)+1,PATH_MAX)); + taosCloseFile(&pFile); + if (line != NULL) taosMemoryFreeClear(line); + uInfo("get apollo url from env file success"); + return 0; + } + } + } + taosCloseFile(&pFile); + if (line != NULL) taosMemoryFreeClear(line); + } + + uInfo("fail get apollo url from cmd env file"); + return -1; +} \ No newline at end of file diff --git a/source/util/src/tenv.c b/source/util/src/tenv.c new file mode 100644 index 0000000000..e717e82c5b --- /dev/null +++ b/source/util/src/tenv.c @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tenv.h" +#include "tconfig.h" + +static char toLowChar(char c) { return (c > 'Z' || c < 'A' ? c : (c - 'A' + 'a')); } + +int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t cfgNameMaxLen) { + if (envNameStr == NULL || cfgNameStr == NULL) return -1; + char *p = cfgNameStr; + if (envNameStr[0] != 'T' || envNameStr[1] != 'A' || envNameStr[2] != 'O' || envNameStr[3] != 'S' || + envNameStr[4] != '_') { + // if(p != envNameStr) strncpy(p, envNameStr, cfgNameMaxLen - 1); + // p[cfgNameMaxLen - 1] = '\0'; + // return strlen(cfgNameStr); + cfgNameStr[0] = '\0'; + return -1; + } + envNameStr += 5; + if (*envNameStr != '\0') { + *p = toLowChar(*envNameStr); + p++; + envNameStr++; + } + + for (size_t i = 1; i < cfgNameMaxLen && *envNameStr != '\0'; i++) { + if (*envNameStr == '_') { + envNameStr++; + *p = *envNameStr; + if (*envNameStr == '\0') break; + } else { + *p = toLowChar(*envNameStr); + } + p++; + envNameStr++; + } + + *p = '\0'; + return strlen(cfgNameStr); +} + +int32_t taosEnvToCfg(const char *envStr, char *cfgStr) { + if (envStr == NULL && cfgStr == NULL) { + return -1; + } + if (cfgStr != envStr) strcpy(cfgStr, envStr); + char *p = strchr(cfgStr, '='); + + if (p != NULL) { + char buf[CFG_NAME_MAX_LEN]; + if (*(p+1) == '\'') { + *(p+1)= ' '; + char *pEnd = &cfgStr[strlen(cfgStr)-1]; + if (*pEnd == '\'') *pEnd = '\0'; + } + *p = '\0'; + int32_t cfgNameLen = taosEnvNameToCfgName(cfgStr, buf, CFG_NAME_MAX_LEN); + if (cfgNameLen > 0) { + memcpy(cfgStr, buf, cfgNameLen); + memset(&cfgStr[cfgNameLen], ' ', p - cfgStr - cfgNameLen + 1); + } + } + return strlen(cfgStr); +} \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9332cb481e..399a2255ac 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -258,7 +258,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") // mnode-infoSchema -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_INFOS_TBL, "Invalid information schema table name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") // mnode-func @@ -403,6 +403,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CHECKSUM, "Invalid msg checksum" TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGLEN, "Invalid msg length") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_MSGTYPE, "Invalid msg type") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync not leader") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") + // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, "Unexpected generic error in wal") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index 0b9727e0bf..558c2258f4 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -144,6 +144,22 @@ char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatt SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName) { return cJSON_GetObjectItem(pJson, pName); } +int32_t tjsonGetObjectName(const SJson* pJson, char** pName) { + *pName = ((cJSON*)pJson)->string; + if (NULL == *pName) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +int32_t tjsonGetObjectValueString(const SJson* pJson, char** pValueString) { + *pValueString = ((cJSON*)pJson)->valuestring; + if (NULL == *pValueString) { + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal) { char* p = cJSON_GetStringValue(tjsonGetObjectItem((cJSON*)pJson, pName)); if (NULL == p) { @@ -309,4 +325,6 @@ bool tjsonValidateJson(const char *jIn) { } } return true; -} \ No newline at end of file +} + +const char* tjsonGetError() { return cJSON_GetErrorPtr(); } \ No newline at end of file diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 03c9c255b0..d8343bc427 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -154,7 +154,8 @@ static void taosProcCleanupQueue(SProcQueue *pQueue) { } static int32_t taosProcQueuePush(SProcObj *pProc, SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, - const char *pBody, int32_t rawBodyLen, int64_t handle, EProcFuncType ftype) { + const char *pBody, int32_t rawBodyLen, int64_t handle, int64_t handleRef, + EProcFuncType ftype) { if (rawHeadLen == 0 || pHead == NULL) { terrno = TSDB_CODE_INVALID_PARA; return -1; @@ -172,7 +173,7 @@ static int32_t taosProcQueuePush(SProcObj *pProc, SProcQueue *pQueue, const char } if (handle != 0 && ftype == PROC_FUNC_REQ) { - if (taosHashPut(pProc->hash, &handle, sizeof(int64_t), &handle, sizeof(int64_t)) != 0) { + if (taosHashPut(pProc->hash, &handle, sizeof(int64_t), &handleRef, sizeof(int64_t)) != 0) { taosThreadMutexUnlock(&pQueue->mutex); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -286,13 +287,13 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea pQueue->head = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8); - memcpy((char*)pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); + memcpy((char *)pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen); pQueue->head = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen); - memcpy((char*)pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); + memcpy((char *)pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); pQueue->head = bodyLen - (remain - 8 - headLen); } else { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); @@ -454,19 +455,25 @@ void taosProcCleanup(SProcObj *pProc) { } int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, - void *handle, EProcFuncType ftype) { + void *handle, int64_t handleRef, EProcFuncType ftype) { if (ftype != PROC_FUNC_REQ) { terrno = TSDB_CODE_INVALID_PARA; return -1; } - return taosProcQueuePush(pProc, pProc->pChildQueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, ftype); + return taosProcQueuePush(pProc, pProc->pChildQueue, pHead, headLen, pBody, bodyLen, (int64_t)handle, handleRef, + ftype); } -void taosProcRemoveHandle(SProcObj *pProc, void *handle) { +int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle) { int64_t h = (int64_t)handle; taosThreadMutexLock(&pProc->pChildQueue->mutex); + + int64_t *handleRef = taosHashGet(pProc->hash, &h, sizeof(int64_t)); taosHashRemove(pProc->hash, &h, sizeof(int64_t)); taosThreadMutexUnlock(&pProc->pChildQueue->mutex); + + if (handleRef == NULL) return 0; + return *handleRef; } void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)) { @@ -484,7 +491,7 @@ void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)) { void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, EProcFuncType ftype) { int32_t retry = 0; - while (taosProcQueuePush(pProc, pProc->pParentQueue, pHead, headLen, pBody, bodyLen, 0, ftype) != 0) { + while (taosProcQueuePush(pProc, pProc->pParentQueue, pHead, headLen, pBody, bodyLen, 0, 0, ftype) != 0) { uWarn("proc:%s, failed to put to queue:%p since %s, retry:%d", pProc->name, pProc->pParentQueue, terrstr(), retry); retry++; taosMsleep(retry); diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index 34a59a7d48..e06d7d8c89 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -110,7 +110,7 @@ typedef struct time_wheel_t { tmr_obj_t** slots; } time_wheel_t; -int32_t tsMaxTmrCtrl = 512; +static int32_t tsMaxTmrCtrl = 512; static TdThreadOnce tmrModuleInit = PTHREAD_ONCE_INIT; static TdThreadMutex tmrCtrlMutex; diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index d98f9f677d..3c52d52d50 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -31,7 +31,7 @@ ENDIF() # TARGET_LINK_LIBRARIES(trefTest util common) #ENDIF () -INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_SOURCE_DIR}/src/util/inc) # freelistTest add_executable(freelistTest "") diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index 712fb2d09a..cd13ebe8ae 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -32,6 +32,7 @@ TEST_F(CfgTest, 01_Str) { EXPECT_STREQ(cfgStypeStr(CFG_STYPE_CFG_FILE), "cfg_file"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_FILE), "env_file"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_VAR), "env_var"); + EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ENV_CMD), "env_cmd"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_APOLLO_URL), "apollo_url"); EXPECT_STREQ(cfgStypeStr(CFG_STYPE_ARG_LIST), "arg_list"); EXPECT_STREQ(cfgStypeStr(ECfgSrcType(1024)), "invalid"); diff --git a/source/util/test/procTest.cpp b/source/util/test/procTest.cpp index 3c014369fb..7ffec04a40 100644 --- a/source/util/test/procTest.cpp +++ b/source/util/test/procTest.cpp @@ -120,20 +120,20 @@ TEST_F(UtilTesProc, 01_Push_Pop_Child) { SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); - ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_RSP), 0); - ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_REGIST), 0); - ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_RELEASE), 0); - ASSERT_NE(taosProcPutToChildQ(cproc, NULL, 12, body, 0, 0, PROC_FUNC_REQ), 0); - ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, PROC_FUNC_REQ), 0); - ASSERT_NE(taosProcPutToChildQ(cproc, &head, shm.size, body, 0, 0, PROC_FUNC_REQ), 0); - ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, shm.size, 0, PROC_FUNC_REQ), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_RSP), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_REGIST), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_RELEASE), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, NULL, 12, body, 0, 0, 0, PROC_FUNC_REQ), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, &head, 0, body, 0, 0, 0, PROC_FUNC_REQ), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, &head, shm.size, body, 0, 0, 0, PROC_FUNC_REQ), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, shm.size, 0, 0, PROC_FUNC_REQ), 0); for (int32_t j = 0; j < 1000; j++) { int32_t i = 0; for (i = 0; i < 20; ++i) { - ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, PROC_FUNC_REQ), 0); + ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, 0, PROC_FUNC_REQ), 0); } - ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, PROC_FUNC_REQ), 0); + ASSERT_NE(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, 0, 0, PROC_FUNC_REQ), 0); cfg.isChild = true; cfg.name = "1235_p"; @@ -236,7 +236,7 @@ TEST_F(UtilTesProc, 03_Handle) { int32_t i = 0; for (i = 0; i < 20; ++i) { head.handle = (void *)((int64_t)i); - ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, (void *)((int64_t)i), PROC_FUNC_REQ), 0); + ASSERT_EQ(taosProcPutToChildQ(cproc, &head, sizeof(STestMsg), body, i, (void *)((int64_t)i), i, PROC_FUNC_REQ), 0); } cfg.isChild = true; @@ -246,9 +246,14 @@ TEST_F(UtilTesProc, 03_Handle) { taosProcRun(pproc); taosProcCleanup(pproc); - taosProcRemoveHandle(cproc, (void *)((int64_t)3)); - taosProcRemoveHandle(cproc, (void *)((int64_t)5)); - taosProcRemoveHandle(cproc, (void *)((int64_t)6)); + int64_t ref = 0; + + ref = taosProcRemoveHandle(cproc, (void *)((int64_t)3)); + EXPECT_EQ(ref, 3); + ref = taosProcRemoveHandle(cproc, (void *)((int64_t)5)); + EXPECT_EQ(ref, 5); + ref = taosProcRemoveHandle(cproc, (void *)((int64_t)6)); + EXPECT_EQ(ref, 6); taosProcCloseHandles(cproc, processHandle); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 31b1fff826..83bd016d2f 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -76,7 +76,7 @@ ./test.sh -f tsim/insert/backquote.sim -m ./test.sh -f tsim/parser/fourArithmetic-basic.sim -m ./test.sh -f tsim/query/interval-offset.sim -m -./test.sh -f tsim/tmq/basic1.sim -m +#./test.sh -f tsim/tmq/basic1.sim -m ./test.sh -f tsim/stable/vnode3.sim -m ./test.sh -f tsim/qnode/basic1.sim -m ./test.sh -f tsim/mnode/basic1.sim -m @@ -85,6 +85,6 @@ ./test.sh -f tsim/sma/tsmaCreateInsertData.sim # --- valgrind -./test.sh -f tsim/valgrind/checkError.sim -v +#./test.sh -f tsim/valgrind/checkError.sim -v #======================b1-end=============== diff --git a/tests/script/sh/massiveTable/compileVersion.sh b/tests/script/sh/massiveTable/compileVersion.sh index dcd1f8aeea..6b45e98ea3 100755 --- a/tests/script/sh/massiveTable/compileVersion.sh +++ b/tests/script/sh/massiveTable/compileVersion.sh @@ -57,7 +57,8 @@ function compileTDengineVersion() { cd ${debugDir} cmake .. - make -j24 + make -j24 + make install } ######################################################################################## ############################### main process ########################################## @@ -67,20 +68,5 @@ cd ${projectDir} gitPullBranchInfo $TDengineBrVer compileTDengineVersion -taos_dir=${projectDir}/debug/build/bin -taosd_dir=${projectDir}/debug/build/bin -exec_process_dir=${projectDir}/debug/build/bin - -rm -f /usr/bin/taos -rm -f /usr/bin/taosd -rm -f /usr/bin/create_table -rm -f /usr/bin/tmq_demo -rm -f /usr/bin/tmq_sim - -ln -s $taos_dir/taos /usr/bin/taos -ln -s $taosd_dir/taosd /usr/bin/taosd -ln -s $exec_process_dir/create_table /usr/bin/create_table -ln -s $exec_process_dir/tmq_demo /usr/bin/tmq_demo -ln -s $exec_process_dir/tmq_sim /usr/bin/tmq_sim diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index f073200a05..4d6b33612f 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -115,19 +115,19 @@ endi if $data00 != 10 then return -1 endi -if $data01 != 0 then - return -1 -endi if $data10 != 10 then return -1 endi -if $data11 != 1 then - return -1 -endi if $data90 != 10 then return -1 endi -if $data91 != 9 then +if $data01 != 7 then + return -1 +endi +if $data11 != 6 then + return -1 +endi +if $data91 != 3 then return -1 endi @@ -143,16 +143,16 @@ if $row != 10 then return -1 endi -if $data00 != @22-01-01 00:00:00.000@ then +if $data00 != @22-01-01 00:00:00.007@ then return -1 endi -if $data01 != 0 then +if $data01 != 7 then return -1 endi -if $data90 != @22-01-01 00:00:00.009@ then +if $data90 != @22-01-01 00:00:00.003@ then return -1 endi -if $data91 != 9 then +if $data91 != 3 then return -1 endi diff --git a/tests/script/tsim/query/crash_sql.sim b/tests/script/tsim/query/crash_sql.sim new file mode 100644 index 0000000000..44671fbb0d --- /dev/null +++ b/tests/script/tsim/query/crash_sql.sim @@ -0,0 +1,103 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi + +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +print =============== create database +sql create database db +sql show databases +if $rows != 2 then + return -1 +endi + +sql use db + +print =============== create super table and child table +sql create table stb1 (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int) +sql show stables +print $rows $data00 $data01 $data02 +if $rows != 1 then + return -1 +endi + +sql create table ct1 using stb1 tags ( 1 ) +sql create table ct2 using stb1 tags ( 2 ) +sql create table ct3 using stb1 tags ( 3 ) +sql create table ct4 using stb1 tags ( 4 ) +sql show tables +print $rows $data00 $data10 $data20 +if $rows != 4 then + return -1 +endi + +print =============== insert data into child table ct1 (s) +sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct1 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct1 values ( '2022-01-01 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct1 values ( '2022-01-01 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct1 values ( '2022-01-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct1 values ( '2022-01-01 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a ) +sql insert into ct1 values ( '2022-01-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+7a ) +sql insert into ct1 values ( '2022-01-01 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+8a ) + +print =============== insert data into child table ct2 (d) +sql insert into ct2 values ( '2022-01-01 01:00:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct2 values ( '2022-01-01 10:00:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct2 values ( '2022-01-01 20:00:01.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct2 values ( '2022-01-02 10:00:01.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct2 values ( '2022-01-02 20:00:01.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct2 values ( '2022-01-03 10:00:01.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", now+6a ) +sql insert into ct2 values ( '2022-01-03 20:00:01.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", now+7a ) + +print =============== insert data into child table ct3 (n) +sql insert into ct3 values ( '2021-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) +sql insert into ct3 values ( '2021-12-31 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct3 values ( '2022-01-01 01:01:06.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct3 values ( '2022-01-07 01:01:10.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct3 values ( '2022-01-31 01:01:16.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct3 values ( '2022-02-01 01:01:20.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct3 values ( '2022-02-28 01:01:26.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a ) +sql insert into ct3 values ( '2022-03-01 01:01:30.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) +sql insert into ct3 values ( '2022-03-08 01:01:36.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + +print =============== insert data into child table ct4 (y) +sql insert into ct4 values ( '2019-01-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) +sql insert into ct4 values ( '2019-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now+1a ) +sql insert into ct4 values ( '2019-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now+2a ) +sql insert into ct4 values ( '2020-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now+3a ) +sql insert into ct4 values ( '2020-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now+4a ) +sql insert into ct4 values ( '2020-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now+5a ) +sql insert into ct4 values ( '2020-12-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) +sql insert into ct4 values ( '2021-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now+6a ) +sql insert into ct4 values ( '2021-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) +sql insert into ct4 values ( '2021-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) +sql insert into ct4 values ( '2022-02-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) +sql insert into ct4 values ( '2022-05-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + + +print ================ start query ====================== +print ================ SQL used to cause taosd or taos shell crash +sql select sum(c1) ,count(c1) from ct4 group by c1 having sum(c10) between 0 and 1 ; + + + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stable/disk.sim b/tests/script/tsim/stable/disk.sim index 97ef779ff2..c1ced6ae10 100644 --- a/tests/script/tsim/stable/disk.sim +++ b/tests/script/tsim/stable/disk.sim @@ -53,6 +53,7 @@ endi sql select count(tbcol) from $mt print select count(tbcol) from $mt ===> $data00 if $data00 != $totalNum then + print expect $totalNum , actual: $data00 return -1 endi diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index f4e8f08c37..d7534338e1 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -175,7 +175,7 @@ $totalMsgOfCtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -246,7 +246,7 @@ $totalMsgOfNtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 5f54715b36..957f1774f9 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -132,39 +132,32 @@ if $data[0][1] == 1 then endi endi -if $data[0][2] != 0 then - if $data[0][2] != $expectmsgcnt then - return -1 +# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb +if $data[0][2] == $totalMsgOfStb then + if $data[1][2] == 0 then + goto check_ok_0 endi - if $data[1][2] != 0 then - return -1 - endi -endi -if $data[1][2] != 0 then - if $data[1][2] != $expectmsgcnt then - return -1 - endi - if $data[0][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfStb then + goto check_ok_0 endi endi +return -1 +check_ok_0: -if $data[0][3] != 0 then - if $data[0][3] != $expectmsgcnt then - return -1 +if $data[0][3] == $totalMsgOfStb then + if $data[1][3] == 0 then + goto check_ok_1 endi - if $data[1][3] != 0 then - return -1 - endi -endi -if $data[1][3] != 0 then - if $data[1][3] != $expectmsgcnt then - return -1 - endi - if $data[0][3] != 0 then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfStb then + goto check_ok_1 endi endi +return -1 +check_ok_1: + $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_stb loop_consume_diff_topic_from_stb_end: @@ -241,39 +234,31 @@ if $data[0][1] == 1 then endi endi -if $data[0][2] != $totalMsgOfCtb then - if $data[1][2] != $totalMsgOfCtb then - return -1 +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_2 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfCtb then - if $data[0][2] != $totalMsgOfCtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_2 endi endi +return -1 +check_ok_2: -if $data[0][3] != $totalMsgOfCtb then - if $data[1][3] != $totalMsgOfCtb then - return -1 +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_3 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfCtb then - if $data[0][3] != $totalMsgOfCtb then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_3 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_3: $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_ctb @@ -351,39 +336,32 @@ if $data[1][1] == 0 then endi endi -if $data[0][2] != $totalMsgOfNtb then - if $data[1][2] != $totalMsgOfNtb then - return -1 +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_4 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfNtb then - if $data[0][2] != $totalMsgOfNtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_4 endi endi +return -1 +check_ok_4: -if $data[0][3] != $totalMsgOfNtb then - if $data[1][3] != $totalMsgOfNtb then - return -1 +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_5 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfNtb then - if $data[0][3] != $totalMsgOfNtb then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_5 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_5: + $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_ntb loop_consume_diff_topic_from_ntb_end: diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index b3e9a67c62..ac0d2bb6df 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -133,7 +133,7 @@ $totalMsgOfCtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -190,7 +190,7 @@ $totalMsgOfNtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index 354260fb47..01ccb2b515 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -103,39 +103,31 @@ if $data[0][1] == 1 then endi endi -if $data[0][2] != 0 then - if $data[0][2] != $expectmsgcnt then - return -1 +# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb +if $data[0][2] == $totalMsgOfStb then + if $data[1][2] == 0 then + goto check_ok_0 endi - if $data[1][2] != 0 then - return -1 - endi -endi -if $data[1][2] != 0 then - if $data[1][2] != $expectmsgcnt then - return -1 - endi - if $data[0][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfStb then + goto check_ok_0 endi endi +return -1 +check_ok_0: -if $data[0][3] != 0 then - if $data[0][3] != $expectmsgcnt then - return -1 +if $data[0][3] == $totalMsgOfStb then + if $data[1][3] == 0 then + goto check_ok_1 endi - if $data[1][3] != 0 then - return -1 - endi -endi -if $data[1][3] != 0 then - if $data[1][3] != $expectmsgcnt then - return -1 - endi - if $data[0][3] != 0 then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfStb then + goto check_ok_1 endi endi +return -1 +check_ok_1: ####################################################################################### # clear consume info and consume result @@ -198,39 +190,31 @@ if $data[0][1] == 1 then endi endi -if $data[0][2] != $totalMsgOfCtb then - if $data[1][2] != $totalMsgOfCtb then - return -1 +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_2 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfCtb then - if $data[0][2] != $totalMsgOfCtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_2 endi endi +return -1 +check_ok_2: -if $data[0][3] != $totalMsgOfCtb then - if $data[1][3] != $totalMsgOfCtb then - return -1 +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_3 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfCtb then - if $data[0][3] != $totalMsgOfCtb then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_3 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_3: ####################################################################################### # clear consume info and consume result @@ -293,39 +277,31 @@ if $data[1][1] == 0 then endi endi -if $data[0][2] != $totalMsgOfNtb then - if $data[1][2] != $totalMsgOfNtb then - return -1 +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_4 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfNtb then - if $data[0][2] != $totalMsgOfNtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_4 endi endi +return -1 +check_ok_4: -if $data[0][3] != $totalMsgOfNtb then - if $data[1][3] != $totalMsgOfNtb then - return -1 +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_5 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfNtb then - if $data[0][3] != $totalMsgOfNtb then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_5 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_5: #------ not need stop consumer, because it exit after pull msg overthan expect msg #system tsim/tmq/consume.sh -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim new file mode 100644 index 0000000000..878e3d9031 --- /dev/null +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -0,0 +1,337 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1Of2Cons.sim: vgroups=1, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic2Of2ConsOverlap.sim: vgroups=1, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics +#basic3Of2Cons.sim: vgroups=4, one topic for 2 consumers, firstly insert data, then start consume. Include six topics +#basic4Of2Cons.sim: vgroups=4, multi topics for 2 consumers, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 5 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +$topicNum = 2 + +#=============================== start consume =============================# + + +print ================ test consume from stb +print == overlap toipcs: topic_stb_column + topic_stb_all, topic_stb_function + topic_stb_all +$topicList = ' . topic_stb_column +$topicList = $topicList . , +$topicList = $topicList . topic_stb_all +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfOneTopic = $ctbNum * $rowsPerCtb +$totalMsgOfStb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfStb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + + +$topicList = ' . topic_stb_all +$topicList = $topicList . , +$topicList = $topicList . topic_stb_function +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. + +if $data[0][2] < $totalMsgOfOneTopic then + return -1 +endi +if $data[0][2] > $totalMsgOfStb then + return -1 +endi +if $data[1][2] < $totalMsgOfOneTopic then + return -1 +endi +if $data[1][2] > $totalMsgOfStb then + return -1 +endi + +$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +$sumOfMsgCnt = $data[0][2] + $data[1][2] +if $sumOfMsgCnt != $totalMsgCons then + return -1 +endi + +# $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. +if $data[0][3] < $totalMsgOfOneTopic then + return -1 +endi +if $data[0][3] > $totalMsgOfStb then + return -1 +endi +if $data[1][3] < $totalMsgOfOneTopic then + return -1 +endi +if $data[1][3] > $totalMsgOfStb then + return -1 +endi + +$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +$sumOfRows = $data[0][3] + $data[1][3] +if $sumOfRows != $totalMsgCons then + return -1 +endi + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ctb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ctb +print == overlap toipcs: topic_ctb_column + topic_ctb_all, topic_ctb_function + topic_ctb_all +$topicList = ' . topic_ctb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . ' +$consumerId = 0 + +$totalMsgOfOneTopic = $rowsPerCtb +$totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfCtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +$topicList = ' . topic_ctb_function +$topicList = $topicList . , +$topicList = $topicList . topic_ctb_all +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb +# or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic +if $data[0][2] == $totalMsgOfOneTopic then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +elif $data[1][2] == $totalMsgOfOneTopic then + if $data[0][2] == $totalMsgOfCtb then + goto check_ok_0 + endi +endi +return -1 +check_ok_0: + +if $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +elif $data[1][3] == $totalMsgOfOneTopic then + if $data[0][3] == $totalMsgOfCtb then + goto check_ok_1 + endi +endi +return -1 +check_ok_1: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table for ntb +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +print == overlap toipcs: topic_ntb_column + topic_ntb_all, topic_ntb_function + topic_ntb_all +$topicList = ' . topic_ntb_column +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . ' + +$consumerId = 0 +$totalMsgOfOneTopic = $rowsPerCtb +$totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + + +$topicList = ' . topic_ntb_function +$topicList = $topicList . , +$topicList = $topicList . topic_ntb_all +$topicList = $topicList . ' +$consumerId = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +print ==> rows[1]: $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +if $rows != 2 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] == 0 then + if $data[1][1] != 1 then + return -1 + endi +endi +if $data[0][1] == 1 then + if $data[1][1] != 0 then + return -1 + endi +endi + +# either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb +# or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic +if $data[0][2] == $totalMsgOfOneTopic then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +elif $data[1][2] == $totalMsgOfOneTopic then + if $data[0][2] == $totalMsgOfNtb then + goto check_ok_2 + endi +endi +return -1 +check_ok_2: + +if $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +elif $data[1][3] == $totalMsgOfOneTopic then + if $data[0][3] == $totalMsgOfNtb then + goto check_ok_3 + endi +endi +return -1 +check_ok_3: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index 7a0ee6f917..c0ba2c97fb 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -175,7 +175,7 @@ $totalMsgOfCtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -246,7 +246,7 @@ $totalMsgOfNtb = $rowsPerCtb $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index b097617326..bf640ae1a1 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -145,6 +145,12 @@ if $data[1][2] >= $expectmsgcnt then return -1 endi +$sumOfMsgCnt = $data[0][2] + $data[1][2] +if $sumOfMsgCnt != $expectmsgcnt then + return -1 +endi + + if $data[0][3] <= 0 then return -1 endi @@ -158,6 +164,12 @@ endi if $data[1][3] >= $expectmsgcnt then return -1 endi + +$sumOfMsgRows = $data[0][3] + $data[1][3] +if $sumOfMsgRows != $expectmsgcnt then + return -1 +endi + $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_stb loop_consume_diff_topic_from_stb_end: @@ -234,39 +246,32 @@ if $data[0][1] == 1 then endi endi -if $data[0][2] != $totalMsgOfCtb then - if $data[1][2] != $totalMsgOfCtb then - return -1 +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_0 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfCtb then - if $data[0][2] != $totalMsgOfCtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[1][2] == $totalMsgOfCtb then + if $data[0][2] == 0 then + goto check_ok_0 endi endi +return -1 +check_ok_0: -if $data[0][3] != $totalMsgOfCtb then - if $data[1][3] != $totalMsgOfCtb then - return -1 +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_1 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfCtb then - if $data[0][3] != $totalMsgOfCtb then - return -1 +elif $data[1][3] == $totalMsgOfCtb then + if $data[0][3] == 0 then + goto check_ok_1 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_1: + $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_ctb loop_consume_diff_topic_from_ctb_end: @@ -343,39 +348,32 @@ if $data[1][1] == 0 then endi endi -if $data[0][2] != $totalMsgOfNtb then - if $data[1][2] != $totalMsgOfNtb then - return -1 +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_2 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfNtb then - if $data[0][2] != $totalMsgOfNtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[1][2] == $totalMsgOfNtb then + if $data[0][2] == 0 then + goto check_ok_2 endi endi +return -1 +check_ok_2: -if $data[0][3] != $totalMsgOfNtb then - if $data[1][3] != $totalMsgOfNtb then - return -1 +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_3 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfNtb then - if $data[0][3] != $totalMsgOfNtb then - return -1 +elif $data[1][3] == $totalMsgOfNtb then + if $data[0][3] == 0 then + goto check_ok_3 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_3: + $loop_cnt = $loop_cnt + 1 goto loop_consume_diff_topic_from_ntb loop_consume_diff_topic_from_ntb_end: diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 1af02751b1..1eed93463c 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -130,7 +130,7 @@ $totalMsgOfCtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfCtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -187,7 +187,7 @@ $totalMsgOfNtb = $rowsPerCtb * $topicNum $expectmsgcnt = $totalMsgOfNtb sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) -print == start consumer to pull msgs from stb +print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index 75868bd974..a17d8a2f45 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -114,6 +114,11 @@ if $data[1][2] >= $expectmsgcnt then return -1 endi +$sumOfConsMsg = $data[0][2] + $data[1][2] +if $sumOfConsMsg != $expectmsgcnt then + return -1 +endi + if $data[0][3] <= 0 then return -1 endi @@ -128,6 +133,11 @@ if $data[1][3] >= $expectmsgcnt then return -1 endi +$sumOfConsRow = $data[0][3] + $data[1][3] +if $sumOfConsRow != $expectmsgcnt then + return -1 +endi + ####################################################################################### # clear consume info and consume result #run tsim/tmq/clearConsume.sim @@ -189,39 +199,32 @@ if $data[0][1] == 1 then endi endi -if $data[0][2] != $totalMsgOfCtb then - if $data[1][2] != $totalMsgOfCtb then - return -1 +# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb +if $data[0][2] == $totalMsgOfCtb then + if $data[1][2] == 0 then + goto check_ok_0 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfCtb then - if $data[0][2] != $totalMsgOfCtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfCtb then + goto check_ok_0 endi endi +return -1 +check_ok_0: -if $data[0][3] != $totalMsgOfCtb then - if $data[1][3] != $totalMsgOfCtb then - return -1 +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == 0 then + goto check_ok_1 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfCtb then - if $data[0][3] != $totalMsgOfCtb then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfCtb then + goto check_ok_1 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_1: + ####################################################################################### # clear consume info and consume result @@ -284,39 +287,31 @@ if $data[1][1] == 0 then endi endi -if $data[0][2] != $totalMsgOfNtb then - if $data[1][2] != $totalMsgOfNtb then - return -1 +# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb +if $data[0][2] == $totalMsgOfNtb then + if $data[1][2] == 0 then + goto check_ok_2 endi - if $data[0][2] != 0 then - return -1 - endi -endi -if $data[1][2] != $totalMsgOfNtb then - if $data[0][2] != $totalMsgOfNtb then - return -1 - endi - if $data[1][2] != 0 then - return -1 +elif $data[0][2] == 0 then + if $data[1][2] == $totalMsgOfNtb then + goto check_ok_2 endi endi +return -1 +check_ok_2: -if $data[0][3] != $totalMsgOfNtb then - if $data[1][3] != $totalMsgOfNtb then - return -1 +if $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == 0 then + goto check_ok_3 endi - if $data[0][3] != 0 then - return -1 - endi -endi -if $data[1][3] != $totalMsgOfNtb then - if $data[0][3] != $totalMsgOfNtb then - return -1 +elif $data[0][3] == 0 then + if $data[1][3] == $totalMsgOfNtb then + goto check_ok_3 endi - if $data[1][3] != 0 then - return -1 - endi endi +return -1 +check_ok_3: #------ not need stop consumer, because it exit after pull msg overthan expect msg #system tsim/tmq/consume.sh -s stop -x SIGINT diff --git a/tests/system-test/2-query/Now.py b/tests/system-test/2-query/Now.py new file mode 100644 index 0000000000..7f0e173439 --- /dev/null +++ b/tests/system-test/2-query/Now.py @@ -0,0 +1,310 @@ + + +from util.dnodes import * +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + # for func now() , today(), timezone() + tdSql.prepare() + today_date = datetime.datetime.strptime( + datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") + + tdLog.printNoPrefix("==========step1:create tables==========") + tdSql.execute( + '''create table if not exists ntb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) + ''' + ) + tdSql.execute( + '''create table if not exists stb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) + ''' + ) + tdSql.execute( + '''create table if not exists stb_1 using stb tags(100) + ''' + ) + tdLog.printNoPrefix("==========step2:insert data into ntb==========") + tdSql.execute( + 'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.execute( + 'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdLog.printNoPrefix("==========step2:query test of ntb ==========") + + # test function now() + # ntable + tdSql.query("select now() from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1w from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1w from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1d from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1d from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1h from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1h from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1m from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1m from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1s from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1s from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1a from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1a from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1u from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1u from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1b from ntb") + tdSql.checkRows(3) + tdSql.query("select now() +1b from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1w from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1w from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1d from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1d from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1h from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1h from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1m from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1m from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1s from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1s from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1a from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1a from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1u from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1u from db.ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1b from ntb") + tdSql.checkRows(3) + tdSql.query("select now() -1b from db.ntb") + tdSql.checkRows(3) + tdSql.query("select * from ntb where ts=now()") + tdSql.checkRows(0) + tdSql.query("select * from ntb where ts>now()") + tdSql.checkRows(0) + tdSql.query("select now() from ntb where ts=today()") + tdSql.checkRows(1) + + # stable + tdSql.query("select now() from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1w from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1w from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() +1d from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1d from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() +1h from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1h from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() +1m from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1m from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() +1s from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1s from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() +1a from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1a from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() +1u from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1u from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() +1b from stb") + tdSql.checkRows(3) + tdSql.query("select now() +1b from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1w from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1w from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1d from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1d from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1h from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1h from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1m from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1m from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1s from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1s from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1a from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1a from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1u from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1u from db.stb") + tdSql.checkRows(3) + tdSql.query("select now() -1b from stb") + tdSql.checkRows(3) + tdSql.query("select now() -1b from db.stb") + tdSql.checkRows(3) + # tdSql.query("select * from stb where ts=now()") + # tdSql.checkRows(0) + # tdSql.query("select * from stb where ts>now()") + # tdSql.checkRows(0) + tdSql.query("select now() from stb where ts=today()") + tdSql.checkRows(1) + + # table + tdSql.query("select now() from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1w from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1w from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1d from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1d from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1h from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1h from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1m from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1m from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1s from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1s from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1a from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1a from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1u from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1u from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1b from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() +1b from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1w from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1w from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1d from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1d from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1h from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1h from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1m from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1m from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1s from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1s from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1a from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1a from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1u from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1u from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1b from stb_1") + tdSql.checkRows(3) + tdSql.query("select now() -1b from db.stb_1") + tdSql.checkRows(3) + tdSql.query("select * from stb_1 where ts=now()") + tdSql.checkRows(0) + tdSql.query("select * from stb_1 where ts>now()") + tdSql.checkRows(0) + + # tdSql.query("select * from stb_1 where ts=now") + # tdSql.checkRows(0) + # tdSql.query("select * from stb_1 where ts>now") + # tdSql.checkRows(0) + + tdSql.query("select now() from stb_1 where ts=today()") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/To_iso8061.py b/tests/system-test/2-query/To_iso8061.py new file mode 100644 index 0000000000..fca53572ca --- /dev/null +++ b/tests/system-test/2-query/To_iso8061.py @@ -0,0 +1,78 @@ +from time import sleep + +from util.log import * +from util.sql import * +from util.cases import * + + + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + tdSql.prepare() + # get system timezone + today_date = datetime.datetime.strptime( + datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") + + tdLog.printNoPrefix("==========step1:create tables==========") + tdSql.execute( + '''create table if not exists ntb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) + ''' + ) + tdSql.execute( + '''create table if not exists stb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) + ''' + ) + tdSql.execute( + '''create table if not exists stb_1 using stb tags(100) + ''' + ) + + tdLog.printNoPrefix("==========step2:insert data==========") + tdSql.execute('insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.execute('insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + + tdSql.query("select to_iso8601(ts) from ntb") + tdSql.checkRows(3) + tdSql.query("select c1 from ntb where ts = to_iso8601(1577808000000)") + tdSql.checkRows(1) + tdSql.checkData(0,0,10) + tdSql.query("select * from ntb where ts = to_iso8601(1577808000000)") + tdSql.checkRows(1) + tdSql.query("select to_iso8601(ts) from ntb where ts=today()") + tdSql.checkRows(1) + # tdSql.checkData(0,0,10) + for i in range(1,10): + tdSql.query("select to_iso8601(1) from ntb") + tdSql.checkData(0,0,"1970-01-01T08:00:01+0800") + i+=1 + sleep(0.2) + tdSql.checkRows(3) + tdSql.query("select to_iso8601(ts) from ntb") + tdSql.checkRows(3) + tdSql.query("select to_iso8601(ts) from db.ntb") + + tdSql.query("select to_iso8601(today()) from ntb") + tdSql.checkRows(3) + tdSql.query("select to_iso8601(now()) from ntb") + tdSql.checkRows(3) + + tdSql.error("select to_iso8601(timezone()) from ntb") + tdSql.error("select to_iso8601('abc') from ntb") + + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/Today.py b/tests/system-test/2-query/Today.py new file mode 100644 index 0000000000..ad4e48bda5 --- /dev/null +++ b/tests/system-test/2-query/Today.py @@ -0,0 +1,318 @@ + + +from util.dnodes import * +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def run(self): # sourcery skip: extract-duplicate-method + # for func now() , today(), timezone() + tdSql.prepare() + today_date = datetime.datetime.strptime( + datetime.datetime.now().strftime("%Y-%m-%d"), "%Y-%m-%d") + + tdLog.printNoPrefix("==========step1:create tables==========") + tdSql.execute( + '''create table if not exists ntb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) + ''' + ) + tdSql.execute( + '''create table if not exists stb + (ts timestamp, c1 int, c2 float,c3 double,c4 timestamp) tags(t0 int) + ''' + ) + tdSql.execute( + '''create table if not exists stb_1 using stb tags(100) + ''' + ) + tdLog.printNoPrefix("==========step2:insert data into ntb==========") + tdSql.execute( + 'insert into ntb values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdSql.execute( + 'insert into stb_1 values(now,1,1.55,100.555555,today())("2020-1-1 00:00:00",10,11.11,99.999999,now())(today(),3,3.333,333.333333,now())') + tdLog.printNoPrefix("==========step2:query test of ntb ==========") + + # test function today() + # normal table + tdSql.query("select today() from ntb") + tdSql.checkRows(3) + tdSql.checkData(0, 0, str(today_date)) + tdSql.query("select today() from db.ntb") + tdSql.checkRows(3) + tdSql.checkData(0, 0, str(today_date)) + tdSql.query("select today() +1w from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1w from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1d from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1d from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1h from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1h from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1m from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1m from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1s from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1s from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1a from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1a from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1u from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1u from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1b from ntb") + tdSql.checkRows(3) + tdSql.query("select today() +1b from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1w from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1w from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1d from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1d from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1h from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1h from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1m from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1m from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1s from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1s from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1a from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1a from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1u from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1u from db.ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1b from ntb") + tdSql.checkRows(3) + tdSql.query("select today() -1b from db.ntb") + tdSql.checkRows(3) + tdSql.query("select * from ntb where ts=today()") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 3) + tdSql.query("select * from ntb where ts<=today()") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 10) + # for bug + # tdSql.query("select * from ntb where ts=today()") + tdSql.checkRows(2) + tdSql.checkData(0, 1, 3) + # tdSql.query("select * from ntb where ts>today()") + # tdSql.checkRows(1) + tdSql.query("select c4 from ntb where c4=today()") + tdSql.checkRows(1) + tdSql.checkData(0, 0, str(today_date)) + tdSql.query("select today() from ntb where ts=today()") + # tdSql.checkRows(2) + # tdSql.query("select * from ntb where ts>today()") + # tdSql.checkRows(1) + tdSql.query("select c4 from stb where c4=today()") + tdSql.checkRows(1) + tdSql.checkData(0, 0, str(today_date)) + tdSql.query("select today() from stb where ts=today()") + tdSql.checkRows(2) + tdSql.query("select * from ntb where ts>today()") + tdSql.checkRows(1) + tdSql.query("select c4 from stb_1 where c4=today()") + tdSql.checkRows(1) + tdSql.query("select today() from stb_1 where tsi32; diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 3b22b30852..284693795e 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -3,7 +3,7 @@ aux_source_directory(src SHELL_SRC) add_executable(shell ${SHELL_SRC}) target_link_libraries( shell - PUBLIC taos_static + PUBLIC taos PRIVATE os common transport util ) target_include_directories( diff --git a/tools/shell/inc/shell.h b/tools/shell/inc/shell.h deleted file mode 100644 index 866cd63bdb..0000000000 --- a/tools/shell/inc/shell.h +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_SHELL_H_ -#define _TD_SHELL_H_ - -#include "os.h" - -#include "taos.h" -#include "taosdef.h" - -#define MAX_USERNAME_SIZE 64 -#define MAX_DBNAME_SIZE 64 -#define MAX_IP_SIZE 20 -#define MAX_HISTORY_SIZE 1000 -#define MAX_COMMAND_SIZE 1048586 -#define HISTORY_FILE ".taos_history" - -#define DEFAULT_RES_SHOW_NUM 100 - -typedef struct SShellHistory { - char* hist[MAX_HISTORY_SIZE]; - int hstart; - int hend; -} SShellHistory; - -typedef struct SShellArguments { - char* host; - char* password; - char* user; - char* auth; - char* database; - char* timezone; - bool is_raw_time; - bool is_use_passwd; - bool dump_config; - char file[TSDB_FILENAME_LEN]; - char dir[TSDB_FILENAME_LEN]; - int threadNum; - int check; - bool status; - bool verbose; - char* commands; - int abort; - int port; - int pktLen; - int pktNum; - char* pktType; - char* netTestRole; -} SShellArguments; - -/**************** Function declarations ****************/ -extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments); -extern TAOS* shellInit(SShellArguments* args); -extern void* shellLoopQuery(void* arg); -extern void taos_error(TAOS_RES* tres, int64_t st); -extern int regex_match(const char* s, const char* reg, int cflags); -int32_t shellReadCommand(TAOS* con, char command[]); -int32_t shellRunCommand(TAOS* con, char* command); -void shellRunCommandOnServer(TAOS* con, char command[]); -void read_history(); -void write_history(); -void source_file(TAOS* con, char* fptr); -void source_dir(TAOS* con, SShellArguments* args); -void get_history_path(char* history); -void shellCheck(TAOS* con, SShellArguments* args); -void cleanup_handler(void* arg); -void exitShell(); -int shellDumpResult(TAOS_RES* con, char* fname, int* error_no, bool printMode); -void shellGetGrantInfo(void *con); -int isCommentLine(char *line); - -/**************** Global variable declarations ****************/ -extern char PROMPT_HEADER[]; -extern char CONTINUE_PROMPT[]; -extern int prompt_size; -extern SShellHistory history; -extern SShellArguments args; -extern int64_t result; - -#endif diff --git a/tools/shell/inc/shellCommand.h b/tools/shell/inc/shellCommand.h deleted file mode 100644 index 49f7dc0133..0000000000 --- a/tools/shell/inc/shellCommand.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_SHELL_COMMAND_H_ -#define _TD_SHELL_COMMAND_H_ - -#include "shell.h" - -#define LEFT 1 -#define RIGHT 2 -#define UP 3 -#define DOWN 4 - -typedef struct Command Command; -struct Command { - char * buffer; - char * command; - unsigned commandSize; - unsigned bufferSize; - unsigned cursorOffset; - unsigned screenOffset; - unsigned endOffset; -}; - -extern void backspaceChar(Command *cmd); -extern void clearLineBefore(Command *cmd); -extern void clearLineAfter(Command *cmd); -extern void deleteChar(Command *cmd); -extern void moveCursorLeft(Command *cmd); -extern void moveCursorRight(Command *cmd); -extern void positionCursorHome(Command *cmd); -extern void positionCursorEnd(Command *cmd); -extern void showOnScreen(Command *cmd); -extern void updateBuffer(Command *cmd); -extern int isReadyGo(Command *cmd); -extern void resetCommand(Command *cmd, const char s[]); - -int countPrefixOnes(unsigned char c); -void clearScreen(int ecmd_pos, int cursor_pos); -void printChar(char c, int times); -void positionCursor(int step, int direction); - -#endif diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h new file mode 100644 index 0000000000..af28373510 --- /dev/null +++ b/tools/shell/inc/shellInt.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_SHELL_INT_H_ +#define _TD_SHELL_INT_H_ + +#include "os.h" +#include "taos.h" +#include "taosdef.h" +#include "taoserror.h" +#include "tconfig.h" +#include "tglobal.h" +#include "trpc.h" +#include "ttypes.h" +#include "tutil.h" + +#define SHELL_MAX_HISTORY_SIZE 1000 +#define SHELL_MAX_COMMAND_SIZE 1048586 +#define SHELL_HISTORY_FILE ".taos_history" +#define SHELL_DEFAULT_RES_SHOW_NUM 100 +#define SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 +#define SHELL_MAX_PKG_LEN 2 * 1024 * 1024 +#define SHELL_MIN_PKG_LEN 1 +#define SHELL_DEF_PKG_LEN 1024 +#define SHELL_MAX_PKG_NUM 1 * 1024 * 1024 +#define SHELL_MIN_PKG_NUM 1 +#define SHELL_DEF_PKG_NUM 100 + +typedef struct { + char* hist[SHELL_MAX_HISTORY_SIZE]; + char file[TSDB_FILENAME_LEN]; + int32_t hstart; + int32_t hend; +} SShellHistory; + +typedef struct { + const char* host; + const char* user; + const char* auth; + const char* database; + const char* cfgdir; + const char* commands; + const char* netrole; + char file[PATH_MAX]; + char password[TSDB_USET_PASSWORD_LEN]; + bool is_gen_auth; + bool is_raw_time; + bool is_version; + bool is_dump_config; + bool is_check; + bool is_startup; + bool is_help; + uint16_t port; + int32_t pktLen; + int32_t pktNum; + int32_t displayWidth; + int32_t abort; +} SShellArgs; + +typedef struct { + const char* clientVersion; + const char* promptHeader; + const char* promptContinue; + const char* osname; + int32_t promptSize; + char programVersion[32]; +} SShellOsDetails; + +typedef struct { + SShellArgs args; + SShellHistory history; + SShellOsDetails info; + TAOS* conn; + TdThread pid; + tsem_t cancelSem; + int64_t result; +} SShellObj; + +// shellArguments.c +int32_t shellParseArgs(int32_t argc, char* argv[]); + +// shellCommand.c +int32_t shellReadCommand(char* command); + +// shellEngine.c +int32_t shellExecute(); + +// shellUtil.c +int32_t shellCheckIntSize(); +void shellPrintVersion(); +void shellPrintHelp(); +void shellGenerateAuth(); +void shellDumpConfig(); +void shellCheckServerStatus(); +bool shellRegexMatch(const char* s, const char* reg, int32_t cflags); +void shellExit(); + +// shellNettest.c +void shellTestNetWork(); + +// shellMain.c +extern SShellObj shell; +extern void taos_init(); + +#endif /*_TD_SHELL_INT_H_*/ diff --git a/tools/shell/inc/syncMsg.h b/tools/shell/inc/syncMsg.h deleted file mode 100644 index 85ac9c78af..0000000000 --- a/tools/shell/inc/syncMsg.h +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_MSG_H -#define TDENGINE_SYNC_MSG_H - -#ifdef __cplusplus -extern "C" { -#endif -#include "tsync.h" - -typedef enum { - TAOS_SMSG_START = 0, - TAOS_SMSG_SYNC_DATA = 1, - TAOS_SMSG_SYNC_DATA_RSP = 2, - TAOS_SMSG_SYNC_FWD = 3, - TAOS_SMSG_SYNC_FWD_RSP = 4, - TAOS_SMSG_SYNC_REQ = 5, - TAOS_SMSG_SYNC_REQ_RSP = 6, - TAOS_SMSG_SYNC_MUST = 7, - TAOS_SMSG_SYNC_MUST_RSP = 8, - TAOS_SMSG_STATUS = 9, - TAOS_SMSG_STATUS_RSP = 10, - TAOS_SMSG_SETUP = 11, - TAOS_SMSG_SETUP_RSP = 12, - TAOS_SMSG_SYNC_FILE = 13, - TAOS_SMSG_SYNC_FILE_RSP = 14, - TAOS_SMSG_TEST = 15, - TAOS_SMSG_END = 16 -} ESyncMsgType; - -typedef enum { - SYNC_STATUS_BROADCAST, - SYNC_STATUS_BROADCAST_RSP, - SYNC_STATUS_SETUP_CONN, - SYNC_STATUS_SETUP_CONN_RSP, - SYNC_STATUS_EXCHANGE_DATA, - SYNC_STATUS_EXCHANGE_DATA_RSP, - SYNC_STATUS_CHECK_ROLE, - SYNC_STATUS_CHECK_ROLE_RSP -} ESyncStatusType; - -#pragma pack(push, 1) - -typedef struct { - int8_t type; // msg type - int8_t protocol; // protocol version - uint16_t signature; // fixed value - int32_t code; // - int32_t cId; // cluster Id - int32_t vgId; // vg ID - int32_t len; // content length, does not include head - uint32_t cksum; -} SSyncHead; - -typedef struct { - SSyncHead head; - uint16_t port; - uint16_t tranId; - int32_t sourceId; // only for arbitrator - char fqdn[TSDB_FQDN_LEN]; -} SSyncMsg; - -typedef struct { - SSyncHead head; - int8_t sync; - int8_t reserved; - uint16_t tranId; - int8_t reserverd[4]; -} SSyncRsp; - -typedef struct { - int8_t role; - uint64_t version; -} SPeerStatus; - -typedef struct { - SSyncHead head; - int8_t role; - int8_t ack; - int8_t type; - int8_t reserved[3]; - uint16_t tranId; - uint64_t version; - SPeerStatus peersStatus[TAOS_SYNC_MAX_REPLICA]; -} SPeersStatus; - -typedef struct { - SSyncHead head; - uint64_t fversion; -} SFileVersion; - -typedef struct { - SSyncHead head; - int8_t ack; -} SFileAck; - -typedef struct { - SSyncHead head; - uint64_t version; - int32_t code; -} SFwdRsp; - -#pragma pack(pop) - -#define SYNC_PROTOCOL_VERSION 1 -#define SYNC_SIGNATURE ((uint16_t)(0xCDEF)) - -extern char *statusType[]; - -uint16_t syncGenTranId(); -int32_t syncCheckHead(SSyncHead *pHead); - -void syncBuildSyncFwdMsg(SSyncHead *pHead, int32_t vgId, int32_t len); -void syncBuildSyncFwdRsp(SFwdRsp *pMsg, int32_t vgId, uint64_t version, int32_t code); -void syncBuildSyncReqMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildSyncDataMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildSyncSetupMsg(SSyncMsg *pMsg, int32_t vgId); -void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId); -void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId); - -void syncBuildFileAck(SFileAck *pMsg, int32_t vgId); -void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_VNODEPEER_H diff --git a/tools/shell/inc/tsync.h b/tools/shell/inc/tsync.h deleted file mode 100644 index d1b68e3f5a..0000000000 --- a/tools/shell/inc/tsync.h +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_SYNC_H -#define TDENGINE_SYNC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TAOS_SYNC_MAX_REPLICA 5 -#define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF - -typedef enum { - TAOS_SYNC_ROLE_OFFLINE = 0, - TAOS_SYNC_ROLE_UNSYNCED = 1, - TAOS_SYNC_ROLE_SYNCING = 2, - TAOS_SYNC_ROLE_SLAVE = 3, - TAOS_SYNC_ROLE_MASTER = 4 -} ESyncRole; - -typedef enum { - TAOS_SYNC_STATUS_INIT = 0, - TAOS_SYNC_STATUS_START = 1, - TAOS_SYNC_STATUS_FILE = 2, - TAOS_SYNC_STATUS_CACHE = 3 -} ESyncStatus; - -typedef struct { - uint32_t nodeId; // node ID assigned by TDengine - uint16_t nodePort; // node sync Port - char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN -} SNodeInfo; - -typedef struct { - int8_t quorum; // number of confirms required, >=1 - int8_t replica; // number of replications, >=1 - SNodeInfo nodeInfo[TAOS_SYNC_MAX_REPLICA]; -} SSyncCfg; - -typedef struct { - int32_t selfIndex; - uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; - int32_t role[TAOS_SYNC_MAX_REPLICA]; -} SNodesRole; - -// get the wal file from index or after -// return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); - -// when a forward pkt is received, call this to handle data -typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); - -// when forward is confirmed by peer, master call this API to notify app -typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); - -// when role is changed, call this to notify app -typedef void (*FNotifyRole)(int32_t vgId, int8_t role); - -// if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); - -// when data file is synced successfully, notity app -typedef void (*FStartSyncFile)(int32_t vgId); -typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); - -// get file version -typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); - -typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); -typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); - -typedef struct { - int32_t vgId; // vgroup ID - uint64_t version; // initial version - SSyncCfg syncCfg; // configuration from mgmt - char path[TSDB_FILENAME_LEN]; // path to the file - void * pTsdb; - FGetWalInfo getWalInfoFp; - FWriteToCache writeToCacheFp; - FConfirmForward confirmForward; - FNotifyRole notifyRoleFp; - FNotifyFlowCtrl notifyFlowCtrlFp; - FStartSyncFile startSyncFileFp; - FStopSyncFile stopSyncFileFp; - FGetVersion getVersionFp; - FSendFile sendFileFp; - FRecvFile recvFileFp; -} SSyncInfo; - -typedef void *tsync_h; - -int32_t syncInit(); -void syncCleanUp(); - -int64_t syncStart(const SSyncInfo *); -void syncStop(int64_t rid); -int32_t syncReconfig(int64_t rid, const SSyncCfg *); -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype, bool force); -void syncConfirmForward(int64_t rid, uint64_t version, int32_t code, bool force); -void syncRecover(int64_t rid); // recover from other nodes: -int32_t syncGetNodesRole(int64_t rid, SNodesRole *); - -extern char *syncRole[]; - -//global configurable parameters -extern int32_t sDebugFlag; -extern char tsArbitrator[]; -extern uint16_t tsSyncPort; - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_SYNC_H diff --git a/tools/shell/src/backup/shellCheck.c b/tools/shell/src/backup/shellCheck.c deleted file mode 100644 index d1f0683fea..0000000000 --- a/tools/shell/src/backup/shellCheck.c +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _GNU_SOURCE -#define _XOPEN_SOURCE -#define _DEFAULT_SOURCE - -#include "os.h" -#include "shell.h" -#include "shellCommand.h" -#include "tglobal.h" -#include "tutil.h" - -#define SHELL_SQL_LEN 1024 -static int32_t tbNum = 0; -static int32_t tbMallocNum = 0; -static char ** tbNames = NULL; -static int32_t checkedNum = 0; -static int32_t errorNum = 0; - -typedef struct { - TdThread threadID; - int threadIndex; - int totalThreads; - void * taos; - char * db; -} ShellThreadObj; - -static int32_t shellUseDb(TAOS *con, char *db) { - if (db == NULL) { - fprintf(stdout, "no dbname input\n"); - return -1; - } - - char sql[SHELL_SQL_LEN] = {0}; - snprintf(sql, SHELL_SQL_LEN, "use %s", db); - - TAOS_RES *pSql = taos_query(con, sql); - int32_t code = taos_errno(pSql); - if (code != 0) { - fprintf(stdout, "failed to execute sql:%s since %s", sql, taos_errstr(pSql)); - } - - taos_free_result(pSql); - return code; -} - -static int32_t shellShowTables(TAOS *con, char *db) { - char sql[SHELL_SQL_LEN] = {0}; - snprintf(sql, SHELL_SQL_LEN, "show %s.tables", db); - - TAOS_RES *pSql = taos_query(con, sql); - int32_t code = taos_errno(pSql); - - if (code != 0) { - fprintf(stdout, "failed to execute sql:%s since %s\n", sql, taos_errstr(pSql)); - } else { - TAOS_ROW row; - while ((row = taos_fetch_row(pSql))) { - int32_t tbIndex = tbNum++; - if (tbMallocNum < tbNum) { - tbMallocNum = (tbMallocNum * 2 + 1); - char** tbNames1 = taosMemoryRealloc(tbNames, tbMallocNum * sizeof(char *)); - if (tbNames1 == NULL) { - fprintf(stdout, "failed to malloc tablenames, num:%d\n", tbMallocNum); - code = TSDB_CODE_TSC_OUT_OF_MEMORY; - break; - } - tbNames = tbNames1; - } - - tbNames[tbIndex] = taosMemoryMalloc(TSDB_TABLE_NAME_LEN); - strncpy(tbNames[tbIndex], (const char *)row[0], TSDB_TABLE_NAME_LEN); - if (tbIndex % 100000 == 0 && tbIndex != 0) { - fprintf(stdout, "%d tablenames fetched\n", tbIndex); - } - } - } - - taos_free_result(pSql); - - fprintf(stdout, "total %d tablenames fetched, over\n", tbNum); - return code; -} - -static void shellFreeTbnames() { - for (int32_t i = 0; i < tbNum; ++i) { - taosMemoryFree(tbNames[i]); - } - taosMemoryFree(tbNames); -} - -static void *shellCheckThreadFp(void *arg) { - ShellThreadObj *pThread = (ShellThreadObj *)arg; - - setThreadName("shellCheckThrd"); - - int32_t interval = tbNum / pThread->totalThreads + 1; - int32_t start = pThread->threadIndex * interval; - int32_t end = (pThread->threadIndex + 1) * interval; - - if (end > tbNum) end = tbNum + 1; - - char file[32] = {0}; - snprintf(file, 32, "tb%d.txt", pThread->threadIndex); - - TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); - if (!fp) { - fprintf(stdout, "failed to open %s, reason:%s", file, strerror(errno)); - return NULL; - } - - char sql[SHELL_SQL_LEN]; - for (int32_t t = start; t < end; ++t) { - char *tbname = tbNames[t]; - if (tbname == NULL) break; - - snprintf(sql, SHELL_SQL_LEN, "select * from %s limit 1", tbname); - - TAOS_RES *pSql = taos_query(pThread->taos, sql); - int32_t code = taos_errno(pSql); - if (code != 0) { - int32_t len = snprintf(sql, SHELL_SQL_LEN, "drop table %s.%s;\n", pThread->db, tbname); - taosWriteFile(pFile, sql, len); - atomic_add_fetch_32(&errorNum, 1); - } - - int32_t cnum = atomic_add_fetch_32(&checkedNum, 1); - if (cnum % 5000 == 0 && cnum != 0) { - fprintf(stdout, "%d tables checked\n", cnum); - } - - taos_free_result(pSql); - } - - taosFsync(pFile); - taosCloseFile(&pFile); - - return NULL; -} - -static void shellRunCheckThreads(TAOS *con, SShellArguments *_args) { - TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); - for (int t = 0; t < _args->threadNum; ++t) { - ShellThreadObj *pThread = threadObj + t; - pThread->threadIndex = t; - pThread->totalThreads = _args->threadNum; - pThread->taos = con; - pThread->db = _args->database; - - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - if (taosThreadCreate(&(pThread->threadID), &thattr, shellCheckThreadFp, (void *)pThread) != 0) { - fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); - exit(0); - } - } - - for (int t = 0; t < _args->threadNum; ++t) { - taosThreadJoin(threadObj[t].threadID, NULL); - } - - for (int t = 0; t < _args->threadNum; ++t) { - taos_close(threadObj[t].taos); - } - taosMemoryFree(threadObj); -} - -void shellCheck(TAOS *con, SShellArguments *_args) { - int64_t start = taosGetTimestampMs(); - - if (shellUseDb(con, _args->database) != 0) { - shellFreeTbnames(); - return; - } - - if (shellShowTables(con, _args->database) != 0) { - shellFreeTbnames(); - return; - } - - fprintf(stdout, "total %d tables will be checked by %d threads\n", tbNum, _args->threadNum); - shellRunCheckThreads(con, _args); - - int64_t end = taosGetTimestampMs(); - fprintf(stdout, "total %d tables checked, failed:%d, time spent %.2f seconds\n", checkedNum, errorNum, - (end - start) / 1000.0); -} diff --git a/tools/shell/src/backup/shellDarwin.c b/tools/shell/src/backup/shellDarwin.c deleted file mode 100644 index 93335776ba..0000000000 --- a/tools/shell/src/backup/shellDarwin.c +++ /dev/null @@ -1,498 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define __USE_XOPEN - -#include "os.h" - -#include "shell.h" -#include "shellCommand.h" -#include "tbase64.h" - -#include "tscLog.h" - -#define OPT_ABORT 1 /* �Cabort */ - -int indicator = 1; -struct termios oldtio; - -void insertChar(Command *cmd, char *c, int size); - - -void printHelp() { - char indent[10] = " "; - printf("taos shell is used to test the TDengine database\n"); - - printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost."); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); - printf("%s%s\n", indent, "-P"); - printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection"); - printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server."); - printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, "Configuration directory."); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell."); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s\n", indent, indent, "Output time as unsigned long.."); - printf("%s%s\n", indent, "-f"); - printf("%s%s%s\n", indent, indent, "Script to run without enter the shell."); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server."); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); - printf("%s%s\n", indent, "-D"); - printf("%s%s%s\n", indent, indent, "Use multi-thread to import all SQL files in the directory separately."); - printf("%s%s\n", indent, "-T"); - printf("%s%s%s\n", indent, indent, "Number of threads when using multi-thread to import data."); - - exit(EXIT_SUCCESS); -} - -char DARWINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[SHELL_MAX_PASSWORD_LEN]; - -void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { - wordexp_t full_path; - for (int i = 1; i < argc; i++) { - // for host - if (strcmp(argv[i], "-h") == 0) { - if (i < argc - 1) { - arguments->host = argv[++i]; - } else { - fprintf(stderr, "option -h requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for password - else if ((strncmp(argv[i], "-p", 2) == 0) - || (strncmp(argv[i], "--password", 10) == 0)) { - strcpy(tsOsName, "Darwin"); - printf(DARWINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%s", g_password) > 1) { - fprintf(stderr, "password read error\n"); - } - taosSetConsoleEcho(true); - getchar(); - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - } - arguments->password = g_password; - arguments->is_use_passwd = true; - strcpy(argv[i], ""); - argc -= 1; - } - // for management port - else if (strcmp(argv[i], "-P") == 0) { - if (i < argc - 1) { - arguments->port = atoi(argv[++i]); - } else { - fprintf(stderr, "option -P requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for user - else if (strcmp(argv[i], "-u") == 0) { - if (i < argc - 1) { - arguments->user = argv[++i]; - } else { - fprintf(stderr, "option -u requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { - fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1); - exit(EXIT_FAILURE); - } - strcpy(configDir, argv[i]); - } else { - fprintf(stderr, "Option -c requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-s") == 0) { - if (i < argc - 1) { - arguments->commands = argv[++i]; - } else { - fprintf(stderr, "option -s requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-r") == 0) { - arguments->is_raw_time = true; - } - // For temperory batch commands to run TODO - else if (strcmp(argv[i], "-f") == 0) { - if (i < argc - 1) { - strcpy(arguments->file, argv[++i]); - } else { - fprintf(stderr, "option -f requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for default database - else if (strcmp(argv[i], "-d") == 0) { - if (i < argc - 1) { - arguments->database = argv[++i]; - } else { - fprintf(stderr, "option -d requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For time zone - else if (strcmp(argv[i], "-t") == 0) { - if (i < argc - 1) { - arguments->timezone = argv[++i]; - } else { - fprintf(stderr, "option -t requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For import directory - else if (strcmp(argv[i], "-D") == 0) { - if (i < argc - 1) { - if (wordexp(argv[++i], &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", argv[i]); - exit(EXIT_FAILURE); - } - strcpy(arguments->dir, full_path.we_wordv[0]); - wordfree(&full_path); - } else { - fprintf(stderr, "option -D requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For time zone - else if (strcmp(argv[i], "-T") == 0) { - if (i < argc - 1) { - arguments->threadNum = atoi(argv[++i]); - } else { - fprintf(stderr, "option -T requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For temperory command TODO - else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(EXIT_FAILURE); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); - } - } -} - -int32_t shellReadCommand(TAOS *con, char *command) { - unsigned hist_counter = history.hend; - char utf8_array[10] = "\0"; - Command cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - showOnScreen(&cmd); - - // Read input. - char c; - while (1) { - c = getchar(); - - if (c < 0) { // For UTF-8 - int count = countPrefixOnes(c); - utf8_array[0] = c; - for (int k = 1; k < count; k++) { - c = getchar(); - utf8_array[k] = c; - } - insertChar(&cmd, utf8_array, count); - } else if (c < '\033') { - // Ctrl keys. TODO: Implement ctrl combinations - switch (c) { - case 1: // ctrl A - positionCursorHome(&cmd); - break; - case 3: - printf("\n"); - resetCommand(&cmd, ""); - kill(0, SIGINT); - break; - case 4: // EOF or Ctrl+D - printf("\n"); - taos_close(con); - // write the history - write_history(); - exitShell(); - break; - case 5: // ctrl E - positionCursorEnd(&cmd); - break; - case 8: - backspaceChar(&cmd); - break; - case '\n': - case '\r': - printf("\n"); - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosMemoryFreeClear(cmd.buffer); - taosMemoryFreeClear(cmd.command); - return 0; - } else { - updateBuffer(&cmd); - } - break; - case 11: // Ctrl + K; - clearLineAfter(&cmd); - break; - case 12: // Ctrl + L; - system("clear"); - showOnScreen(&cmd); - break; - case 21: // Ctrl + U - clearLineBefore(&cmd); - break; - } - } else if (c == '\033') { - c = getchar(); - switch (c) { - case '[': - c = getchar(); - switch (c) { - case 'A': // Up arrow - if (hist_counter != history.hstart) { - hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE; - resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]); - } - break; - case 'B': // Down arrow - if (hist_counter != history.hend) { - int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE; - - if (next_hist != history.hend) { - resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]); - } else { - resetCommand(&cmd, ""); - } - hist_counter = next_hist; - } - break; - case 'C': // Right arrow - moveCursorRight(&cmd); - break; - case 'D': // Left arrow - moveCursorLeft(&cmd); - break; - case '1': - if ((c = getchar()) == '~') { - // Home key - positionCursorHome(&cmd); - } - break; - case '2': - if ((c = getchar()) == '~') { - // Insert key - } - break; - case '3': - if ((c = getchar()) == '~') { - // Delete key - deleteChar(&cmd); - } - break; - case '4': - if ((c = getchar()) == '~') { - // End key - positionCursorEnd(&cmd); - } - break; - case '5': - if ((c = getchar()) == '~') { - // Page up key - } - break; - case '6': - if ((c = getchar()) == '~') { - // Page down key - } - break; - case 72: - // Home key - positionCursorHome(&cmd); - break; - case 70: - // End key - positionCursorEnd(&cmd); - break; - } - break; - } - } else if (c == 0x7f) { - // press delete key - backspaceChar(&cmd); - } else { - insertChar(&cmd, &c, 1); - } - } - - return 0; -} - -void *shellLoopQuery(void *arg) { - if (indicator) { - getOldTerminalMode(); - indicator = 0; - } - - TAOS *con = (TAOS *)arg; - - setThreadName("shellLoopQuery"); - - taosThreadCleanupPush(cleanup_handler, NULL); - - char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); - if (command == NULL){ - tscError("failed to malloc command"); - return NULL; - } - - int32_t err = 0; - - do { - // Read command from shell. - memset(command, 0, MAX_COMMAND_SIZE); - setTerminalMode(); - err = shellReadCommand(con, command); - if (err) { - break; - } - resetTerminalMode(); - } while (shellRunCommand(con, command) == 0); - - taosMemoryFreeClear(command); - exitShell(); - - taosThreadCleanupPop(1); - - return NULL; -} - -void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid())->pw_dir, HISTORY_FILE); } - -void clearScreen(int ecmd_pos, int cursor_pos) { - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - positionCursor(cursor_y, LEFT); - positionCursor(command_x - cursor_x, DOWN); - fprintf(stdout, "\033[2K"); - for (int i = 0; i < command_x; i++) { - positionCursor(1, UP); - fprintf(stdout, "\033[2K"); - } - fflush(stdout); -} - -void showOnScreen(Command *cmd) { - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - //fprintf(stderr, "No stream device\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - TdWchar wc; - int size = 0; - - // Print out the command. - char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); - memset(total_string, '\0', MAX_COMMAND_SIZE); - if (strcmp(cmd->buffer, "") == 0) { - sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); - } else { - sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command); - } - - int remain_column = w.ws_col; - /* size = cmd->commandSize + prompt_size; */ - for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); - if (ret < 0) break; - size += ret; - /* assert(size >= 0); */ - int width = taosWcharWidth(wc); - if (remain_column > width) { - printf("%lc", wc); - remain_column -= width; - } else { - if (remain_column == width) { - printf("%lc\n\r", wc); - remain_column = w.ws_col; - } else { - printf("\n\r%lc", wc); - remain_column = w.ws_col - width; - } - } - - str = total_string + size; - } - - taosMemoryFree(total_string); - /* for (int i = 0; i < size; i++){ */ - /* char c = total_string[i]; */ - /* if (k % w.ws_col == 0) { */ - /* printf("%c\n\r", c); */ - /* } */ - /* else { */ - /* printf("%c", c); */ - /* } */ - /* k += 1; */ - /* } */ - - // Position the cursor - int cursor_pos = cmd->screenOffset + prompt_size; - int ecmd_pos = cmd->endOffset + prompt_size; - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - // int cursor_y = cursor % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - int command_y = ecmd_pos % w.ws_col; - // int command_y = (command.size() + prompt_size) % w.ws_col; - positionCursor(command_y, LEFT); - positionCursor(command_x, UP); - positionCursor(cursor_x, DOWN); - positionCursor(cursor_y, RIGHT); - fflush(stdout); -} - -void cleanup_handler(void *arg) { resetTerminalMode(); } - -void exitShell() { - resetTerminalMode(); - exit(EXIT_SUCCESS); -} diff --git a/tools/shell/src/backup/shellImport.c b/tools/shell/src/backup/shellImport.c deleted file mode 100644 index 130c72a20b..0000000000 --- a/tools/shell/src/backup/shellImport.c +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _GNU_SOURCE -#define _XOPEN_SOURCE -#define _DEFAULT_SOURCE - -#include "os.h" -#include "shell.h" -#include "shellCommand.h" -#include "tglobal.h" -#include "tutil.h" - -static char **shellSQLFiles = NULL; -static int32_t shellSQLFileNum = 0; -static char shellTablesSQLFile[TSDB_FILENAME_LEN] = {0}; - -typedef struct { - TdThread threadID; - int threadIndex; - int totalThreads; - void *taos; -} ShellThreadObj; - -static int shellGetFilesNum(const char *directoryName, const char *prefix) -{ - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | wc -l ", directoryName, prefix); - - char buf[1024] = { 0 }; - if (taosSystem(cmd, buf, sizeof(buf)) < 0) { - fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(0); - } - - int fileNum = 0; - if (sscanf(buf, "%d", &fileNum) != 1) { - fprintf(stderr, "ERROR: failed to execute:%s, parse result error\n", cmd); - exit(0); - } - - if (fileNum <= 0) { - fprintf(stderr, "ERROR: directory:%s is empry\n", directoryName); - exit(0); - } - - return fileNum; -} - -static void shellParseDirectory(const char *directoryName, const char *prefix, char **fileArray, int totalFiles) -{ - char cmd[1024] = { 0 }; - sprintf(cmd, "ls %s/*.%s | sort", directoryName, prefix); - - char buf[1024] = { 0 }; - if (taosSystem(cmd, buf, sizeof(buf)) < 0) { - fprintf(stderr, "ERROR: failed to execute:%s, error:%s\n", cmd, strerror(errno)); - exit(0); - } - - int fileNum = 0; - while (sscanf(buf, "%128s", fileArray[fileNum++])) { - if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) { - fileNum--; - } - if (fileNum >= totalFiles) { - break; - } - } - - if (fileNum != totalFiles) { - fprintf(stderr, "ERROR: directory:%s changed while read\n", directoryName); - exit(0); - } -} - -static void shellCheckTablesSQLFile(const char *directoryName) -{ - sprintf(shellTablesSQLFile, "%s/tables.sql", directoryName); - - if (taosFStatFile(shellTablesSQLFile, NULL, NULL) < 0) { - shellTablesSQLFile[0] = 0; - } -} - -static void shellMallocSQLFiles() -{ - shellSQLFiles = (char**)taosMemoryCalloc(shellSQLFileNum, sizeof(char*)); - for (int i = 0; i < shellSQLFileNum; i++) { - shellSQLFiles[i] = taosMemoryCalloc(1, TSDB_FILENAME_LEN); - } -} - -static void shellGetDirectoryFileList(char *inputDir) -{ - if (!taosDirExist(inputDir)) { - fprintf(stderr, "ERROR: %s not exist\n", inputDir); - exit(0); - } - - if (taosIsDir(inputDir)) { - shellCheckTablesSQLFile(inputDir); - shellSQLFileNum = shellGetFilesNum(inputDir, "sql"); - int totalSQLFileNum = shellSQLFileNum; - if (shellTablesSQLFile[0] != 0) { - shellSQLFileNum--; - } - shellMallocSQLFiles(); - shellParseDirectory(inputDir, "sql", shellSQLFiles, shellSQLFileNum); - fprintf(stdout, "\nstart to dispose %d files in %s\n", totalSQLFileNum, inputDir); - } - else { - fprintf(stderr, "ERROR: %s is not a directory\n", inputDir); - exit(0); - } -} - -static void shellSourceFile(TAOS *con, char *fptr) { - wordexp_t full_path; - int read_len = 0; - char * cmd = taosMemoryMalloc(tsMaxSQLStringLen); - size_t cmd_len = 0; - char * line = NULL; - - if (wordexp(fptr, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: illegal file name\n"); - taosMemoryFree(cmd); - return; - } - - char *fname = full_path.we_wordv[0]; - if (fname == NULL) { - fprintf(stderr, "ERROR: invalid filename\n"); - taosMemoryFree(cmd); - return; - } - - /* - if (access(fname, F_OK) != 0) { - fprintf(stderr, "ERROR: file %s is not exist\n", fptr); - - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - - if (access(fname, R_OK) != 0) { - fprintf(stderr, "ERROR: file %s is not readable\n", fptr); - - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - */ - - // FILE *f = fopen(fname, "r"); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file %s\n", fname); - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - - fprintf(stdout, "begin import file:%s\n", fname); - - int lineNo = 0; - while ((read_len = taosGetLineFile(pFile, &line)) != -1) { - ++lineNo; - if (read_len >= tsMaxSQLStringLen) continue; - line[--read_len] = '\0'; - - if (read_len == 0 || isCommentLine(line)) { // line starts with # - continue; - } - - if (line[read_len - 1] == '\\') { - line[read_len - 1] = ' '; - memcpy(cmd + cmd_len, line, read_len); - cmd_len += read_len; - continue; - } - - memcpy(cmd + cmd_len, line, read_len); - - TAOS_RES* pSql = taos_query(con, cmd); - int32_t code = taos_errno(pSql); - - if (code != 0) { - fprintf(stderr, "DB error: %s: %s (%d)\n", taos_errstr(pSql), fname, lineNo); - } - - /* free local resouce: allocated memory/metric-meta refcnt */ - taos_free_result(pSql); - - memset(cmd, 0, MAX_COMMAND_SIZE); - cmd_len = 0; - } - - taosMemoryFree(cmd); - if(line != NULL) taosMemoryFree(line); - wordfree(&full_path); - taosCloseFile(&pFile); -} - -void* shellImportThreadFp(void *arg) -{ - ShellThreadObj *pThread = (ShellThreadObj*)arg; - setThreadName("shellImportThrd"); - - for (int f = 0; f < shellSQLFileNum; ++f) { - if (f % pThread->totalThreads == pThread->threadIndex) { - char *SQLFileName = shellSQLFiles[f]; - shellSourceFile(pThread->taos, SQLFileName); - } - } - - return NULL; -} - -static void shellRunImportThreads(SShellArguments* _args) -{ - TdThreadAttr thattr; - ShellThreadObj *threadObj = (ShellThreadObj *)taosMemoryCalloc(_args->threadNum, sizeof(ShellThreadObj)); - for (int t = 0; t < _args->threadNum; ++t) { - ShellThreadObj *pThread = threadObj + t; - pThread->threadIndex = t; - pThread->totalThreads = _args->threadNum; - pThread->taos = taos_connect(_args->host, _args->user, _args->password, _args->database, tsDnodeShellPort); - if (pThread->taos == NULL) { - fprintf(stderr, "ERROR: thread:%d failed connect to TDengine, error:%s\n", pThread->threadIndex, "null taos"/*taos_errstr(pThread->taos)*/); - exit(0); - } - - taosThreadAttrInit(&thattr); - taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); - - if (taosThreadCreate(&(pThread->threadID), &thattr, shellImportThreadFp, (void*)pThread) != 0) { - fprintf(stderr, "ERROR: thread:%d failed to start\n", pThread->threadIndex); - exit(0); - } - } - - for (int t = 0; t < _args->threadNum; ++t) { - taosThreadJoin(threadObj[t].threadID, NULL); - } - - for (int t = 0; t < _args->threadNum; ++t) { - taos_close(threadObj[t].taos); - } - taosMemoryFree(threadObj); -} - -void source_dir(TAOS* con, SShellArguments* _args) { - shellGetDirectoryFileList(_args->dir); - int64_t start = taosGetTimestampMs(); - - if (shellTablesSQLFile[0] != 0) { - shellSourceFile(con, shellTablesSQLFile); - int64_t end = taosGetTimestampMs(); - fprintf(stdout, "import %s finished, time spent %.2f seconds\n", shellTablesSQLFile, (end - start) / 1000.0); - } - - shellRunImportThreads(_args); - int64_t end = taosGetTimestampMs(); - fprintf(stdout, "import %s finished, time spent %.2f seconds\n", _args->dir, (end - start) / 1000.0); -} diff --git a/tools/shell/src/backup/shellWindows.c b/tools/shell/src/backup/shellWindows.c deleted file mode 100644 index 92ac7fd721..0000000000 --- a/tools/shell/src/backup/shellWindows.c +++ /dev/null @@ -1,325 +0,0 @@ -/******************************************************************* -* Copyright (c) 2017 by TAOS Technologies, Inc. -* All rights reserved. -* -* This file is proprietary and confidential to TAOS Technologies. -* No part of this file may be reproduced, stored, transmitted, -* disclosed or used in any form or by any means other than as -* expressly provided by the written permission from Jianhui Tao -* -* ****************************************************************/ - -#include -#include -#include -#include "../../../../include/client/taos.h" -#include "os.h" -#include "shell.h" -#include "shellCommand.h" - -extern char configDir[]; - -char WINCLIENT_VERSION[] = "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; - -void printVersion() { - printf("version: %s\n", version); -} - -void printHelp() { - char indent[10] = " "; - printf("taos shell is used to test the TDengine database\n"); - - printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDengine server FQDN to connect. The default host is localhost."); - printf("%s%s\n", indent, "-p"); - printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); - printf("%s%s\n", indent, "-P"); - printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection"); - printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server."); - printf("%s%s\n", indent, "-A"); - printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server."); - printf("%s%s\n", indent, "-c"); - printf("%s%s%s\n", indent, indent, "Configuration directory."); - printf("%s%s\n", indent, "-C"); - printf("%s%s%s\n", indent, indent, "Dump configuration."); - printf("%s%s\n", indent, "-s"); - printf("%s%s%s\n", indent, indent, "Commands to run without enter the shell."); - printf("%s%s\n", indent, "-r"); - printf("%s%s%s\n", indent, indent, "Output time as unsigned long.."); - printf("%s%s\n", indent, "-f"); - printf("%s%s%s\n", indent, indent, "Script to run without enter the shell."); - printf("%s%s\n", indent, "-d"); - printf("%s%s%s\n", indent, indent, "Database to use when connecting to the server."); - printf("%s%s\n", indent, "-t"); - printf("%s%s%s\n", indent, indent, "Time zone of the shell, default is local."); - printf("%s%s\n", indent, "-n"); - printf("%s%s%s\n", indent, indent, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."); - printf("%s%s\n", indent, "-l"); - printf("%s%s%s\n", indent, indent, "Packet length used for net test, default is 1000 bytes."); - printf("%s%s\n", indent, "-N"); - printf("%s%s%s\n", indent, indent, "Packet numbers used for net test, default is 100."); - printf("%s%s\n", indent, "-S"); - printf("%s%s%s\n", indent, indent, "Packet type used for net test, default is TCP."); - printf("%s%s\n", indent, "-V"); - printf("%s%s%s\n", indent, indent, "Print program version."); - - exit(EXIT_SUCCESS); -} - -char g_password[SHELL_MAX_PASSWORD_LEN]; - -void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { - for (int i = 1; i < argc; i++) { - // for host - if (strcmp(argv[i], "-h") == 0) { - if (i < argc - 1) { - arguments->host = argv[++i]; - } else { - fprintf(stderr, "option -h requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for password - else if ((strncmp(argv[i], "-p", 2) == 0) - || (strncmp(argv[i], "--password", 10) == 0)) { - arguments->is_use_passwd = true; - strcpy(tsOsName, "Windows"); - printf(WINCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) - || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%s", g_password) > 1) { - fprintf(stderr, "password read error!\n"); - } - taosSetConsoleEcho(true); - getchar(); - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - } - arguments->password = g_password; - strcpy(argv[i], ""); - argc -= 1; - } - // for management port - else if (strcmp(argv[i], "-P") == 0) { - if (i < argc - 1) { - arguments->port = atoi(argv[++i]); - } else { - fprintf(stderr, "option -P requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for user - else if (strcmp(argv[i], "-u") == 0) { - if (i < argc - 1) { - arguments->user = argv[++i]; - } else { - fprintf(stderr, "option -u requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-A") == 0) { - if (i < argc - 1) { - arguments->auth = argv[++i]; - } else { - fprintf(stderr, "option -A requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-c") == 0) { - if (i < argc - 1) { - char *tmp = argv[++i]; - if (strlen(tmp) >= TSDB_FILENAME_LEN) { - fprintf(stderr, "config file path: %s overflow max len %d\n", tmp, TSDB_FILENAME_LEN - 1); - exit(EXIT_FAILURE); - } - strcpy(configDir, tmp); - } else { - fprintf(stderr, "Option -c requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-C") == 0) { - arguments->dump_config = true; - } else if (strcmp(argv[i], "-s") == 0) { - if (i < argc - 1) { - arguments->commands = argv[++i]; - } else { - fprintf(stderr, "option -s requires an argument\n"); - exit(EXIT_FAILURE); - } - } else if (strcmp(argv[i], "-r") == 0) { - arguments->is_raw_time = true; - } - // For temperory batch commands to run TODO - else if (strcmp(argv[i], "-f") == 0) { - if (i < argc - 1) { - strcpy(arguments->file, argv[++i]); - } else { - fprintf(stderr, "option -f requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // for default database - else if (strcmp(argv[i], "-d") == 0) { - if (i < argc - 1) { - arguments->database = argv[++i]; - } else { - fprintf(stderr, "option -d requires an argument\n"); - exit(EXIT_FAILURE); - } - } - // For time zone - else if (strcmp(argv[i], "-t") == 0) { - if (i < argc - 1) { - arguments->timezone = argv[++i]; - } else { - fprintf(stderr, "option -t requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-n") == 0) { - if (i < argc - 1) { - arguments->netTestRole = argv[++i]; - } else { - fprintf(stderr, "option -n requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-l") == 0) { - if (i < argc - 1) { - arguments->pktLen = atoi(argv[++i]); - } else { - fprintf(stderr, "option -l requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-N") == 0) { - if (i < argc - 1) { - arguments->pktNum = atoi(argv[++i]); - } else { - fprintf(stderr, "option -N requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-S") == 0) { - if (i < argc - 1) { - arguments->pktType = argv[++i]; - } else { - fprintf(stderr, "option -S requires an argument\n"); - exit(EXIT_FAILURE); - } - } - else if (strcmp(argv[i], "-V") == 0) { - printVersion(); - exit(EXIT_SUCCESS); - } - // For temperory command TODO - else if (strcmp(argv[i], "--help") == 0) { - printHelp(); - exit(EXIT_SUCCESS); - } else { - fprintf(stderr, "wrong options\n"); - printHelp(); - exit(EXIT_FAILURE); - } - } -} - -void shellPrintContinuePrompt() { printf("%s", CONTINUE_PROMPT); } - -void shellPrintPrompt() { printf("%s", PROMPT_HEADER); } - -void updateBuffer(Command *cmd) { - if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); - strcat(cmd->buffer, cmd->command); - - memset(cmd->command, 0, MAX_COMMAND_SIZE); - cmd->cursorOffset = 0; -} - -int isReadyGo(Command *cmd) { - char *total = taosMemoryMalloc(MAX_COMMAND_SIZE); - memset(total, 0, MAX_COMMAND_SIZE); - sprintf(total, "%s%s", cmd->buffer, cmd->command); - - char *reg_str = - "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" - "\\s*clear\\s*$)"; - if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { - taosMemoryFree(total); - return 1; - } - - taosMemoryFree(total); - return 0; -} - -void insertChar(Command *cmd, char c) { - // TODO: Check if the length enough. - if (cmd->cursorOffset >= MAX_COMMAND_SIZE) { - fprintf(stdout, "sql is larger than %d bytes", MAX_COMMAND_SIZE); - return; - } - - cmd->command[cmd->cursorOffset++] = c; -} - -int32_t shellReadCommand(TAOS *con, char command[]) { - Command cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - - // Read input. - char c; - while (1) { - c = getchar(); - - switch (c) { - case '\n': - case '\r': - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosMemoryFree(cmd.buffer); - cmd.buffer = NULL; - taosMemoryFree(cmd.command); - cmd.command = NULL; - return 0; - } else { - shellPrintContinuePrompt(); - updateBuffer(&cmd); - } - break; - default: - insertChar(&cmd, c); - } - } - - return 0; -} - -void *shellLoopQuery(void *arg) { - TAOS *con = (TAOS *)arg; - char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); - if (command == NULL) return NULL; - - int32_t err = 0; - - do { - memset(command, 0, MAX_COMMAND_SIZE); - shellPrintPrompt(); - - // Read command from shell. - err = shellReadCommand(con, command); - if (err) { - break; - } - } while (shellRunCommand(con, command) == 0); - - return NULL; -} - -void get_history_path(char *history) { sprintf(history, "C:/TDengine/%s", HISTORY_FILE); } - -void exitShell() { exit(EXIT_SUCCESS); } diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c new file mode 100644 index 0000000000..5391d28277 --- /dev/null +++ b/tools/shell/src/shellArguments.c @@ -0,0 +1,354 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "shellInt.h" + +#define SHELL_HOST "The auth string to use when connecting to the server." +#define SHELL_PORT "The TCP/IP port number to use for the connection." +#define SHELL_USER "The user name to use when connecting to the server." +#define SHELL_PASSWORD "The password to use when connecting to the server." +#define SHELL_AUTH "The auth string to use when connecting to the server." +#define SHELL_GEN_AUTH "Generate auth string from password." +#define SHELL_CFG_DIR "Configuration directory." +#define SHELL_DMP_CFG "Dump configuration." +#define SHELL_CMD "Commands to run without enter the shell." +#define SHELL_RAW_TIME "Output time as uint64_t." +#define SHELL_FILE "Script to run without enter the shell." +#define SHELL_DB "Database to use when connecting to the server." +#define SHELL_CHECK "Check the service status." +#define SHELL_STARTUP "Check the details of the service status." +#define SHELL_WIDTH "Set the default binary display width, default is 30." +#define SHELL_NET_ROLE "Net role when network connectivity test, options: client|server." +#define SHELL_PKG_LEN "Packet length used for net test, default is 1024 bytes." +#define SHELL_PKT_NUM "Packet numbers used for net test, default is 100." +#define SHELL_VERSION "Print program version." +#define SHELL_EMAIL "" + +void shellPrintHelp() { + char indent[] = " "; + printf("Usage: taos [OPTION...] \n\n"); + printf("%s%s%s%s\n", indent, "-a,", indent, SHELL_AUTH); + printf("%s%s%s%s\n", indent, "-A,", indent, SHELL_GEN_AUTH); + printf("%s%s%s%s\n", indent, "-c,", indent, SHELL_CFG_DIR); + printf("%s%s%s%s\n", indent, "-C,", indent, SHELL_DMP_CFG); + printf("%s%s%s%s\n", indent, "-d,", indent, SHELL_DB); + printf("%s%s%s%s\n", indent, "-f,", indent, SHELL_FILE); + printf("%s%s%s%s\n", indent, "-h,", indent, SHELL_HOST); + printf("%s%s%s%s\n", indent, "-k,", indent, SHELL_CHECK); + printf("%s%s%s%s\n", indent, "-l,", indent, SHELL_PKG_LEN); + printf("%s%s%s%s\n", indent, "-n,", indent, SHELL_NET_ROLE); + printf("%s%s%s%s\n", indent, "-N,", indent, SHELL_PKT_NUM); + printf("%s%s%s%s\n", indent, "-p,", indent, SHELL_PASSWORD); + printf("%s%s%s%s\n", indent, "-P,", indent, SHELL_PORT); + printf("%s%s%s%s\n", indent, "-r,", indent, SHELL_RAW_TIME); + printf("%s%s%s%s\n", indent, "-s,", indent, SHELL_CMD); + printf("%s%s%s%s\n", indent, "-t,", indent, SHELL_STARTUP); + printf("%s%s%s%s\n", indent, "-u,", indent, SHELL_USER); + printf("%s%s%s%s\n", indent, "-w,", indent, SHELL_WIDTH); + printf("%s%s%s%s\n", indent, "-V,", indent, SHELL_VERSION); + printf("\n\nReport bugs to %s.\n", SHELL_EMAIL); +} + +static int32_t shellParseSingleOpt(int32_t key, char *arg) { + SShellArgs *pArgs = &shell.args; + + switch (key) { + case 'h': + pArgs->host = arg; + break; + case 'P': + pArgs->port = atoi(arg); + break; + case 'u': + pArgs->user = arg; + break; + case 'p': + break; + case 'a': + pArgs->auth = arg; + break; + case 'A': + pArgs->is_gen_auth = true; + break; + case 'c': + pArgs->cfgdir = arg; + break; + case 'C': + pArgs->is_dump_config = true; + break; + case 's': + pArgs->commands = arg; + break; + case 'r': + pArgs->is_raw_time = true; + break; + case 'f': + tstrncpy(pArgs->file, arg, sizeof(pArgs->file)); + break; + case 'd': + pArgs->database = arg; + break; + case 'k': + pArgs->is_check = true; + break; + case 't': + pArgs->is_startup = true; + break; + case 'w': + pArgs->displayWidth = atoi(arg); + break; + case 'n': + pArgs->netrole = arg; + break; + case 'l': + pArgs->pktLen = atoi(arg); + break; + case 'N': + pArgs->pktNum = atoi(arg); + break; + case 'V': + pArgs->is_version = true; + break; + case '?': + pArgs->is_help = true; + break; + case 1: + pArgs->abort = 1; + break; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; +} + +int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { + SShellArgs *pArgs = &shell.args; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0) { + shellParseSingleOpt('?', NULL); + return 0; + } + + char *key = argv[i]; + int32_t keyLen = strlen(key); + if (keyLen != 2) { + fprintf(stderr, "invalid option %s\n", key); + return -1; + } + if (key[0] != '-') { + fprintf(stderr, "invalid option %s\n", key); + return -1; + } + + if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || + key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N') { + if (i + 1 >= argc) { + fprintf(stderr, "option %s requires an argument\n", key); + return -1; + } + char *val = argv[i + 1]; + if (val[0] == '-') { + fprintf(stderr, "option %s requires an argument\n", key); + return -1; + } + shellParseSingleOpt(key[1], val); + i++; + } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'c' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' || + key[1] == 'V') { + shellParseSingleOpt(key[1], NULL); + } else { + fprintf(stderr, "invalid option %s\n", key); + return -1; + } + } + + return 0; +} + +#ifdef LINUX +#include +#include + +const char *argp_program_version = version; +const char *argp_program_bug_address = SHELL_EMAIL; + +static struct argp_option shellOptions[] = { + {"host", 'h', "HOST", 0, SHELL_HOST}, + {"port", 'P', "PORT", 0, SHELL_PORT}, + {"user", 'u', "USER", 0, SHELL_USER}, + {0, 'p', 0, 0, SHELL_PASSWORD}, + {"auth", 'a', "AUTH", 0, SHELL_AUTH}, + {"generate-auth", 'A', 0, 0, SHELL_GEN_AUTH}, + {"config-dir", 'c', "DIR", 0, SHELL_CFG_DIR}, + {"dump-config", 'C', 0, 0, SHELL_DMP_CFG}, + {"commands", 's', "COMMANDS", 0, SHELL_CMD}, + {"raw-time", 'r', 0, 0, SHELL_RAW_TIME}, + {"file", 'f', "FILE", 0, SHELL_FILE}, + {"database", 'd', "DATABASE", 0, SHELL_DB}, + {"check", 'k', 0, 0, SHELL_CHECK}, + {"startup", 't', 0, 0, SHELL_STARTUP}, + {"display-width", 'w', "WIDTH", 0, SHELL_WIDTH}, + {"netrole", 'n', "NETROLE", 0, SHELL_NET_ROLE}, + {"pktlen", 'l', "PKTLEN", 0, SHELL_PKG_LEN}, + {"pktnum", 'N', "PKTNUM", 0, SHELL_PKT_NUM}, + {0}, +}; + +static error_t shellParseOpt(int32_t key, char *arg, struct argp_state *state) { return shellParseSingleOpt(key, arg); } + +static struct argp shellArgp = {shellOptions, shellParseOpt, "", ""}; + +static void shellParseArgsUseArgp(int argc, char *argv[]) { + argp_program_version = shell.info.programVersion; + argp_parse(&shellArgp, argc, argv, 0, 0, &shell.args); +} + +#endif + +static void shellInitArgs(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strncmp(argv[i], "-p", 2) == 0) { + printf(shell.info.clientVersion, tsOsName, taos_get_client_info()); + if (strlen(argv[i]) == 2) { + printf("Enter password: "); + taosSetConsoleEcho(false); + if (scanf("%20s", shell.args.password) > 1) { + fprintf(stderr, "password reading error\n"); + } + taosSetConsoleEcho(true); + if (EOF == getchar()) { + fprintf(stderr, "getchar() return EOF\n"); + } + } else { + tstrncpy(shell.args.password, (char *)(argv[i] + 2), sizeof(shell.args.password)); + strcpy(argv[i], "-p"); + } + } + } + if (strlen(shell.args.password) == 0) { + tstrncpy(shell.args.password, TSDB_DEFAULT_PASS, sizeof(shell.args.password)); + } + + SShellArgs *pArgs = &shell.args; + pArgs->user = TSDB_DEFAULT_USER; + pArgs->pktLen = SHELL_DEF_PKG_LEN; + pArgs->pktNum = SHELL_DEF_PKG_NUM; + pArgs->displayWidth = SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH; +} + +static int32_t shellCheckArgs() { + SShellArgs *pArgs = &shell.args; + if (pArgs->host != NULL && (strlen(pArgs->host) <= 0 || strlen(pArgs->host) > TSDB_FQDN_LEN)) { + printf("Invalid host:%s\n", pArgs->host); + return -1; + } + + if (pArgs->user != NULL && (strlen(pArgs->user) <= 0 || strlen(pArgs->user) > TSDB_USER_LEN)) { + printf("Invalid user:%s\n", pArgs->user); + return -1; + } + + if (pArgs->auth != NULL && (strlen(pArgs->auth) <= 0 || strlen(pArgs->auth) > TSDB_PASSWORD_LEN)) { + printf("Invalid auth:%s\n", pArgs->auth); + return -1; + } + + if (pArgs->database != NULL && (strlen(pArgs->database) <= 0 || strlen(pArgs->database) > TSDB_DB_NAME_LEN)) { + printf("Invalid database:%s\n", pArgs->database); + return -1; + } + + if (pArgs->file[0] != 0) { + char fullname[PATH_MAX] = {0}; + if (taosExpandDir(pArgs->file, fullname, PATH_MAX) == 0) { + tstrncpy(pArgs->file, fullname, PATH_MAX); + } + } + + if (pArgs->cfgdir != NULL) { + if (strlen(pArgs->cfgdir) <= 0 || strlen(pArgs->cfgdir) >= PATH_MAX) { + printf("Invalid cfgdir:%s\n", pArgs->cfgdir); + return -1; + } else { + if (taosExpandDir(pArgs->cfgdir, configDir, PATH_MAX) != 0) { + tstrncpy(configDir, pArgs->cfgdir, PATH_MAX); + } + } + } + + if (pArgs->commands != NULL && (strlen(pArgs->commands) <= 0)) { + printf("Invalid commands:%s\n", pArgs->commands); + return -1; + } + + if (pArgs->netrole != NULL && !(strcmp(pArgs->netrole, "client") == 0 || strcmp(pArgs->netrole, "server") == 0)) { + printf("Invalid netrole:%s\n", pArgs->netrole); + return -1; + } + + if (pArgs->password != NULL && (strlen(pArgs->password) <= 0)) { + printf("Invalid password\n"); + return -1; + } + + if (pArgs->pktLen < SHELL_MIN_PKG_LEN || pArgs->pktLen > SHELL_MAX_PKG_LEN) { + printf("Invalid pktLen:%d, range:[%d, %d]\n", pArgs->pktLen, SHELL_MIN_PKG_LEN, SHELL_MAX_PKG_LEN); + return -1; + } + + if (pArgs->pktNum < SHELL_MIN_PKG_NUM || pArgs->pktNum > SHELL_MAX_PKG_NUM) { + printf("Invalid pktNum:%d, range:[%d, %d]\n", pArgs->pktNum, SHELL_MIN_PKG_NUM, SHELL_MAX_PKG_NUM); + return -1; + } + + if (pArgs->displayWidth <= 0 || pArgs->displayWidth > 10 * 1024) { + printf("Invalid displayWidth:%d, range:[1, 10 * 1024]\n", pArgs->displayWidth); + return -1; + } + + return 0; +} + +int32_t shellParseArgs(int32_t argc, char *argv[]) { + shellInitArgs(argc, argv); + shell.info.clientVersion = + "Welcome to the TDengine shell from %s, Client Version:%s\n" + "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; + shell.info.promptHeader = "taos> "; + shell.info.promptContinue = " -> "; + shell.info.promptSize = 6; + snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "version: %s", version); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + shell.info.osname = "Windows"; + snprintf(shell.history.file, TSDB_FILENAME_LEN, "C:/TDengine/%s", SHELL_HISTORY_FILE); + if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; +#elif defined(_TD_DARWIN_64) + shell.info.osname = "Darwin"; + snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getpwuid(getuid())->pw_dir, SHELL_HISTORY_FILE); + if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; +#else + shell.info.osname = "Linux"; + snprintf(shell.history.file, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), SHELL_HISTORY_FILE); + shellParseArgsUseArgp(argc, argv); + // if (shellParseArgsWithoutArgp(argc, argv) != 0) return -1; + if (shell.args.abort) { + return -1; + } +#endif + + return shellCheckArgs(); +} diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index 546b19f83c..fcdbf2d020 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -14,22 +14,122 @@ */ #define __USE_XOPEN +#include "shellInt.h" -#include "shellCommand.h" -#include "os.h" -#include "shell.h" - -#include +#define LEFT 1 +#define RIGHT 2 +#define UP 3 +#define DOWN 4 +#define PSIZE shell.info.promptSize typedef struct { - char widthInString; - char widthOnScreen; -} UTFCodeInfo; + char *buffer; + char *command; + uint32_t commandSize; + uint32_t bufferSize; + uint32_t cursorOffset; + uint32_t screenOffset; + uint32_t endOffset; +} SShellCmd; -int countPrefixOnes(unsigned char c) { - unsigned char mask = 127; +static int32_t shellCountPrefixOnes(uint8_t c); +static void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width); +static void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width); +static void shellInsertChar(SShellCmd *cmd, char *c, int size); +static void shellBackspaceChar(SShellCmd *cmd); +static void shellClearLineBefore(SShellCmd *cmd); +static void shellClearLineAfter(SShellCmd *cmd); +static void shellDeleteChar(SShellCmd *cmd); +static void shellMoveCursorLeft(SShellCmd *cmd); +static void shellMoveCursorRight(SShellCmd *cmd); +static void shellPositionCursorHome(SShellCmd *cmd); +static void shellPositionCursorEnd(SShellCmd *cmd); +static void shellPrintChar(char c, int32_t times); +static void shellPositionCursor(int32_t step, int32_t direction); +static void shellUpdateBuffer(SShellCmd *cmd); +static int32_t shellIsReadyGo(SShellCmd *cmd); +static void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width); +static void shellResetCommand(SShellCmd *cmd, const char s[]); +static void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos); +static void shellShowOnScreen(SShellCmd *cmd); + +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +static void shellPrintContinuePrompt() { printf("%s", shell.args.promptContinue); } +static void shellPrintPrompt() { printf("%s", shell.args.promptHeader); } + +void shellUpdateBuffer(SShellCmd *cmd) { + if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); + strcat(cmd->buffer, cmd->command); + + memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE); + cmd->cursorOffset = 0; +} + +int shellIsReadyGo(SShellCmd *cmd) { + char *total = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); + memset(total, 0, SHELL_MAX_COMMAND_SIZE); + sprintf(total, "%s%s", cmd->buffer, cmd->command); + + char *reg_str = + "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" + "\\s*clear\\s*$)"; + if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { + taosMemoryFree(total); + return 1; + } + + taosMemoryFree(total); + return 0; +} + +void shellInsertChar(SShellCmd *cmd, char c) { + if (cmd->cursorOffset >= SHELL_MAX_COMMAND_SIZE) { + fprintf(stdout, "sql is larger than %d bytes", SHELL_MAX_COMMAND_SIZE); + return; + } + cmd->command[cmd->cursorOffset++] = c; +} + +int32_t shellReadCommand(char command[]) { + SShellCmd cmd; + memset(&cmd, 0, sizeof(cmd)); + cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + + // Read input. + char c; + while (1) { + c = getchar(); + + switch (c) { + case '\n': + case '\r': + if (shellIsReadyGo(&cmd)) { + sprintf(command, "%s%s", cmd.buffer, cmd.command); + taosMemoryFree(cmd.buffer); + cmd.buffer = NULL; + taosMemoryFree(cmd.command); + cmd.command = NULL; + return 0; + } else { + shellPrintContinuePrompt(); + shellUpdateBuffer(&cmd); + } + break; + default: + shellInsertChar(&cmd, c); + } + } + + return 0; +} + +#else + +int32_t shellCountPrefixOnes(uint8_t c) { + uint8_t mask = 127; mask = ~mask; - int ret = 0; + int32_t ret = 0; while ((c & mask) != 0) { ret++; c <<= 1; @@ -38,7 +138,7 @@ int countPrefixOnes(unsigned char c) { return ret; } -void getPrevCharSize(const char *str, int pos, int *size, int *width) { +void shellGetPrevCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { assert(pos > 0); TdWchar wc; @@ -48,16 +148,16 @@ void getPrevCharSize(const char *str, int pos, int *size, int *width) { while (--pos >= 0) { *size += 1; - if (str[pos] > 0 || countPrefixOnes((unsigned char)str[pos]) > 1) break; + if (str[pos] > 0 || shellCountPrefixOnes((uint8_t)str[pos]) > 1) break; } - int rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + int32_t rc = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); assert(rc == *size); *width = taosWcharWidth(wc); } -void getNextCharSize(const char *str, int pos, int *size, int *width) { +void shellGetNextCharSize(const char *str, int32_t pos, int32_t *size, int32_t *width) { assert(pos >= 0); TdWchar wc; @@ -65,13 +165,13 @@ void getNextCharSize(const char *str, int pos, int *size, int *width) { *width = taosWcharWidth(wc); } -void insertChar(Command *cmd, char *c, int size) { +void shellInsertChar(SShellCmd *cmd, char *c, int32_t size) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); TdWchar wc; if (taosMbToWchar(&wc, c, size) < 0) return; - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); /* update the buffer */ memmove(cmd->command + cmd->cursorOffset + size, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); @@ -81,122 +181,122 @@ void insertChar(Command *cmd, char *c, int size) { cmd->cursorOffset += size; cmd->screenOffset += taosWcharWidth(wc); cmd->endOffset += taosWcharWidth(wc); - showOnScreen(cmd); + shellShowOnScreen(cmd); } -void backspaceChar(Command *cmd) { +void shellBackspaceChar(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset > 0) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); memmove(cmd->command + cmd->cursorOffset - size, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); cmd->commandSize -= size; cmd->cursorOffset -= size; cmd->screenOffset -= width; cmd->endOffset -= width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void clearLineBefore(Command *cmd) { +void shellClearLineBefore(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); memmove(cmd->command, cmd->command + cmd->cursorOffset, cmd->commandSize - cmd->cursorOffset); cmd->commandSize -= cmd->cursorOffset; cmd->cursorOffset = 0; cmd->screenOffset = 0; cmd->endOffset = cmd->commandSize; - showOnScreen(cmd); + shellShowOnScreen(cmd); } -void clearLineAfter(Command *cmd) { +void shellClearLineAfter(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->commandSize -= cmd->endOffset - cmd->cursorOffset; cmd->endOffset = cmd->cursorOffset; - showOnScreen(cmd); + shellShowOnScreen(cmd); } -void deleteChar(Command *cmd) { +void shellDeleteChar(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset < cmd->commandSize) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); memmove(cmd->command + cmd->cursorOffset, cmd->command + cmd->cursorOffset + size, cmd->commandSize - cmd->cursorOffset - size); cmd->commandSize -= size; cmd->endOffset -= width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void moveCursorLeft(Command *cmd) { +void shellMoveCursorLeft(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset > 0) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetPrevCharSize(cmd->command, cmd->cursorOffset, &size, &width); cmd->cursorOffset -= size; cmd->screenOffset -= width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void moveCursorRight(Command *cmd) { +void shellMoveCursorRight(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset < cmd->commandSize) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - int size = 0; - int width = 0; - getNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + int32_t size = 0; + int32_t width = 0; + shellGetNextCharSize(cmd->command, cmd->cursorOffset, &size, &width); cmd->cursorOffset += size; cmd->screenOffset += width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void positionCursorHome(Command *cmd) { +void shellPositionCursorHome(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset > 0) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->cursorOffset = 0; cmd->screenOffset = 0; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void positionCursorEnd(Command *cmd) { +void shellPositionCursorEnd(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); if (cmd->cursorOffset < cmd->commandSize) { - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); cmd->cursorOffset = cmd->commandSize; cmd->screenOffset = cmd->endOffset; - showOnScreen(cmd); + shellShowOnScreen(cmd); } } -void printChar(char c, int times) { - for (int i = 0; i < times; i++) { +void shellPrintChar(char c, int32_t times) { + for (int32_t i = 0; i < times; i++) { fprintf(stdout, "%c", c); } fflush(stdout); } -void positionCursor(int step, int direction) { +void shellPositionCursor(int32_t step, int32_t direction) { if (step > 0) { if (direction == LEFT) { fprintf(stdout, "\033[%dD", step); @@ -211,32 +311,32 @@ void positionCursor(int step, int direction) { } } -void updateBuffer(Command *cmd) { +void shellUpdateBuffer(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - if (regex_match(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); + if (shellRegexMatch(cmd->buffer, "(\\s+$)|(^$)", REG_EXTENDED)) strcat(cmd->command, " "); strcat(cmd->buffer, cmd->command); cmd->bufferSize += cmd->commandSize; - memset(cmd->command, 0, MAX_COMMAND_SIZE); + memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE); cmd->cursorOffset = 0; cmd->screenOffset = 0; cmd->commandSize = 0; cmd->endOffset = 0; - showOnScreen(cmd); + shellShowOnScreen(cmd); } -int isReadyGo(Command *cmd) { +int32_t shellIsReadyGo(SShellCmd *cmd) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - char *total = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - memset(cmd->command + cmd->commandSize, 0, MAX_COMMAND_SIZE - cmd->commandSize); + char *total = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + memset(cmd->command + cmd->commandSize, 0, SHELL_MAX_COMMAND_SIZE - cmd->commandSize); sprintf(total, "%s%s", cmd->buffer, cmd->command); char *reg_str = "(^.*;\\s*$)|(^\\s*$)|(^\\s*exit\\s*$)|(^\\s*q\\s*$)|(^\\s*quit\\s*$)|(^" "\\s*clear\\s*$)"; - if (regex_match(total, reg_str, REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(total, reg_str, REG_EXTENDED | REG_ICASE)) { taosMemoryFree(total); return 1; } @@ -245,28 +345,268 @@ int isReadyGo(Command *cmd) { return 0; } -void getMbSizeInfo(const char *str, int *size, int *width) { - TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), MAX_COMMAND_SIZE); +void shellGetMbSizeInfo(const char *str, int32_t *size, int32_t *width) { + TdWchar *wc = (TdWchar *)taosMemoryCalloc(sizeof(TdWchar), SHELL_MAX_COMMAND_SIZE); *size = strlen(str); - taosMbsToWchars(wc, str, MAX_COMMAND_SIZE); - *width = taosWcharsWidth(wc, MAX_COMMAND_SIZE); + taosMbsToWchars(wc, str, SHELL_MAX_COMMAND_SIZE); + *width = taosWcharsWidth(wc, SHELL_MAX_COMMAND_SIZE); taosMemoryFree(wc); } -void resetCommand(Command *cmd, const char s[]) { +void shellResetCommand(SShellCmd *cmd, const char s[]) { assert(cmd->cursorOffset <= cmd->commandSize && cmd->endOffset >= cmd->screenOffset); - clearScreen(cmd->endOffset + prompt_size, cmd->screenOffset + prompt_size); - memset(cmd->buffer, 0, MAX_COMMAND_SIZE); - memset(cmd->command, 0, MAX_COMMAND_SIZE); - strncpy(cmd->command, s, MAX_COMMAND_SIZE); - int size = 0; - int width = 0; - getMbSizeInfo(s, &size, &width); + shellClearScreen(cmd->endOffset + PSIZE, cmd->screenOffset + PSIZE); + memset(cmd->buffer, 0, SHELL_MAX_COMMAND_SIZE); + memset(cmd->command, 0, SHELL_MAX_COMMAND_SIZE); + strncpy(cmd->command, s, SHELL_MAX_COMMAND_SIZE); + int32_t size = 0; + int32_t width = 0; + shellGetMbSizeInfo(s, &size, &width); cmd->bufferSize = 0; cmd->commandSize = size; cmd->cursorOffset = size; cmd->screenOffset = width; cmd->endOffset = width; - showOnScreen(cmd); + shellShowOnScreen(cmd); } + +void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) { + struct winsize w; + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); + w.ws_col = 120; + w.ws_row = 30; + } + + int32_t cursor_x = cursor_pos / w.ws_col; + int32_t cursor_y = cursor_pos % w.ws_col; + int32_t command_x = ecmd_pos / w.ws_col; + shellPositionCursor(cursor_y, LEFT); + shellPositionCursor(command_x - cursor_x, DOWN); + fprintf(stdout, "\033[2K"); + for (int32_t i = 0; i < command_x; i++) { + shellPositionCursor(1, UP); + fprintf(stdout, "\033[2K"); + } + fflush(stdout); +} + +void shellShowOnScreen(SShellCmd *cmd) { + struct winsize w; + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + // fprintf(stderr, "No stream device\n"); + w.ws_col = 120; + w.ws_row = 30; + } + + TdWchar wc; + int32_t size = 0; + + // Print out the command. + char *total_string = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); + memset(total_string, '\0', SHELL_MAX_COMMAND_SIZE); + if (strcmp(cmd->buffer, "") == 0) { + sprintf(total_string, "%s%s", shell.info.promptHeader, cmd->command); + } else { + sprintf(total_string, "%s%s", shell.info.promptContinue, cmd->command); + } + + int32_t remain_column = w.ws_col; + for (char *str = total_string; size < cmd->commandSize + PSIZE;) { + int32_t ret = taosMbToWchar(&wc, str, MB_CUR_MAX); + if (ret < 0) break; + size += ret; + /* assert(size >= 0); */ + int32_t width = taosWcharWidth(wc); + if (remain_column > width) { + printf("%lc", wc); + remain_column -= width; + } else { + if (remain_column == width) { + printf("%lc\n\r", wc); + remain_column = w.ws_col; + } else { + printf("\n\r%lc", wc); + remain_column = w.ws_col - width; + } + } + + str = total_string + size; + } + + taosMemoryFree(total_string); + + // Position the cursor + int32_t cursor_pos = cmd->screenOffset + PSIZE; + int32_t ecmd_pos = cmd->endOffset + PSIZE; + + int32_t cursor_x = cursor_pos / w.ws_col; + int32_t cursor_y = cursor_pos % w.ws_col; + // int32_t cursor_y = cursor % w.ws_col; + int32_t command_x = ecmd_pos / w.ws_col; + int32_t command_y = ecmd_pos % w.ws_col; + // int32_t command_y = (command.size() + PSIZE) % w.ws_col; + shellPositionCursor(command_y, LEFT); + shellPositionCursor(command_x, UP); + shellPositionCursor(cursor_x, DOWN); + shellPositionCursor(cursor_y, RIGHT); + fflush(stdout); +} + +int32_t shellReadCommand(char *command) { + SShellHistory *pHistory = &shell.history; + SShellCmd cmd = {0}; + uint32_t hist_counter = pHistory->hend; + char utf8_array[10] = "\0"; + + cmd.buffer = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + cmd.command = (char *)taosMemoryCalloc(1, SHELL_MAX_COMMAND_SIZE); + shellShowOnScreen(&cmd); + + // Read input. + char c; + while (1) { + c = (char)getchar(); // getchar() return an 'int32_t' value + + if (c == EOF) { + return c; + } + + if (c < 0) { // For UTF-8 + int32_t count = shellCountPrefixOnes(c); + utf8_array[0] = c; + for (int32_t k = 1; k < count; k++) { + c = (char)getchar(); + utf8_array[k] = c; + } + shellInsertChar(&cmd, utf8_array, count); + } else if (c < '\033') { + // Ctrl keys. TODO: Implement ctrl combinations + switch (c) { + case 1: // ctrl A + shellPositionCursorHome(&cmd); + break; + case 3: + printf("\n"); + shellResetCommand(&cmd, ""); + kill(0, SIGINT); + break; + case 4: // EOF or Ctrl+D + printf("\n"); + return -1; + case 5: // ctrl E + shellPositionCursorEnd(&cmd); + break; + case 8: + shellBackspaceChar(&cmd); + break; + case '\n': + case '\r': + printf("\n"); + if (shellIsReadyGo(&cmd)) { + sprintf(command, "%s%s", cmd.buffer, cmd.command); + taosMemoryFreeClear(cmd.buffer); + taosMemoryFreeClear(cmd.command); + return 0; + } else { + shellUpdateBuffer(&cmd); + } + break; + case 11: // Ctrl + K; + shellClearLineAfter(&cmd); + break; + case 12: // Ctrl + L; + system("clear"); + shellShowOnScreen(&cmd); + break; + case 21: // Ctrl + U; + shellClearLineBefore(&cmd); + break; + } + } else if (c == '\033') { + c = (char)getchar(); + switch (c) { + case '[': + c = (char)getchar(); + switch (c) { + case 'A': // Up arrow + if (hist_counter != pHistory->hstart) { + hist_counter = (hist_counter + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE; + shellResetCommand(&cmd, (pHistory->hist[hist_counter] == NULL) ? "" : pHistory->hist[hist_counter]); + } + break; + case 'B': // Down arrow + if (hist_counter != pHistory->hend) { + int32_t next_hist = (hist_counter + 1) % SHELL_MAX_HISTORY_SIZE; + + if (next_hist != pHistory->hend) { + shellResetCommand(&cmd, (pHistory->hist[next_hist] == NULL) ? "" : pHistory->hist[next_hist]); + } else { + shellResetCommand(&cmd, ""); + } + hist_counter = next_hist; + } + break; + case 'C': // Right arrow + shellMoveCursorRight(&cmd); + break; + case 'D': // Left arrow + shellMoveCursorLeft(&cmd); + break; + case '1': + if ((c = (char)getchar()) == '~') { + // Home key + shellPositionCursorHome(&cmd); + } + break; + case '2': + if ((c = (char)getchar()) == '~') { + // Insert key + } + break; + case '3': + if ((c = (char)getchar()) == '~') { + // Delete key + shellDeleteChar(&cmd); + } + break; + case '4': + if ((c = (char)getchar()) == '~') { + // End key + shellPositionCursorEnd(&cmd); + } + break; + case '5': + if ((c = (char)getchar()) == '~') { + // Page up key + } + break; + case '6': + if ((c = (char)getchar()) == '~') { + // Page down key + } + break; + case 72: + // Home key + shellPositionCursorHome(&cmd); + break; + case 70: + // End key + shellPositionCursorEnd(&cmd); + break; + } + break; + } + } else if (c == 0x7f) { + // press delete key + shellBackspaceChar(&cmd); + } else { + shellInsertChar(&cmd, &c, 1); + } + } + + return 0; +} + +#endif \ No newline at end of file diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 12206ce290..1f036ab25b 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -17,134 +17,35 @@ #define _GNU_SOURCE #define _XOPEN_SOURCE #define _DEFAULT_SOURCE +#include "shellInt.h" -#include "os.h" -#include "shell.h" -#include "shellCommand.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tconfig.h" -#include "tglobal.h" -#include "ttypes.h" -#include "tutil.h" +static bool shellIsEmptyCommand(const char *cmd); +static int32_t shellRunSingleCommand(char *command); +static int32_t shellRunCommand(char *command); +static void shellRunSingleCommandImp(char *command); +static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); +static void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, + int32_t precision); +static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); +static void shellPrintNChar(const char *str, int32_t length, int32_t width); +static void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); +static int32_t shellVerticalPrintResult(TAOS_RES *tres); +static int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); +static void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); +static int32_t shellHorizontalPrintResult(TAOS_RES *tres); +static int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical); +static void shellReadHistory(); +static void shellWriteHistory(); +static void shellPrintError(TAOS_RES *tres, int64_t st); +static bool shellIsCommentLine(char *line); +static void shellSourceFile(const char *file); +static void shellGetGrantInfo(); +static void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context); +static void shellCleanup(void *arg); +static void *shellCancelHandler(void *arg); +static void *shellThreadLoop(void *arg); -#include - -/**************** Global variables ****************/ -#ifdef _TD_POWER_ -char CLIENT_VERSION[] = - "Welcome to the PowerDB shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by PowerDB, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "power> "; - -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 7; -#elif (_TD_TQ_ == true) -char CLIENT_VERSION[] = - "Welcome to the TQ shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TQ, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "tq> "; - -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 4; -#else -char CLIENT_VERSION[] = - "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char PROMPT_HEADER[] = "taos> "; - -char CONTINUE_PROMPT[] = " -> "; -int prompt_size = 6; -#endif - -int64_t result = 0; -SShellHistory history; - -#define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 -extern int32_t tsMaxBinaryDisplayWidth; -extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); - -/* - * FUNCTION: Initialize the shell. - */ -TAOS *shellInit(SShellArguments *_args) { - printf("\n"); - if (!_args->is_use_passwd) { -#ifdef TD_WINDOWS - strcpy(tsOsName, "Windows"); -#elif defined(TD_DARWIN) - strcpy(tsOsName, "Darwin"); -#endif - printf(CLIENT_VERSION, tsOsName, taos_get_client_info()); - } - - fflush(stdout); - - // set options before initializing - if (_args->timezone != NULL) { - taos_options(TSDB_OPTION_TIMEZONE, _args->timezone); - } - - if (!_args->is_use_passwd) { - _args->password = TSDB_DEFAULT_PASS; - } - - if (_args->user == NULL) { - _args->user = TSDB_DEFAULT_USER; - } - - // Connect to the database. - TAOS *con = NULL; - if (_args->auth == NULL) { - con = taos_connect(_args->host, _args->user, _args->password, _args->database, _args->port); - } else { - con = taos_connect_auth(_args->host, _args->user, _args->auth, _args->database, _args->port); - } - - if (con == NULL) { - fflush(stdout); - return con; - } - - /* Read history TODO : release resources here*/ - read_history(); - - // Check if it is temperory run - if (_args->commands != NULL || _args->file[0] != 0) { - if (_args->commands != NULL) { - printf("%s%s\n", PROMPT_HEADER, _args->commands); - shellRunCommand(con, _args->commands); - } - - if (_args->file[0] != 0) { - source_file(con, _args->file); - } - - taos_close(con); - write_history(); - exit(EXIT_SUCCESS); - } - -#if 0 -#ifndef WINDOWS - if (_args->dir[0] != 0) { - source_dir(con, _args); - taos_close(con); - exit(EXIT_SUCCESS); - } - - if (_args->check != 0) { - shellCheck(con, _args); - taos_close(con); - exit(EXIT_SUCCESS); - } -#endif -#endif - - return con; -} - -static bool isEmptyCommand(const char *cmd) { +bool shellIsEmptyCommand(const char *cmd) { for (char c = *cmd++; c != 0; c = *cmd++) { if (c != ' ' && c != '\t' && c != ';') { return false; @@ -153,73 +54,67 @@ static bool isEmptyCommand(const char *cmd) { return true; } -static int32_t shellRunSingleCommand(TAOS *con, char *command) { - /* If command is empty just return */ - if (isEmptyCommand(command)) { +int32_t shellRunSingleCommand(char *command) { + if (shellIsEmptyCommand(command)) { return 0; } - // Analyse the command. - if (regex_match(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { - taos_close(con); - write_history(); -#ifdef WINDOWS - exit(EXIT_SUCCESS); -#endif + if (shellRegexMatch(command, "^[ \t]*(quit|q|exit)[ \t;]*$", REG_EXTENDED | REG_ICASE)) { + shellWriteHistory(); return -1; } - if (regex_match(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) { - // If clear the screen. + if (shellRegexMatch(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) { system("clear"); return 0; } - if (regex_match(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", - REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(command, "^[\t ]*set[ \t]+max_binary_display_width[ \t]+(default|[1-9][0-9]*)[ \t;]*$", + REG_EXTENDED | REG_ICASE)) { strtok(command, " \t"); strtok(NULL, " \t"); char *p = strtok(NULL, " \t"); - if (strcasecmp(p, "default") == 0) { - tsMaxBinaryDisplayWidth = DEFAULT_MAX_BINARY_DISPLAY_WIDTH; + if (strncasecmp(p, "default", 7) == 0) { + shell.args.displayWidth = SHELL_DEFAULT_MAX_BINARY_DISPLAY_WIDTH; } else { - tsMaxBinaryDisplayWidth = atoi(p); + int32_t displayWidth = atoi(p); + displayWidth = TRANGE(displayWidth, 1, 10 * 1024); + shell.args.displayWidth = displayWidth; } return 0; } - if (regex_match(command, "^[ \t]*source[\t ]+[^ ]+[ \t;]*$", REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(command, "^[ \t]*source[\t ]+[^ ]+[ \t;]*$", REG_EXTENDED | REG_ICASE)) { /* If source file. */ char *c_ptr = strtok(command, " ;"); assert(c_ptr != NULL); c_ptr = strtok(NULL, " ;"); assert(c_ptr != NULL); - source_file(con, c_ptr); + shellSourceFile(c_ptr); return 0; } - shellRunCommandOnServer(con, command); + shellRunSingleCommandImp(command); return 0; } -int32_t shellRunCommand(TAOS *con, char *command) { - /* If command is empty just return */ - if (isEmptyCommand(command)) { +int32_t shellRunCommand(char *command) { + if (shellIsEmptyCommand(command)) { return 0; } - /* Update the history vector. */ - if (history.hstart == history.hend || - history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || - strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { - if (history.hist[history.hend] != NULL) { - taosMemoryFreeClear(history.hist[history.hend]); + SShellHistory *pHistory = &shell.history; + if (pHistory->hstart == pHistory->hend || + pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE] == NULL || + strcmp(command, pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE]) != 0) { + if (pHistory->hist[pHistory->hend] != NULL) { + taosMemoryFreeClear(pHistory->hist[pHistory->hend]); } - history.hist[history.hend] = strdup(command); + pHistory->hist[pHistory->hend] = strdup(command); - history.hend = (history.hend + 1) % MAX_HISTORY_SIZE; - if (history.hend == history.hstart) { - history.hstart = (history.hstart + 1) % MAX_HISTORY_SIZE; + pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; + if (pHistory->hend == pHistory->hstart) { + pHistory->hstart = (pHistory->hstart + 1) % SHELL_MAX_HISTORY_SIZE; } } @@ -271,7 +166,7 @@ int32_t shellRunCommand(TAOS *con, char *command) { if (c == ';' && quote == 0) { c = *p; *p = 0; - if (shellRunSingleCommand(con, cmd) < 0) { + if (shellRunSingleCommand(cmd) < 0) { return -1; } *p = c; @@ -280,26 +175,15 @@ int32_t shellRunCommand(TAOS *con, char *command) { } *p = 0; - return shellRunSingleCommand(con, cmd); + return shellRunSingleCommand(cmd); } -void freeResultWithRid(int64_t rid) { -#if 0 - SSqlObj* pSql = taosAcquireRef(tscObjRef, rid); - if(pSql){ - taos_free_result(pSql); - taosReleaseRef(tscObjRef, rid); - } -#endif -} - -void shellRunCommandOnServer(TAOS *con, char command[]) { - int64_t st, et; - wordexp_t full_path; - char *sptr = NULL; - char *cptr = NULL; - char *fname = NULL; - bool printMode = false; +void shellRunSingleCommandImp(char *command) { + int64_t st, et; + char *sptr = NULL; + char *cptr = NULL; + char *fname = NULL; + bool printMode = false; if ((sptr = strstr(command, ">>")) != NULL) { cptr = strstr(command, ";"); @@ -307,12 +191,8 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { *cptr = '\0'; } - if (wordexp(sptr + 2, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: invalid filename: %s\n", sptr + 2); - return; - } + fname = sptr + 2; *sptr = '\0'; - fname = full_path.we_wordv[0]; } if ((sptr = strstr(command, "\\G")) != NULL) { @@ -327,20 +207,19 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); - TAOS_RES *pSql = taos_query(con, command); + TAOS_RES *pSql = taos_query(shell.conn, command); if (taos_errno(pSql)) { - taos_error(pSql, st); + shellPrintError(pSql, st); return; } - int64_t oresult = atomic_load_64(&result); + int64_t oresult = atomic_load_64(&shell.result); - if (regex_match(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { + if (shellRegexMatch(command, "^\\s*use\\s+[a-zA-Z0-9_]+\\s*;\\s*$", REG_EXTENDED | REG_ICASE)) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - atomic_store_64(&result, 0); - freeResultWithRid(oresult); + atomic_store_64(&shell.result, 0); taos_free_result(pSql); return; @@ -348,12 +227,11 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { TAOS_FIELD *pFields = taos_fetch_fields(pSql); if (pFields != NULL) { // select and show kinds of commands - int error_no = 0; + int32_t error_no = 0; - int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); + int32_t numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - atomic_store_64(&result, 0); - freeResultWithRid(oresult); + atomic_store_64(&shell.result, 0); return; } @@ -365,7 +243,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { } taos_free_result(pSql); } else { - int num_rows_affacted = taos_affected_rows(pSql); + int32_t num_rows_affacted = taos_affected_rows(pSql); taos_free_result(pSql); et = taosGetTimestampUs(); printf("Query OK, %d of %d row(s) in database (%.6fs)\n", num_rows_affacted, num_rows_affacted, (et - st) / 1E6); @@ -373,45 +251,11 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { printf("\n"); - if (fname != NULL) { - wordfree(&full_path); - } - - atomic_store_64(&result, 0); - freeResultWithRid(oresult); + atomic_store_64(&shell.result, 0); } -/* Function to do regular expression check */ -int regex_match(const char *s, const char *reg, int cflags) { - regex_t regex; - char msgbuf[100] = {0}; - - /* Compile regular expression */ - if (regcomp(®ex, reg, cflags) != 0) { - fprintf(stderr, "Fail to compile regex"); - exitShell(); - } - - /* Execute regular expression */ - int reti = regexec(®ex, s, 0, NULL, 0); - if (!reti) { - regfree(®ex); - return 1; - } else if (reti == REG_NOMATCH) { - regfree(®ex); - return 0; - } else { - regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); - fprintf(stderr, "Regex match failed: %s\n", msgbuf); - regfree(®ex); - exitShell(); - } - - return 0; -} - -static char *formatTimestamp(char *buf, int64_t val, int precision) { - if (args.is_raw_time) { +char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) { + if (shell.args.is_raw_time) { sprintf(buf, "%" PRId64, val); return buf; } @@ -429,13 +273,14 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { ms = val % 1000; } - /* comment out as it make testcases like select_with_tags.sim fail. + /* + comment out as it make testcases like select_with_tags.sim fail. but in windows, this may cause the call to localtime crash if tt < 0, need to find a better solution. if (tt < 0) { tt = 0; } - */ + */ #ifdef WINDOWS if (tt < 0) tt = 0; @@ -465,7 +310,7 @@ static char *formatTimestamp(char *buf, int64_t val, int precision) { return buf; } -static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int precision) { +void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, int32_t length, int32_t precision) { if (val == NULL) { taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); return; @@ -501,7 +346,7 @@ static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, taosFprintfFile(pFile, "\'%s\'", buf); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t *)val, precision); + shellFormatTimestamp(buf, *(int64_t *)val, precision); taosFprintfFile(pFile, "'%s'", buf); break; default: @@ -509,35 +354,28 @@ static void dumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, } } -static int dumpResultToFile(const char *fname, TAOS_RES *tres) { +int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres) { + char fullname[PATH_MAX] = {0}; + if (taosExpandDir(fname, fullname, PATH_MAX) != 0) { + tstrncpy(fullname, fname, PATH_MAX); + } + TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - wordexp_t full_path; - - if (wordexp((char *)fname, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: invalid file name: %s\n", fname); - return -1; - } - - // FILE *fp = fopen(full_path.we_wordv[0], "w"); - TdFilePtr pFile = - taosOpenFile(full_path.we_wordv[0], TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file: %s\n", full_path.we_wordv[0]); - wordfree(&full_path); + fprintf(stderr, "failed to open file: %s\n", fullname); return -1; } - wordfree(&full_path); - - int num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int32_t num_fields = taos_num_fields(tres); + int32_t precision = taos_result_precision(tres); - for (int col = 0; col < num_fields; col++) { + for (int32_t col = 0; col < num_fields; col++) { if (col > 0) { taosFprintfFile(pFile, ","); } @@ -545,14 +383,14 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { } taosFprintfFile(pFile, "\n"); - int numOfRows = 0; + int32_t numOfRows = 0; do { int32_t *length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { + for (int32_t i = 0; i < num_fields; i++) { if (i > 0) { taosFprintfFile(pFile, "\n"); } - dumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); + shellDumpFieldToFile(pFile, (const char *)row[i], fields + i, length[i], precision); } taosFprintfFile(pFile, "\n"); @@ -560,19 +398,19 @@ static int dumpResultToFile(const char *fname, TAOS_RES *tres) { row = taos_fetch_row(tres); } while (row != NULL); - result = 0; + atomic_store_64(&shell.result, 0); taosCloseFile(&pFile); return numOfRows; } -static void shellPrintNChar(const char *str, int length, int width) { +void shellPrintNChar(const char *str, int32_t length, int32_t width) { TdWchar tail[3]; - int pos = 0, cols = 0, totalCols = 0, tailLen = 0; + int32_t pos = 0, cols = 0, totalCols = 0, tailLen = 0; while (pos < length) { TdWchar wc; - int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); + int32_t bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); if (bytes == 0) { break; } @@ -582,9 +420,9 @@ static void shellPrintNChar(const char *str, int length, int width) { } #ifdef WINDOWS - int w = bytes; + int32_t w = bytes; #else - int w = taosWcharWidth(wc); + int32_t w = taosWcharWidth(wc); #endif if (w <= 0) { continue; @@ -610,7 +448,7 @@ static void shellPrintNChar(const char *str, int length, int width) { if (totalCols > width) { // width could be 1 or 2, so printf("...") cannot be used - for (int i = 0; i < 3; i++) { + for (int32_t i = 0; i < 3; i++) { if (cols >= width) { break; } @@ -618,7 +456,7 @@ static void shellPrintNChar(const char *str, int length, int width) { ++cols; } } else { - for (int i = 0; i < tailLen; i++) { + for (int32_t i = 0; i < tailLen; i++) { printf("%lc", tail[i]); } cols = totalCols; @@ -629,9 +467,9 @@ static void shellPrintNChar(const char *str, int length, int width) { } } -static void printField(const char *val, TAOS_FIELD *field, int width, int32_t length, int precision) { +void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision) { if (val == NULL) { - int w = width; + int32_t w = width; if (field->type < TSDB_DATA_TYPE_TINYINT || field->type > TSDB_DATA_TYPE_DOUBLE) { w = 0; } @@ -682,7 +520,7 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: - formatTimestamp(buf, *(int64_t *)val, precision); + shellFormatTimestamp(buf, *(int64_t *)val, precision); printf("%s", buf); break; default: @@ -690,30 +528,19 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le } } -bool isSelectQuery(TAOS_RES *tres) { -#if 0 - char *sql = tscGetSqlStr(tres); - - if (regex_match(sql, "^[\t ]*select[ \t]*", REG_EXTENDED | REG_ICASE)) { - return true; - } -#endif - return false; -} - -static int verticalPrintResult(TAOS_RES *tres) { +int32_t shellVerticalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int32_t num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int32_t precision = taos_result_precision(tres); - int maxColNameLen = 0; - for (int col = 0; col < num_fields; col++) { - int len = (int)strlen(fields[col].name); + int32_t maxColNameLen = 0; + for (int32_t col = 0; col < num_fields; col++) { + int32_t len = (int32_t)strlen(fields[col].name); if (len > maxColNameLen) { maxColNameLen = len; } @@ -721,25 +548,25 @@ static int verticalPrintResult(TAOS_RES *tres) { uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /*&& !tscIsQueryWithLimit(tres)*/) { - resShowMaxNum = DEFAULT_RES_SHOW_NUM; + if (shell.args.commands == NULL && shell.args.file[0] == 0) { + resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } - int numOfRows = 0; - int showMore = 1; + int32_t numOfRows = 0; + int32_t showMore = 1; do { if (numOfRows < resShowMaxNum) { printf("*************************** %d.row ***************************\n", numOfRows + 1); int32_t *length = taos_fetch_lengths(tres); - for (int i = 0; i < num_fields; i++) { + for (int32_t i = 0; i < num_fields; i++) { TAOS_FIELD *field = fields + i; - int padding = (int)(maxColNameLen - strlen(field->name)); + int32_t padding = (int32_t)(maxColNameLen - strlen(field->name)); printf("%*.s%s: ", padding, " ", field->name); - printField((const char *)row[i], field, 0, length[i], precision); + shellPrintField((const char *)row[i], field, 0, length[i], precision); putchar('\n'); } } else if (showMore) { @@ -755,8 +582,8 @@ static int verticalPrintResult(TAOS_RES *tres) { return numOfRows; } -static int calcColWidth(TAOS_FIELD *field, int precision) { - int width = (int)strlen(field->name); +int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) { + int32_t width = (int32_t)strlen(field->name); switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -785,23 +612,23 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { return TMAX(25, width); case TSDB_DATA_TYPE_BINARY: - if (field->bytes > tsMaxBinaryDisplayWidth) { - return TMAX(tsMaxBinaryDisplayWidth, width); + if (field->bytes > shell.args.displayWidth) { + return TMAX(shell.args.displayWidth, width); } else { return TMAX(field->bytes, width); } case TSDB_DATA_TYPE_NCHAR: { int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; - if (bytes > tsMaxBinaryDisplayWidth) { - return TMAX(tsMaxBinaryDisplayWidth, width); + if (bytes > shell.args.displayWidth) { + return TMAX(shell.args.displayWidth, width); } else { return TMAX(bytes, width); } } case TSDB_DATA_TYPE_TIMESTAMP: - if (args.is_raw_time) { + if (shell.args.is_raw_time) { return TMAX(14, width); } if (precision == TSDB_TIME_PRECISION_NANO) { @@ -819,55 +646,55 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { return 0; } -static void printHeader(TAOS_FIELD *fields, int *width, int num_fields) { - int rowWidth = 0; - for (int col = 0; col < num_fields; col++) { +void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) { + int32_t rowWidth = 0; + for (int32_t col = 0; col < num_fields; col++) { TAOS_FIELD *field = fields + col; - int padding = (int)(width[col] - strlen(field->name)); - int left = padding / 2; + int32_t padding = (int32_t)(width[col] - strlen(field->name)); + int32_t left = padding / 2; printf(" %*.s%s%*.s |", left, " ", field->name, padding - left, " "); rowWidth += width[col] + 3; } putchar('\n'); - for (int i = 0; i < rowWidth; i++) { + for (int32_t i = 0; i < rowWidth; i++) { putchar('='); } putchar('\n'); } -static int horizontalPrintResult(TAOS_RES *tres) { +int32_t shellHorizontalPrintResult(TAOS_RES *tres) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { return 0; } - int num_fields = taos_num_fields(tres); + int32_t num_fields = taos_num_fields(tres); TAOS_FIELD *fields = taos_fetch_fields(tres); - int precision = taos_result_precision(tres); + int32_t precision = taos_result_precision(tres); - int width[TSDB_MAX_COLUMNS]; - for (int col = 0; col < num_fields; col++) { - width[col] = calcColWidth(fields + col, precision); + int32_t width[TSDB_MAX_COLUMNS]; + for (int32_t col = 0; col < num_fields; col++) { + width[col] = shellCalcColWidth(fields + col, precision); } - printHeader(fields, width, num_fields); + shellPrintHeader(fields, width, num_fields); uint64_t resShowMaxNum = UINT64_MAX; - if (args.commands == NULL && args.file[0] == 0 && isSelectQuery(tres) /* && !tscIsQueryWithLimit(tres)*/) { - resShowMaxNum = DEFAULT_RES_SHOW_NUM; + if (shell.args.commands == NULL && shell.args.file[0] == 0) { + resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } - int numOfRows = 0; - int showMore = 1; + int32_t numOfRows = 0; + int32_t showMore = 1; do { int32_t *length = taos_fetch_lengths(tres); if (numOfRows < resShowMaxNum) { - for (int i = 0; i < num_fields; i++) { + for (int32_t i = 0; i < num_fields; i++) { putchar(' '); - printField((const char *)row[i], fields + i, width[i], length[i], precision); + shellPrintField((const char *)row[i], fields + i, width[i], length[i], precision); putchar(' '); putchar('|'); } @@ -885,50 +712,35 @@ static int horizontalPrintResult(TAOS_RES *tres) { return numOfRows; } -int shellDumpResult(TAOS_RES *tres, char *fname, int *error_no, bool vertical) { - int numOfRows = 0; +int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical) { + int32_t numOfRows = 0; if (fname != NULL) { - numOfRows = dumpResultToFile(fname, tres); + numOfRows = shellDumpResultToFile(fname, tres); } else if (vertical) { - numOfRows = verticalPrintResult(tres); + numOfRows = shellVerticalPrintResult(tres); } else { - numOfRows = horizontalPrintResult(tres); + numOfRows = shellHorizontalPrintResult(tres); } *error_no = taos_errno(tres); return numOfRows; } -void read_history() { - // Initialize history - memset(history.hist, 0, sizeof(char *) * MAX_HISTORY_SIZE); - history.hstart = 0; - history.hend = 0; - char *line = NULL; - int read_size = 0; - - char f_history[TSDB_FILENAME_LEN]; - get_history_path(f_history); - - // FILE *f = fopen(f_history, "r"); - TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) { -#ifndef WINDOWS - if (errno != ENOENT) { - fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno)); - } -#endif - return; - } +void shellReadHistory() { + SShellHistory *pHistory = &shell.history; + TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); + if (pFile == NULL) return; + char *line = NULL; + int32_t read_size = 0; while ((read_size = taosGetLineFile(pFile, &line)) != -1) { line[read_size - 1] = '\0'; - history.hist[history.hend] = strdup(line); + pHistory->hist[pHistory->hend] = strdup(line); - history.hend = (history.hend + 1) % MAX_HISTORY_SIZE; + pHistory->hend = (pHistory->hend + 1) % SHELL_MAX_HISTORY_SIZE; - if (history.hend == history.hstart) { - history.hstart = (history.hstart + 1) % MAX_HISTORY_SIZE; + if (pHistory->hend == pHistory->hstart) { + pHistory->hstart = (pHistory->hstart + 1) % SHELL_MAX_HISTORY_SIZE; } } @@ -936,72 +748,48 @@ void read_history() { taosCloseFile(&pFile); } -void write_history() { - char f_history[TSDB_FILENAME_LEN]; - get_history_path(f_history); +void shellWriteHistory() { + SShellHistory *pHistory = &shell.history; + TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); + if (pFile == NULL) return; - // FILE *f = fopen(f_history, "w"); - TdFilePtr pFile = taosOpenFile(f_history, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); - if (pFile == NULL) { -#ifndef WINDOWS - fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno)); -#endif - return; - } - - for (int i = history.hstart; i != history.hend;) { - if (history.hist[i] != NULL) { - taosFprintfFile(pFile, "%s\n", history.hist[i]); - taosMemoryFreeClear(history.hist[i]); + for (int32_t i = pHistory->hstart; i != pHistory->hend;) { + if (pHistory->hist[i] != NULL) { + taosFprintfFile(pFile, "%s\n", pHistory->hist[i]); + taosMemoryFreeClear(pHistory->hist[i]); } - i = (i + 1) % MAX_HISTORY_SIZE; + i = (i + 1) % SHELL_MAX_HISTORY_SIZE; } + taosFsyncFile(pFile); taosCloseFile(&pFile); } -void taos_error(TAOS_RES *tres, int64_t st) { +void shellPrintError(TAOS_RES *tres, int64_t st) { int64_t et = taosGetTimestampUs(); - atomic_store_ptr(&result, 0); + atomic_store_ptr(&shell.result, 0); fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } -int isCommentLine(char *line) { - if (line == NULL) return 1; - - return regex_match(line, "^\\s*#.*", REG_EXTENDED); +bool shellIsCommentLine(char *line) { + if (line == NULL) return true; + return shellRegexMatch(line, "^\\s*#.*", REG_EXTENDED); } -void source_file(TAOS *con, char *fptr) { - wordexp_t full_path; - int read_len = 0; - char *cmd = taosMemoryCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); - size_t cmd_len = 0; - char *line = NULL; +void shellSourceFile(const char *file) { + int32_t read_len = 0; + char *cmd = taosMemoryCalloc(1, TSDB_MAX_ALLOWED_SQL_LEN + 1); + size_t cmd_len = 0; + char *line = NULL; + char fullname[PATH_MAX] = {0}; - if (wordexp(fptr, &full_path, 0) != 0) { - fprintf(stderr, "ERROR: illegal file name\n"); - taosMemoryFree(cmd); - return; + if (taosExpandDir(file, fullname, PATH_MAX) != 0) { + tstrncpy(fullname, file, PATH_MAX); } - char *fname = full_path.we_wordv[0]; - - /* - if (access(fname, F_OK) != 0) { - fprintf(stderr, "ERROR: file %s is not exist\n", fptr); - - wordfree(&full_path); - taosMemoryFree(cmd); - return; - } - */ - - // FILE *f = fopen(fname, "r"); - TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ | TD_FILE_STREAM); + TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { - fprintf(stderr, "ERROR: failed to open file %s\n", fname); - wordfree(&full_path); + fprintf(stderr, "failed to open file %s\n", fullname); taosMemoryFree(cmd); return; } @@ -1010,7 +798,7 @@ void source_file(TAOS *con, char *fptr) { if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; - if (read_len == 0 || isCommentLine(line)) { // line starts with # + if (read_len == 0 || shellIsCommentLine(line)) { // line starts with # continue; } @@ -1022,36 +810,38 @@ void source_file(TAOS *con, char *fptr) { } memcpy(cmd + cmd_len, line, read_len); - printf("%s%s\n", PROMPT_HEADER, cmd); - shellRunCommand(con, cmd); + printf("%s%s\n", shell.info.promptHeader, cmd); + shellRunCommand(cmd); memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } taosMemoryFree(cmd); if (line != NULL) taosMemoryFree(line); - wordfree(&full_path); taosCloseFile(&pFile); } -void shellGetGrantInfo(void *con) { - return; -#if 0 +void shellGetGrantInfo() { + char sinfo[1024] = {0}; + tstrncpy(sinfo, taos_get_server_info(shell.conn), sizeof(sinfo)); + strtok(sinfo, "\n"); + char sql[] = "show grants"; - TAOS_RES* tres = taos_query(con, sql); + TAOS_RES *tres = taos_query(shell.conn, sql); - int code = taos_errno(tres); + int32_t code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { - if (code == TSDB_CODE_COM_OPS_NOT_SUPPORT) { - fprintf(stdout, "Server is Community Edition, version is %s\n\n", taos_get_server_info(con)); + if (code == TSDB_CODE_OPS_NOT_SUPPORT) { + fprintf(stdout, "Server is Community Edition, %s\n\n", sinfo); } else { - fprintf(stderr, "Failed to check Server Edition, Reason:%d:%s\n\n", taos_errno(con), taos_errstr(con)); + fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\n\n", taos_errno(shell.conn), + taos_errstr(shell.conn)); } return; } - int num_fields = taos_field_count(tres); + int32_t num_fields = taos_field_count(tres); if (num_fields == 0) { fprintf(stderr, "\nInvalid grant information.\n"); exit(0); @@ -1062,7 +852,7 @@ void shellGetGrantInfo(void *con) { } TAOS_FIELD *fields = taos_fetch_fields(tres); - TAOS_ROW row = taos_fetch_row(tres); + TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { fprintf(stderr, "\nFailed to get grant information from server. Abort.\n"); exit(0); @@ -1077,15 +867,128 @@ void shellGetGrantInfo(void *con) { memcpy(expired, row[2], fields[2].bytes); if (strcmp(expiretime, "unlimited") == 0) { - fprintf(stdout, "Server is Enterprise %s Edition, version is %s and will never expire.\n", serverVersion, taos_get_server_info(con)); + fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\n", serverVersion, sinfo); } else { - fprintf(stdout, "Server is Enterprise %s Edition, version is %s and will expire at %s.\n", serverVersion, taos_get_server_info(con), expiretime); + fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\n", serverVersion, sinfo, expiretime); } - result = NULL; + atomic_store_64(&shell.result, 0); taos_free_result(tres); } fprintf(stdout, "\n"); -#endif +} + +void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&shell.cancelSem); } + +void shellSigintHandler(int32_t signum, void *sigInfo, void *context) { + // do nothing +} + +void shellCleanup(void *arg) { taosResetTerminalMode(); } + +void *shellCancelHandler(void *arg) { + setThreadName("shellCancelHandler"); + while (1) { + if (tsem_wait(&shell.cancelSem) != 0) { + taosMsleep(10); + continue; + } + + taosResetTerminalMode(); + printf("\nReceive SIGTERM or other signal, quit shell.\n"); + shellWriteHistory(); + shellExit(); + } + + return NULL; +} + +void *shellThreadLoop(void *arg) { + setThreadName("shellThreadLoop"); + taosGetOldTerminalMode(); + taosThreadCleanupPush(shellCleanup, NULL); + + char *command = taosMemoryMalloc(SHELL_MAX_COMMAND_SIZE); + if (command == NULL) { + printf("failed to malloc command\n"); + return NULL; + } + + do { + memset(command, 0, SHELL_MAX_COMMAND_SIZE); + taosSetTerminalMode(); + + if (shellReadCommand(command) != 0) { + break; + } + + taosResetTerminalMode(); + } while (shellRunCommand(command) == 0); + + taosMemoryFreeClear(command); + shellWriteHistory(); + shellExit(); + + taosThreadCleanupPop(1); + return NULL; +} + +int32_t shellExecute() { + printf(shell.info.clientVersion, shell.info.osname, taos_get_client_info()); + fflush(stdout); + + SShellArgs *pArgs = &shell.args; + if (shell.args.auth == NULL) { + shell.conn = taos_connect(pArgs->host, pArgs->user, pArgs->password, pArgs->database, pArgs->port); + } else { + shell.conn = taos_connect_auth(pArgs->host, pArgs->user, pArgs->auth, pArgs->database, pArgs->port); + } + + if (shell.conn == NULL) { + fflush(stdout); + return -1; + } + + shellReadHistory(); + + if (pArgs->commands != NULL || pArgs->file[0] != 0) { + if (pArgs->commands != NULL) { + printf("%s%s\n", shell.info.promptHeader, pArgs->commands); + char *cmd = strdup(pArgs->commands); + shellRunCommand(cmd); + taosMemoryFree(cmd); + } + + if (pArgs->file[0] != 0) { + shellSourceFile(pArgs->file); + } + + taos_close(shell.conn); + shellWriteHistory(); + return 0; + } + + if (tsem_init(&shell.cancelSem, 0, 0) != 0) { + printf("failed to create cancel semphore\n"); + return -1; + } + + TdThread spid = {0}; + taosThreadCreate(&spid, NULL, shellCancelHandler, NULL); + + taosSetSignal(SIGTERM, shellQueryInterruptHandler); + taosSetSignal(SIGHUP, shellQueryInterruptHandler); + taosSetSignal(SIGABRT, shellQueryInterruptHandler); + + taosSetSignal(SIGINT, shellSigintHandler); + + shellGetGrantInfo(shell.conn); + + while (1) { + taosThreadCreate(&shell.pid, NULL, shellThreadLoop, shell.conn); + taosThreadJoin(shell.pid, NULL); + } + + return 0; } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index db73f2fc5c..6672cee367 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -14,697 +14,53 @@ */ #define __USE_XOPEN -#include "shellCommand.h" -#include "tglobal.h" -#include "tlog.h" +#include "shellInt.h" -#ifndef WINDOWS -#include -#include -#include -#endif - -#define OPT_ABORT 1 /* abort */ - -int indicator = 1; - -void insertChar(Command *cmd, char *c, int size); -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType); -const char *argp_program_version = version; -const char *argp_program_bug_address = ""; -static char doc[] = ""; -static char args_doc[] = ""; - -TdThread pid; -static tsem_t cancelSem; -extern void taos_init(); - -#ifndef WINDOWS -static struct argp_option options[] = { - {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, - {"password", 'p', NULL, 0, "The password to use when connecting to the server."}, - {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, - {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, - {"auth", 'A', "Auth", 0, "The auth string to use when connecting to the server."}, - {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, - {"dump-config",'C', NULL, 0, "Dump configuration."}, - {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, - {"raw-time", 'r', NULL, 0, "Output time as uint64_t."}, - {"file", 'f', "FILE", 0, "Script to run without enter the shell."}, - {"directory", 'D', "DIRECTORY", 0, "Use multi-thread to import all SQL files in the directory separately."}, - {"thread", 'T', "THREADNUM", 0, "Number of threads when using multi-thread to import data."}, - {"check", 'k', "CHECK", 0, "Check tables."}, - {"database", 'd', "DATABASE", 0, "Database to use when connecting to the server."}, - {"timezone", 'z', "TIMEZONE", 0, "Time zone of the shell, default is local."}, - {"status", 't', NULL, 0, "Check the service status."}, - {"verbose", 'v', NULL, 0, "Check the details of the service status."}, - {"netrole", 'n', "NETROLE", 0, "Net role when network connectivity test, default is startup, options: client|server|rpc|startup|sync|speed|fqdn."}, - {"pktlen", 'l', "PKTLEN", 0, "Packet length used for net test, default is 1000 bytes."}, - {"pktnum", 'N', "PKTNUM", 0, "Packet numbers used for net test, default is 100."}, -// Shuduo: 3.0 does not support UDP any more -// {"pkttype", 'S', "PKTTYPE", 0, "Packet type used for net test, default is TCP."}, - {0}}; - -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - SShellArguments *arguments = state->input; - wordexp_t full_path; - - switch (key) { - case 'h': - arguments->host = arg; - break; - case 'p': - break; - case 'P': - if (arg) { - arguments->port = atoi(arg); - } else { - fprintf(stderr, "Invalid port\n"); - return -1; - } - - break; - case 'z': - arguments->timezone = arg; - break; - case 'u': - arguments->user = arg; - break; - case 'A': - arguments->auth = arg; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) { - fprintf(stderr, "config file path: %s overflow max len %d\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1); - wordfree(&full_path); - return -1; - } - tstrncpy(configDir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'C': - arguments->dump_config = true; - break; - case 's': - arguments->commands = arg; - break; - case 'r': - arguments->is_raw_time = true; - break; - case 'f': - if ((0 == strlen(arg)) || (wordexp(arg, &full_path, 0) != 0)) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->file, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'D': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - tstrncpy(arguments->dir, full_path.we_wordv[0], TSDB_FILENAME_LEN); - wordfree(&full_path); - break; - case 'T': - if (arg) { - arguments->threadNum = atoi(arg); - } else { - fprintf(stderr, "Invalid number of threads\n"); - return -1; - } - break; - case 'k': - arguments->check = atoi(arg); - break; - case 't': - arguments->status = true; - break; - case 'v': - arguments->verbose = true; - break; - case 'd': - arguments->database = arg; - break; - case 'n': - arguments->netTestRole = arg; - break; - case 'l': - if (arg) { - arguments->pktLen = atoi(arg); - } else { - fprintf(stderr, "Invalid packet length\n"); - return -1; - } - break; - case 'N': - if (arg) { - arguments->pktNum = atoi(arg); - } else { - fprintf(stderr, "Invalid packet number\n"); - return -1; - } - break; - case 'S': - arguments->pktType = arg; - break; - case OPT_ABORT: - arguments->abort = 1; - break; - default: - return ARGP_ERR_UNKNOWN; - } - return 0; -} - -/* Our argp parser. */ -static struct argp argp = {options, parse_opt, args_doc, doc}; - -char LINUXCLIENT_VERSION[] = - "Welcome to the TDengine shell from %s, Client Version:%s\n" - "Copyright (c) 2020 by TAOS Data, Inc. All rights reserved.\n\n"; -char g_password[SHELL_MAX_PASSWORD_LEN]; - -static void parse_args(int argc, char *argv[], SShellArguments *arguments) { - for (int i = 1; i < argc; i++) { - if ((strncmp(argv[i], "-p", 2) == 0) || (strncmp(argv[i], "--password", 10) == 0)) { - printf(LINUXCLIENT_VERSION, tsOsName, taos_get_client_info()); - if ((strlen(argv[i]) == 2) || (strncmp(argv[i], "--password", 10) == 0)) { - printf("Enter password: "); - taosSetConsoleEcho(false); - if (scanf("%20s", g_password) > 1) { - fprintf(stderr, "password reading error\n"); - } - taosSetConsoleEcho(true); - if (EOF == getchar()) { - fprintf(stderr, "getchar() return EOF\n"); - } - } else { - tstrncpy(g_password, (char *)(argv[i] + 2), SHELL_MAX_PASSWORD_LEN); - strcpy(argv[i], "-p"); - } - arguments->password = g_password; - arguments->is_use_passwd = true; - } - } -} - -#endif -void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { -#ifdef WINDOWS -#else - static char verType[32] = {0}; - sprintf(verType, "version: %s\n", version); - - argp_program_version = verType; - - if (argc > 1) { - parse_args(argc, argv, arguments); - } - - argp_parse(&argp, argc, argv, 0, 0, arguments); - if (arguments->abort) { -#ifndef _ALPINE -#if 0 - error(10, 0, "ABORTED"); -#endif -#else - abort(); -#endif - } -#endif -} - -int32_t shellReadCommand(TAOS *con, char *command) { -#ifdef WINDOWS -#else - unsigned hist_counter = history.hend; - char utf8_array[10] = "\0"; - Command cmd; - memset(&cmd, 0, sizeof(cmd)); - cmd.buffer = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - cmd.command = (char *)taosMemoryCalloc(1, MAX_COMMAND_SIZE); - showOnScreen(&cmd); - - // Read input. - char c; - while (1) { - c = (char)getchar(); // getchar() return an 'int' value - - if (c == EOF) { - return c; - } - - if (c < 0) { // For UTF-8 - int count = countPrefixOnes(c); - utf8_array[0] = c; - for (int k = 1; k < count; k++) { - c = (char)getchar(); - utf8_array[k] = c; - } - insertChar(&cmd, utf8_array, count); - } else if (c < '\033') { - // Ctrl keys. TODO: Implement ctrl combinations - switch (c) { - case 1: // ctrl A - positionCursorHome(&cmd); - break; - case 3: - printf("\n"); - resetCommand(&cmd, ""); - kill(0, SIGINT); - break; - case 4: // EOF or Ctrl+D - printf("\n"); - taos_close(con); - // write the history - write_history(); - exitShell(); - break; - case 5: // ctrl E - positionCursorEnd(&cmd); - break; - case 8: - backspaceChar(&cmd); - break; - case '\n': - case '\r': - printf("\n"); - if (isReadyGo(&cmd)) { - sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosMemoryFreeClear(cmd.buffer); - taosMemoryFreeClear(cmd.command); - return 0; - } else { - updateBuffer(&cmd); - } - break; - case 11: // Ctrl + K; - clearLineAfter(&cmd); - break; - case 12: // Ctrl + L; - system("clear"); - showOnScreen(&cmd); - break; - case 21: // Ctrl + U; - clearLineBefore(&cmd); - break; - } - } else if (c == '\033') { - c = (char)getchar(); - switch (c) { - case '[': - c = (char)getchar(); - switch (c) { - case 'A': // Up arrow - if (hist_counter != history.hstart) { - hist_counter = (hist_counter + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE; - resetCommand(&cmd, (history.hist[hist_counter] == NULL) ? "" : history.hist[hist_counter]); - } - break; - case 'B': // Down arrow - if (hist_counter != history.hend) { - int next_hist = (hist_counter + 1) % MAX_HISTORY_SIZE; - - if (next_hist != history.hend) { - resetCommand(&cmd, (history.hist[next_hist] == NULL) ? "" : history.hist[next_hist]); - } else { - resetCommand(&cmd, ""); - } - hist_counter = next_hist; - } - break; - case 'C': // Right arrow - moveCursorRight(&cmd); - break; - case 'D': // Left arrow - moveCursorLeft(&cmd); - break; - case '1': - if ((c = (char)getchar()) == '~') { - // Home key - positionCursorHome(&cmd); - } - break; - case '2': - if ((c = (char)getchar()) == '~') { - // Insert key - } - break; - case '3': - if ((c = (char)getchar()) == '~') { - // Delete key - deleteChar(&cmd); - } - break; - case '4': - if ((c = (char)getchar()) == '~') { - // End key - positionCursorEnd(&cmd); - } - break; - case '5': - if ((c = (char)getchar()) == '~') { - // Page up key - } - break; - case '6': - if ((c = (char)getchar()) == '~') { - // Page down key - } - break; - case 72: - // Home key - positionCursorHome(&cmd); - break; - case 70: - // End key - positionCursorEnd(&cmd); - break; - } - break; - } - } else if (c == 0x7f) { - // press delete key - backspaceChar(&cmd); - } else { - insertChar(&cmd, &c, 1); - } - } - -#endif - return 0; -} - -void *shellLoopQuery(void *arg) { - if (indicator) { - getOldTerminalMode(); - indicator = 0; - } - - TAOS *con = (TAOS *)arg; - - setThreadName("shellLoopQuery"); - - taosThreadCleanupPush(cleanup_handler, NULL); - - char *command = taosMemoryMalloc(MAX_COMMAND_SIZE); - if (command == NULL) { - uError("failed to malloc command"); - return NULL; - } - - int32_t err = 0; - - do { - // Read command from shell. - memset(command, 0, MAX_COMMAND_SIZE); - setTerminalMode(); - err = shellReadCommand(con, command); - if (err) { - break; - } - resetTerminalMode(); - } while (shellRunCommand(con, command) == 0); - - taosMemoryFreeClear(command); - exitShell(); - - taosThreadCleanupPop(1); - - return NULL; -} - -void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } - -void clearScreen(int ecmd_pos, int cursor_pos) { -#ifdef WINDOWS -#else - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - positionCursor(cursor_y, LEFT); - positionCursor(command_x - cursor_x, DOWN); - fprintf(stdout, "\033[2K"); - for (int i = 0; i < command_x; i++) { - positionCursor(1, UP); - fprintf(stdout, "\033[2K"); - } - fflush(stdout); -#endif -} - -void showOnScreen(Command *cmd) { -#ifdef WINDOWS -#else - struct winsize w; - if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - // fprintf(stderr, "No stream device\n"); - w.ws_col = 120; - w.ws_row = 30; - } - - TdWchar wc; - int size = 0; - - // Print out the command. - char *total_string = taosMemoryMalloc(MAX_COMMAND_SIZE); - memset(total_string, '\0', MAX_COMMAND_SIZE); - if (strcmp(cmd->buffer, "") == 0) { - sprintf(total_string, "%s%s", PROMPT_HEADER, cmd->command); - } else { - sprintf(total_string, "%s%s", CONTINUE_PROMPT, cmd->command); - } - - int remain_column = w.ws_col; - /* size = cmd->commandSize + prompt_size; */ - for (char *str = total_string; size < cmd->commandSize + prompt_size;) { - int ret = taosMbToWchar(&wc, str, MB_CUR_MAX); - if (ret < 0) break; - size += ret; - /* assert(size >= 0); */ - int width = taosWcharWidth(wc); - if (remain_column > width) { - printf("%lc", wc); - remain_column -= width; - } else { - if (remain_column == width) { - printf("%lc\n\r", wc); - remain_column = w.ws_col; - } else { - printf("\n\r%lc", wc); - remain_column = w.ws_col - width; - } - } - - str = total_string + size; - } - - taosMemoryFree(total_string); - /* for (int i = 0; i < size; i++){ */ - /* char c = total_string[i]; */ - /* if (k % w.ws_col == 0) { */ - /* printf("%c\n\r", c); */ - /* } */ - /* else { */ - /* printf("%c", c); */ - /* } */ - /* k += 1; */ - /* } */ - - // Position the cursor - int cursor_pos = cmd->screenOffset + prompt_size; - int ecmd_pos = cmd->endOffset + prompt_size; - - int cursor_x = cursor_pos / w.ws_col; - int cursor_y = cursor_pos % w.ws_col; - // int cursor_y = cursor % w.ws_col; - int command_x = ecmd_pos / w.ws_col; - int command_y = ecmd_pos % w.ws_col; - // int command_y = (command.size() + prompt_size) % w.ws_col; - positionCursor(command_y, LEFT); - positionCursor(command_x, UP); - positionCursor(cursor_x, DOWN); - positionCursor(cursor_y, RIGHT); - fflush(stdout); -#endif -} - -void cleanup_handler(void *arg) { resetTerminalMode(); } - -void exitShell() { - taos_cleanup(); - exit(EXIT_SUCCESS); -} - -void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) { tsem_post(&cancelSem); } - -void *cancelHandler(void *arg) { - setThreadName("cancelHandler"); - - while (1) { - if (tsem_wait(&cancelSem) != 0) { - taosMsleep(10); - continue; - } - - resetTerminalMode(); - printf("\nReceive ctrl+c or other signal, quit shell.\n"); - exitShell(); - } - - return NULL; -} - -int checkVersion() { - if (sizeof(int8_t) != 1) { - printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t)); - return 0; - } - if (sizeof(int16_t) != 2) { - printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t)); - return 0; - } - if (sizeof(int32_t) != 4) { - printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t)); - return 0; - } - if (sizeof(int64_t) != 8) { - printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t)); - return 0; - } - return 1; -} - -// Global configurations -SShellArguments args = { - .host = NULL, - .user = NULL, - .database = NULL, - .timezone = NULL, - .is_raw_time = false, - .is_use_passwd = false, - .dump_config = false, - .file = "\0", - .dir = "\0", - .threadNum = 5, - .commands = NULL, - .pktLen = 1000, - .pktNum = 100, - .pktType = "TCP", - .netTestRole = NULL, -#ifndef TD_WINDOWS - .password = NULL, -#endif -}; - -void shellDumpConfig() { - if (!args.dump_config) return; - - SConfig *pCfg = taosGetCfg(); - if (NULL == pCfg) { - printf("TDengine read global config failed!\n"); - exit(EXIT_FAILURE); - } - cfgDumpCfg(pCfg, 0, 1); - exitShell(); -} - -void shellTestNetWork() { - if (args.netTestRole && args.netTestRole[0] != 0) { - taosNetTest(args.netTestRole, args.host, args.port, args.pktLen, args.pktNum, args.pktType); - exitShell(); - } -} - -void shellCheckServerStatus() { - if (!args.status && !args.verbose) return; - - TSDB_SERVER_STATUS code; - do { - char details[1024] = {0}; - code = taos_check_server_status(args.host, args.port, details, args.verbose ? 1024 : 0); - switch (code) { - case TSDB_SRV_STATUS_UNAVAILABLE: - printf("0: unavailable\n"); - break; - case TSDB_SRV_STATUS_NETWORK_OK: - printf("1: network ok\n"); - break; - case TSDB_SRV_STATUS_SERVICE_OK: - printf("2: service ok\n"); - break; - case TSDB_SRV_STATUS_SERVICE_DEGRADED: - printf("3: service degraded\n"); - break; - case TSDB_SRV_STATUS_EXTING: - printf("4: exiting\n"); - break; - } - if (strlen(details) != 0) { - printf("%s\n\n", details); - } - if (code == TSDB_SRV_STATUS_NETWORK_OK && args.verbose) { - taosMsleep(1000); - } else { - break; - } - } while (1); - - exitShell(); -} - -void shellExecute() { - TAOS *con = shellInit(&args); - if (con == NULL) { - exitShell(); - } - - if (tsem_init(&cancelSem, 0, 0) != 0) { - printf("failed to create cancel semphore\n"); - exitShell(); - } - - TdThread spid; - taosThreadCreate(&spid, NULL, cancelHandler, NULL); - - taosSetSignal(SIGTERM, shellQueryInterruptHandler); - taosSetSignal(SIGINT, shellQueryInterruptHandler); - taosSetSignal(SIGHUP, shellQueryInterruptHandler); - taosSetSignal(SIGABRT, shellQueryInterruptHandler); - - shellGetGrantInfo(con); - - while (1) { - taosThreadCreate(&pid, NULL, shellLoopQuery, con); - taosThreadJoin(pid, NULL); - } -} +SShellObj shell = {0}; int main(int argc, char *argv[]) { - if (!checkVersion()) exitShell(); + if (shellCheckIntSize() != 0) { + return -1; + } - shellParseArgument(argc, argv, &args); + if (shellParseArgs(argc, argv) != 0) { + return -1; + } + + if (shell.args.is_version) { + shellPrintVersion(); + return 0; + } + + if (shell.args.is_gen_auth) { + shellGenerateAuth(); + return 0; + } + + if (shell.args.is_help) { + shellPrintHelp(); + return 0; + } taos_init(); - shellDumpConfig(); - shellCheckServerStatus(); - shellTestNetWork(); - shellExecute(); - return 0; + if (shell.args.is_dump_config) { + shellDumpConfig(); + taos_cleanup(); + return 0; + } + + if (shell.args.is_startup || shell.args.is_check) { + shellCheckServerStatus(); + taos_cleanup(); + return 0; + } + + if (shell.args.netrole != NULL) { + shellTestNetWork(); + taos_cleanup(); + return 0; + } + + return shellExecute(); } diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c new file mode 100644 index 0000000000..c8ec31c48b --- /dev/null +++ b/tools/shell/src/shellNettest.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _GNU_SOURCE +#include "shellInt.h" + +static void shellWorkAsClient() { + SShellArgs *pArgs = &shell.args; + SRpcInit rpcInit = {0}; + SEpSet epSet = {.inUse = 0, .numOfEps = 1}; + SRpcMsg rpcRsp = {0}; + void *clientRpc = NULL; + char pass[TSDB_PASSWORD_LEN + 1] = {0}; + + taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass); + rpcInit.label = "CHK"; + rpcInit.numOfThreads = 1; + rpcInit.sessions = 16; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = tsShellActivityTimer * 1000; + rpcInit.user = "_dnd"; + rpcInit.ckey = "_key"; + rpcInit.spi = 1; + rpcInit.secret = pass; + + clientRpc = rpcOpen(&rpcInit); + if (clientRpc == NULL) { + printf("failed to init net test client since %s\n", terrstr()); + goto _OVER; + } + + if (pArgs->host == NULL) { + pArgs->host = tsFirst; + } + char fqdn[TSDB_FQDN_LEN] = {0}; + tstrncpy(fqdn, pArgs->host, TSDB_FQDN_LEN); + strtok(fqdn, ":"); + + if (pArgs->port == 0) { + pArgs->port = tsServerPort; + } + + printf("network test client is initialized, the server is %s:%u\n", fqdn, pArgs->port); + + tstrncpy(epSet.eps[0].fqdn, fqdn, TSDB_FQDN_LEN); + epSet.eps[0].port = (uint16_t)pArgs->port; + + int32_t totalSucc = 0; + uint64_t startTime = taosGetTimestampUs(); + + for (int32_t i = 0; i < pArgs->pktNum; ++i) { + SRpcMsg rpcMsg = {.ahandle = (void *)0x9525, .msgType = TDMT_DND_NET_TEST}; + rpcMsg.pCont = rpcMallocCont(pArgs->pktLen); + rpcMsg.contLen = pArgs->pktLen; + + printf("request is sent, size:%d\n", rpcMsg.contLen); + rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp); + if (rpcRsp.code == 0 && rpcRsp.contLen == rpcMsg.contLen) { + printf("response is received, size:%d\n", rpcMsg.contLen); + if (rpcRsp.code == 0) totalSucc++; + } else { + printf("response not received since %s\n", tstrerror(rpcRsp.code)); + } + + rpcFreeCont(rpcRsp.pCont); + rpcRsp.pCont = NULL; + } + + uint64_t endTime = taosGetTimestampUs(); + uint64_t elT = endTime - startTime; + + printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pArgs->pktNum, elT / 1000.0, + pArgs->pktLen / (elT / 1000000.0) / 1024.0 / 1024.0 * totalSucc); + +_OVER: + if (clientRpc != NULL) { + rpcClose(clientRpc); + } + if (rpcRsp.pCont != NULL) { + rpcFreeCont(rpcRsp.pCont); + } +} + +static void shellProcessMsg(void *p, SRpcMsg *pRpc, SEpSet *pEpSet) { + printf("request is received, size:%d\n", pRpc->contLen); + fflush(stdout); + SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0}; + rsp.pCont = rpcMallocCont(pRpc->contLen); + if (rsp.pCont == NULL) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.contLen = pRpc->contLen; + } + rpcSendResponse(&rsp); +} + +void shellNettestHandler(int32_t signum, void *sigInfo, void *context) { shellExit(); } + +static void shellWorkAsServer() { + SShellArgs *pArgs = &shell.args; + + if (pArgs->port == 0) { + pArgs->port = tsServerPort; + } + + SRpcInit rpcInit = {0}; + rpcInit.localPort = pArgs->port; + rpcInit.label = "CHK"; + rpcInit.numOfThreads = tsNumOfRpcThreads; + rpcInit.cfp = (RpcCfp)shellProcessMsg; + rpcInit.sessions = 10; + rpcInit.connType = TAOS_CONN_SERVER; + rpcInit.idleTime = tsShellActivityTimer * 1000; + + void *serverRpc = rpcOpen(&rpcInit); + if (serverRpc == NULL) { + printf("failed to init net test server since %s", terrstr()); + } else { + printf("network test server is initialized, port:%u\n", pArgs->port); + taosSetSignal(SIGTERM, shellNettestHandler); + while (1) taosMsleep(10); + } +} + +void shellTestNetWork() { + if (strcmp(shell.args.netrole, "client") == 0) { + shellWorkAsClient(); + } + + if (strcmp(shell.args.netrole, "server") == 0) { + shellWorkAsServer(); + } +} diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c new file mode 100644 index 0000000000..1529ac0e52 --- /dev/null +++ b/tools/shell/src/shellUtil.c @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _BSD_SOURCE +#define _GNU_SOURCE +#define _XOPEN_SOURCE +#define _DEFAULT_SOURCE + +#include "shellInt.h" + +bool shellRegexMatch(const char *s, const char *reg, int32_t cflags) { + regex_t regex = {0}; + char msgbuf[100] = {0}; + + /* Compile regular expression */ + if (regcomp(®ex, reg, cflags) != 0) { + fprintf(stderr, "Fail to compile regex"); + shellExit(); + } + + /* Execute regular expression */ + int32_t reti = regexec(®ex, s, 0, NULL, 0); + if (!reti) { + regfree(®ex); + return true; + } else if (reti == REG_NOMATCH) { + regfree(®ex); + return false; + } else { + regerror(reti, ®ex, msgbuf, sizeof(msgbuf)); + fprintf(stderr, "Regex match failed: %s\n", msgbuf); + regfree(®ex); + shellExit(); + } + + return false; +} + +int32_t shellCheckIntSize() { + if (sizeof(int8_t) != 1) { + printf("taos int8 size is %d(!= 1)", (int)sizeof(int8_t)); + return -1; + } + if (sizeof(int16_t) != 2) { + printf("taos int16 size is %d(!= 2)", (int)sizeof(int16_t)); + return -1; + } + if (sizeof(int32_t) != 4) { + printf("taos int32 size is %d(!= 4)", (int)sizeof(int32_t)); + return -1; + } + if (sizeof(int64_t) != 8) { + printf("taos int64 size is %d(!= 8)", (int)sizeof(int64_t)); + return -1; + } + return 0; +} + +void shellPrintVersion() { printf("version: %s\n", version); } + +void shellGenerateAuth() { + char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; + taosEncryptPass_c((uint8_t *)shell.args.password, strlen(shell.args.password), secretEncrypt); + printf("%s\n", secretEncrypt); + fflush(stdout); +} + +void shellDumpConfig() { + SConfig *pCfg = taosGetCfg(); + if (pCfg == NULL) { + printf("TDengine read global config failed!\n"); + } else { + cfgDumpCfg(pCfg, 1, true); + } + fflush(stdout); +} + +void shellCheckServerStatus() { + TSDB_SERVER_STATUS code; + + do { + char details[1024] = {0}; + code = taos_check_server_status(shell.args.host, shell.args.port, details, 1024); + switch (code) { + case TSDB_SRV_STATUS_UNAVAILABLE: + printf("0: unavailable\n"); + break; + case TSDB_SRV_STATUS_NETWORK_OK: + printf("1: network ok\n"); + break; + case TSDB_SRV_STATUS_SERVICE_OK: + printf("2: service ok\n"); + break; + case TSDB_SRV_STATUS_SERVICE_DEGRADED: + printf("3: service degraded\n"); + break; + case TSDB_SRV_STATUS_EXTING: + printf("4: exiting\n"); + break; + } + if (strlen(details) != 0) { + printf("%s\n\n", details); + } + fflush(stdout); + if (code == TSDB_SRV_STATUS_NETWORK_OK && shell.args.is_startup) { + taosMsleep(1000); + } else { + break; + } + } while (1); +} + +void shellExit() { + if (shell.conn != NULL) { + taos_close(shell.conn); + shell.conn = NULL; + } + taos_cleanup(); + exit(EXIT_FAILURE); +} \ No newline at end of file diff --git a/tools/shell/src/tnettest.c b/tools/shell/src/tnettest.c deleted file mode 100644 index 04b99ebc75..0000000000 --- a/tools/shell/src/tnettest.c +++ /dev/null @@ -1,514 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#define ALLOW_FORBID_FUNC -#include "os.h" -#include "taosdef.h" -#include "tmsg.h" -#include "taoserror.h" -#include "tlog.h" -#include "tglobal.h" -#include "trpc.h" -#include "rpcHead.h" -#include "tchecksum.h" -#include "syncMsg.h" - -#include "osSocket.h" - -#define MAX_PKG_LEN (64 * 1000) -#define MAX_SPEED_PKG_LEN (1024 * 1024 * 1024) -#define MIN_SPEED_PKG_LEN 1024 -#define MAX_SPEED_PKG_NUM 10000 -#define MIN_SPEED_PKG_NUM 1 -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) - -extern int tsRpcMaxUdpSize; - -typedef struct { - char * hostFqdn; - uint32_t hostIp; - int32_t port; - int32_t pktLen; -} STestInfo; - -static void *taosNetBindUdpPort(void *sarg) { - STestInfo *pinfo = (STestInfo *)sarg; - int32_t port = pinfo->port; - SOCKET serverSocket; - char buffer[BUFFER_SIZE]; - int32_t iDataNum; - socklen_t sin_size; - int32_t bufSize = 1024000; - - struct sockaddr_in server_addr; - struct sockaddr_in clientAddr; - - setThreadName("netBindUdpPort"); - - if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - uError("failed to create UDP socket since %s", strerror(errno)); - return NULL; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - uError("failed to bind UDP port:%d since %s", port, strerror(errno)); - return NULL; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(serverSocket); - return NULL; - } - pSocket->fd = serverSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("UDP server at port:%d is listening", port); - - while (1) { - memset(buffer, 0, BUFFER_SIZE); - sin_size = sizeof(*(struct sockaddr *)&server_addr); - iDataNum = recvfrom(serverSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&clientAddr, &sin_size); - - if (iDataNum < 0) { - uDebug("failed to perform recvfrom func at %d since %s", port, strerror(errno)); - continue; - } - - uInfo("UDP: recv:%d bytes from %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); - - if (iDataNum > 0) { - iDataNum = taosSendto(pSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int32_t)sin_size); - } - - uInfo("UDP: send:%d bytes to %s at %d", iDataNum, taosInetNtoa(clientAddr.sin_addr), port); - } - - taosCloseSocket(&pSocket); - return NULL; -} - -static void *taosNetBindTcpPort(void *sarg) { - struct sockaddr_in server_addr; - struct sockaddr_in clientAddr; - - STestInfo *pinfo = sarg; - int32_t port = pinfo->port; - SOCKET serverSocket; - int32_t addr_len = sizeof(clientAddr); - SOCKET client; - char buffer[BUFFER_SIZE]; - - setThreadName("netBindTcpPort"); - - if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { - uError("failed to create TCP socket since %s", strerror(errno)); - return NULL; - } - - bzero(&server_addr, sizeof(server_addr)); - server_addr.sin_family = AF_INET; - server_addr.sin_port = htons(port); - server_addr.sin_addr.s_addr = htonl(INADDR_ANY); - - int32_t reuse = 1; - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(serverSocket); - return NULL; - } - pSocket->fd = serverSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { - uError("failed to bind TCP port:%d since %s", port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (taosKeepTcpAlive(pSocket) < 0) { - uError("failed to set tcp server keep-alive option since %s", strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - if (listen(serverSocket, 10) < 0) { - uError("failed to listen TCP port:%d since %s", port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("TCP server at port:%d is listening", port); - - while (1) { - client = accept(serverSocket, (struct sockaddr *)&clientAddr, (socklen_t *)&addr_len); - if (client < 0) { - uDebug("TCP: failed to accept at port:%d since %s", port, strerror(errno)); - continue; - } - - int32_t ret = taosReadMsg(pSocket, buffer, pinfo->pktLen); - if (ret < 0 || ret != pinfo->pktLen) { - uError("TCP: failed to read %d bytes at port:%d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("TCP: read:%d bytes from %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); - - ret = taosWriteMsg(pSocket, buffer, pinfo->pktLen); - if (ret < 0) { - uError("TCP: failed to write %d bytes at %d since %s", pinfo->pktLen, port, strerror(errno)); - taosCloseSocket(&pSocket); - return NULL; - } - - uInfo("TCP: write:%d bytes to %s at %d", pinfo->pktLen, taosInetNtoa(clientAddr.sin_addr), port); - } - - taosCloseSocket(&pSocket); - return NULL; -} - -static int32_t taosNetCheckTcpPort(STestInfo *info) { - SOCKET clientSocket; - char buffer[BUFFER_SIZE] = {0}; - - if ((clientSocket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - uError("failed to create TCP client socket since %s", strerror(errno)); - return -1; - } - - int32_t reuse = 1; - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(clientSocket); - return -1; - } - pSocket->fd = clientSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - struct sockaddr_in serverAddr; - memset((char *)&serverAddr, 0, sizeof(serverAddr)); - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = (uint16_t)htons((uint16_t)info->port); - serverAddr.sin_addr.s_addr = info->hostIp; - - if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - uError("TCP: failed to connect port %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - taosKeepTcpAlive(pSocket); - - sprintf(buffer, "client send TCP pkg to %s:%d, content: 1122334455", taosIpStr(info->hostIp), info->port); - sprintf(buffer + info->pktLen - 16, "1122334455667788"); - - int32_t ret = taosWriteMsg(pSocket, buffer, info->pktLen); - if (ret < 0) { - uError("TCP: failed to write msg to %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - ret = taosReadMsg(pSocket, buffer, info->pktLen); - if (ret < 0) { - uError("TCP: failed to read msg from %s:%d since %s", taosIpStr(info->hostIp), info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - taosCloseSocket(&pSocket); - return 0; -} - -static int32_t taosNetCheckUdpPort(STestInfo *info) { - SOCKET clientSocket; - char buffer[BUFFER_SIZE] = {0}; - int32_t iDataNum = 0; - int32_t bufSize = 1024000; - - struct sockaddr_in serverAddr; - - if ((clientSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) { - uError("failed to create udp client socket since %s", strerror(errno)); - return -1; - } - - TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); - if (pSocket == NULL) { - taosCloseSocketNoCheck1(clientSocket); - return -1; - } - pSocket->fd = clientSocket; - pSocket->refId = 0; - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the send buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return -1; - } - - if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { - uError("failed to set the receive buffer size for UDP socket\n"); - taosCloseSocket(&pSocket); - return -1; - } - - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(info->port); - serverAddr.sin_addr.s_addr = info->hostIp; - - struct in_addr ipStr; - memcpy(&ipStr, &info->hostIp, 4); - sprintf(buffer, "client send udp pkg to %s:%d, content: 1122334455", taosInetNtoa(ipStr), info->port); - sprintf(buffer + info->pktLen - 16, "1122334455667788"); - - socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); - - iDataNum = taosSendto(pSocket, buffer, info->pktLen, 0, (struct sockaddr *)&serverAddr, (int32_t)sin_size); - if (iDataNum < 0 || iDataNum != info->pktLen) { - uError("UDP: failed to perform sendto func since %s", strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - memset(buffer, 0, BUFFER_SIZE); - sin_size = sizeof(*(struct sockaddr *)&serverAddr); - iDataNum = recvfrom(clientSocket, buffer, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); - - if (iDataNum < 0 || iDataNum != info->pktLen) { - uError("UDP: received ack:%d bytes(expect:%d) from port:%d since %s", iDataNum, info->pktLen, info->port, strerror(errno)); - taosCloseSocket(&pSocket); - return -1; - } - - taosCloseSocket(&pSocket); - return 0; -} - -static void taosNetCheckPort(uint32_t hostIp, int32_t startPort, int32_t endPort, int32_t pktLen) { - int32_t ret; - STestInfo info; - - memset(&info, 0, sizeof(STestInfo)); - info.hostIp = hostIp; - info.pktLen = pktLen; - - for (int32_t port = startPort; port <= endPort; port++) { - info.port = port; - ret = taosNetCheckTcpPort(&info); - if (ret != 0) { - printf("failed to test TCP port:%d\n", port); - } else { - printf("successed to test TCP port:%d\n", port); - } - - ret = taosNetCheckUdpPort(&info); - if (ret != 0) { - printf("failed to test UDP port:%d\n", port); - } else { - printf("successed to test UDP port:%d\n", port); - } - } -} - -static void taosNetTestClient(char *host, int32_t startPort, int32_t pkgLen) { - uInfo("work as client, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); - - uint32_t serverIp = taosGetIpv4FromFqdn(host); - if (serverIp == 0xFFFFFFFF) { - uError("failed to resolve fqdn:%s", host); - exit(-1); - } - - uInfo("server ip:%s is resolved from host:%s", taosIpStr(serverIp), host); - taosNetCheckPort(serverIp, startPort, startPort, pkgLen); -} - -static void taosNetTestServer(char *host, int32_t startPort, int32_t pkgLen) { - uInfo("work as server, host:%s Port:%d pkgLen:%d\n", host, startPort, pkgLen); - - int32_t port = startPort; - int32_t num = 1; - if (num < 0) num = 1; - - TdThread *pids = taosMemoryMalloc(2 * num * sizeof(TdThread)); - STestInfo *tinfos = taosMemoryMalloc(num * sizeof(STestInfo)); - STestInfo *uinfos = taosMemoryMalloc(num * sizeof(STestInfo)); - - for (int32_t i = 0; i < num; i++) { - STestInfo *tcpInfo = tinfos + i; - tcpInfo->port = port + i; - tcpInfo->pktLen = pkgLen; - - if (taosThreadCreate(pids + i, NULL, taosNetBindTcpPort, tcpInfo) != 0) { - uInfo("failed to create TCP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); - exit(-1); - } - - STestInfo *udpInfo = uinfos + i; - udpInfo->port = port + i; - tcpInfo->pktLen = pkgLen; - if (taosThreadCreate(pids + num + i, NULL, taosNetBindUdpPort, udpInfo) != 0) { - uInfo("failed to create UDP test thread, %s:%d", tcpInfo->hostFqdn, tcpInfo->port); - exit(-1); - } - } - - for (int32_t i = 0; i < num; i++) { - taosThreadJoin(pids[i], NULL); - taosThreadJoin(pids[(num + i)], NULL); - } -} - -static void taosNetCheckSpeed(char *host, int32_t port, int32_t pkgLen, - int32_t pkgNum, char *pkgType) { -#if 0 - - // record config - int32_t compressTmp = tsCompressMsgSize; - int32_t maxUdpSize = tsRpcMaxUdpSize; - int32_t forceTcp = tsRpcForceTcp; - - if (0 == strcmp("tcp", pkgType)){ - tsRpcForceTcp = 1; - tsRpcMaxUdpSize = 0; // force tcp - } else { - tsRpcForceTcp = 0; - tsRpcMaxUdpSize = INT_MAX; - } - tsCompressMsgSize = -1; - - SEpSet epSet; - SRpcMsg reqMsg; - SRpcMsg rspMsg; - void * pRpcConn; - char secretEncrypt[32] = {0}; - char spi = 0; - pRpcConn = taosNetInitRpc(secretEncrypt, spi); - if (NULL == pRpcConn) { - uError("failed to init client rpc"); - return; - } - - printf("check net spend, host:%s port:%d pkgLen:%d pkgNum:%d pkgType:%s\n\n", host, port, pkgLen, pkgNum, pkgType); - int32_t totalSucc = 0; - uint64_t startT = taosGetTimestampUs(); - for (int32_t i = 1; i <= pkgNum; i++) { - uint64_t startTime = taosGetTimestampUs(); - - memset(&epSet, 0, sizeof(SEpSet)); - strcpy(epSet.eps[0].fqdn, host); - epSet.eps[0].port = port; - epSet.numOfEps = 1; - - reqMsg.msgType = TDMT_DND_NETWORK_TEST; - reqMsg.pCont = rpcMallocCont(pkgLen); - reqMsg.contLen = pkgLen; - reqMsg.code = 0; - reqMsg.handle = NULL; // rpc handle returned to app - reqMsg.ahandle = NULL; // app handle set by client - strcpy(reqMsg.pCont, "nettest speed"); - - rpcSendRecv(pRpcConn, &epSet, &reqMsg, &rspMsg); - - int code = 0; - if ((rspMsg.code != 0) || (rspMsg.msgType != TDMT_DND_NETWORK_TEST + 1)) { - uError("ret code 0x%x %s", rspMsg.code, tstrerror(rspMsg.code)); - code = -1; - }else{ - totalSucc ++; - } - - rpcFreeCont(rspMsg.pCont); - - uint64_t endTime = taosGetTimestampUs(); - uint64_t el = endTime - startTime; - printf("progress:%5d/%d\tstatus:%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", i, pkgNum, code, el/1000.0, pkgLen/(el/1000000.0)/1024.0/1024.0); - } - int64_t endT = taosGetTimestampUs(); - uint64_t elT = endT - startT; - printf("\ntotal succ:%5d/%d\tcost:%8.2lf ms\tspeed:%8.2lf MB/s\n", totalSucc, pkgNum, elT/1000.0, pkgLen/(elT/1000000.0)/1024.0/1024.0*totalSucc); - - rpcClose(pRpcConn); - - // return config - tsCompressMsgSize = compressTmp; - tsRpcMaxUdpSize = maxUdpSize; - tsRpcForceTcp = forceTcp; - return; -#endif -} - -void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType) { - tsLogEmbedded = 1; - if (host == NULL) host = tsLocalFqdn; - if (port == 0) port = tsServerPort; - if (0 == strcmp("speed", role)) { - if (pkgLen <= MIN_SPEED_PKG_LEN) pkgLen = MIN_SPEED_PKG_LEN; - if (pkgLen > MAX_SPEED_PKG_LEN) pkgLen = MAX_SPEED_PKG_LEN; - if (pkgNum <= MIN_SPEED_PKG_NUM) pkgNum = MIN_SPEED_PKG_NUM; - if (pkgNum > MAX_SPEED_PKG_NUM) pkgNum = MAX_SPEED_PKG_NUM; - } else { - if (pkgLen <= 10) pkgLen = 1000; - if (pkgLen > MAX_PKG_LEN) pkgLen = MAX_PKG_LEN; - } - - if (0 == strcmp("client", role)) { - taosNetTestClient(host, port, pkgLen); - } else if (0 == strcmp("server", role)) { - taosNetTestServer(host, port, pkgLen); - } else if (0 == strcmp("speed", role)) { - tsLogEmbedded = 0; - char type[10] = {0}; - taosNetCheckSpeed(host, port, pkgLen, pkgNum, strtolower(type, pkgType)); - } else { - TASSERT(1); - } - - tsLogEmbedded = 0; -}