diff --git a/Jenkinsfile2 b/Jenkinsfile2 index a2043797c0..441cf89626 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -88,12 +88,6 @@ def pre_test(){ cmake .. > /dev/null make -j4> /dev/null ''' - sh''' - cd ${WKPY} - git reset --hard - git pull - pip3 install . - ''' return 1 } @@ -103,7 +97,6 @@ pipeline { environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' } stages { stage('pre_build'){ @@ -124,11 +117,6 @@ pipeline { ./test-all.sh b1fq ''' sh''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - sh''' cd ${WKC}/debug ctest ''' diff --git a/cmake/cmake.define b/cmake/cmake.define index c985ba1cc7..c9a188600a 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.16) +set(CMAKE_VERBOSE_MAKEFILE ON) + #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) @@ -43,11 +45,14 @@ ENDIF () IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) - SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(COMMON_FLAGS "/W3 /D_WIN32") - IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) - SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") - ENDIF () + # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) + # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") + # ENDIF () + + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}") ELSE () IF (${SANITIZER} MATCHES "true") diff --git a/cmake/cmake.options b/cmake/cmake.options index b51f096185..f32c5acdd1 100644 --- a/cmake/cmake.options +++ b/cmake/cmake.options @@ -25,9 +25,28 @@ IF(${TD_WINDOWS}) ON ) -ENDIF () + MESSAGE("build msvcregex Win32") + option( + BUILD_MSVCREGEX + "If build msvcregex on Windows" + ON + ) -IF(${TD_LINUX} MATCHES TRUE) + MESSAGE("build wcwidth Win32") + option( + BUILD_WCWIDTH + "If build wcwidth on Windows" + ON + ) + + MESSAGE("build wingetopt Win32") + option( + BUILD_WINGETOPT + "If build wingetopt on Windows" + ON + ) + +ENDIF () option( BUILD_TEST @@ -35,8 +54,6 @@ option( ON ) -ENDIF () - option( BUILD_WITH_LEVELDB "If build with leveldb" diff --git a/cmake/msvcregex_CMakeLists.txt.in b/cmake/msvcregex_CMakeLists.txt.in new file mode 100644 index 0000000000..686ac4cb5d --- /dev/null +++ b/cmake/msvcregex_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# msvcregex +ExternalProject_Add(msvcregex + GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/cmake/pthread_CMakeLists.txt.in b/cmake/pthread_CMakeLists.txt.in index 2553df6500..774f5dd972 100644 --- a/cmake/pthread_CMakeLists.txt.in +++ b/cmake/pthread_CMakeLists.txt.in @@ -3,7 +3,7 @@ ExternalProject_Add(pthread GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32 GIT_TAG v3.0.3.1 - SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32" + SOURCE_DIR "${TD_CONTRIB_DIR}/pthread" BINARY_DIR "" #BUILD_IN_SOURCE TRUE CONFIGURE_COMMAND "" diff --git a/cmake/wcwidth_CMakeLists.txt.in b/cmake/wcwidth_CMakeLists.txt.in new file mode 100644 index 0000000000..234ccbf915 --- /dev/null +++ b/cmake/wcwidth_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# wcwidth +ExternalProject_Add(wcwidth + GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/cmake/wingetopt_CMakeLists.txt.in b/cmake/wingetopt_CMakeLists.txt.in new file mode 100644 index 0000000000..b93925c170 --- /dev/null +++ b/cmake/wingetopt_CMakeLists.txt.in @@ -0,0 +1,13 @@ + +# wingetopt +ExternalProject_Add(wingetopt + GIT_REPOSITORY https://github.com/alex85k/wingetopt.git + GIT_TAG master + SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt" + BINARY_DIR "" + #BUILD_IN_SOURCE TRUE + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1ddc765c5c..19923a5ad6 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,9 +14,24 @@ if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() -# gnu regex -if(${BUILD_GNUREGEX}) - cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# iconv +if(${BUILD_WITH_ICONV}) + cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# msvc regex +if(${BUILD_MSVCREGEX}) + cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# wcwidth +if(${BUILD_WCWIDTH}) + cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + +# wingetopt +if(${BUILD_WINGETOPT}) + cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() # googletest @@ -99,8 +114,27 @@ if(${BUILD_TEST}) target_include_directories( gtest PUBLIC $ - PUBLIC $ ) + if(${TD_WINDOWS}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_WINDOWS}) + if(${TD_LINUX}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_LINUX}) + if(${TD_DARWIN}) + target_include_directories( + gtest + PUBLIC $ + ) + endif(${TD_DARWIN}) + + endif(${BUILD_TEST}) # cJson @@ -182,6 +216,53 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) +# pthread +if(${BUILD_PTHREAD}) + set(CMAKE_BUILD_TYPE release) + add_definitions(-DPTW32_STATIC_LIB) + add_subdirectory(pthread) + set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) + add_library(pthread STATIC IMPORTED GLOBAL) + SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) +endif() + +# iconv +if(${BUILD_WITH_ICONV}) + add_subdirectory(iconv) +endif(${BUILD_WITH_ICONV}) + +# wingetopt +if(${BUILD_WINGETOPT}) + add_subdirectory(wingetopt) +endif(${BUILD_WINGETOPT}) + +# msvcregex +if(${BUILD_MSVCREGEX}) + add_library(msvcregex STATIC "") + target_sources(msvcregex + PRIVATE "msvcregex/regex.c" + ) + target_include_directories(msvcregex + PRIVATE "msvcregex" + ) + target_link_libraries(msvcregex + INTERFACE Shell32 + ) + SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex) +endif(${BUILD_MSVCREGEX}) + +# msvcregex +if(${BUILD_WCWIDTH}) + add_library(wcwidth STATIC "") + target_sources(wcwidth + PRIVATE "wcwidth/wcwidth.c" + ) + target_include_directories(wcwidth + PRIVATE "wcwidth" + ) + SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth) +endif(${BUILD_WCWIDTH}) + # CRAFT if(${BUILD_WITH_CRAFT}) add_library(craft STATIC IMPORTED GLOBAL) @@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE}) target_link_libraries(sqlite INTERFACE m INTERFACE pthread - INTERFACE dl ) + if(NOT TD_WINDOWS) + target_link_libraries(sqlite + INTERFACE dl + ) + endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) # pthread diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index e0f6ae64bd..ec8b44b673 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,2 +1,9 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) +if(${BUILD_WINGETOPT}) + target_link_libraries(simulate_vnode PUBLIC wingetopt) + target_include_directories( + simulate_vnode + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) +endif() \ No newline at end of file diff --git a/contrib/test/tdev/src/main.c b/contrib/test/tdev/src/main.c index 5e1de83e88..e40040ce97 100644 --- a/contrib/test/tdev/src/main.c +++ b/contrib/test/tdev/src/main.c @@ -6,43 +6,39 @@ #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) -#define tPutA(buf, val) \ - ({ \ - memcpy(buf, &val, sizeof(val)); \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutA(void **buf, uint64_t val) { + memcpy(buf, &val, sizeof(val)); + *buf = POINTER_SHIFT(buf, sizeof(val)); +} -#define tPutB(buf, val) \ - ({ \ - ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \ - ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \ - ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \ - ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \ - ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ - ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ - ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ - ((uint8_t *)buf)[0] = (val)&0xff; \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutB(void **buf, uint64_t val) { + ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; + ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; + ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; + ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; + ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; + ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; + ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; + ((uint8_t *)buf)[0] = (val)&0xff; + *buf = POINTER_SHIFT(buf, sizeof(val)); +} -#define tPutC(buf, val) \ - ({ \ - if (buf) { \ - ((uint64_t *)buf)[0] = (val); \ - POINTER_SHIFT(buf, sizeof(val)); \ - } \ - NULL; \ - }) +static inline void tPutC(void **buf, uint64_t val) { + if (buf) { + ((uint64_t *)buf)[0] = (val); + POINTER_SHIFT(buf, sizeof(val)); + } + *buf = NULL; +} -#define tPutD(buf, val) \ - ({ \ - uint64_t tmp = val; \ - for (size_t i = 0; i < sizeof(val); i++) { \ - ((uint8_t *)buf)[i] = tmp & 0xff; \ - tmp >>= 8; \ - } \ - POINTER_SHIFT(buf, sizeof(val)); \ - }) +static inline void tPutD(void **buf, uint64_t val) { + uint64_t tmp = val; + for (size_t i = 0; i < sizeof(val); i++) { + ((uint8_t *)buf)[i] = tmp & 0xff; + tmp >>= 8; + } + *buf = POINTER_SHIFT(buf, sizeof(val)); +} static inline void tPutE(void **buf, uint64_t val) { if (buf) { @@ -61,7 +57,7 @@ static void func(T t) { switch (t) { case A: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutA(pBuf, val); + tPutA(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -69,7 +65,7 @@ static void func(T t) { break; case B: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutB(pBuf, val); + tPutB(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -77,7 +73,7 @@ static void func(T t) { break; case C: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutC(pBuf, val); + tPutC(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -85,7 +81,7 @@ static void func(T t) { break; case D: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - pBuf = tPutD(pBuf, val); + tPutD(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } diff --git a/example/CMakeLists.txt b/example/CMakeLists.txt index fe5f8cf7ed..80059dde4d 100644 --- a/example/CMakeLists.txt +++ b/example/CMakeLists.txt @@ -39,6 +39,7 @@ target_include_directories(tstream target_include_directories(demoapi PUBLIC "${TD_SOURCE_DIR}/include/client" + PUBLIC "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) diff --git a/example/src/demoapi.c b/example/src/demoapi.c index afa9e19d4d..c3a6d26776 100644 --- a/example/src/demoapi.c +++ b/example/src/demoapi.c @@ -4,10 +4,12 @@ #include #include #include -#include +// #include #include +#ifndef WINDOWS #include - +#endif +#include "osSleep.h" #include "taos.h" #define debugPrint(fmt, ...) \ @@ -32,6 +34,7 @@ int64_t g_num_of_tb = 2; int64_t g_num_of_rec = 3; +#ifndef WINDOWS static struct argp_option options[] = { {"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."}, {"records", 'n', "NUMBER", 0, @@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } static struct argp argp = {options, parse_opt, "", ""}; - +#endif static void prepare_data(TAOS* taos) { TAOS_RES *res; res = taos_query(taos, "drop database if exists test;"); taos_free_result(res); - usleep(100000); + taosMsleep(100); res = taos_query(taos, "create database test;"); taos_free_result(res); - usleep(100000); + taosMsleep(100); taos_select_db(taos, "test"); char command[1024] = {0}; @@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) { taos_free_result(res); for (int64_t i = 0; i < g_num_of_tb; i ++) { + // sprintf(command, "create table t%"PRId64" using meters " + // "tags(%"PRId64", '%s', '%s', '%s');", + // i, i, (i%2)?"beijing":"shanghai", + // (i%2)?"朝阳区":"黄浦区", + // (i%2)?"长安街":"中山路"); sprintf(command, "create table t%"PRId64" using meters " - "tags(%"PRId64", '%s', '%s', '%s');", - i, i, (i%2)?"beijing":"shanghai", - (i%2)?"朝阳区":"黄浦区", - (i%2)?"长安街":"中山路"); -/* sprintf(command, "create table t%"PRId64" using meters " "tags(%"PRId64", '%s', '%s', '%s');", i, i, (i%2)?"beijing":"shanghai", (i%2)?"chaoyang":"huangpu", (i%2?"changan street":"jianguo rd")); - */ res = taos_query(taos, command); if ((res) && (0 == taos_errno(res))) { okPrint("t%" PRId64 " created\n", i); @@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) { "'%c%d', '%s%c%d', '%c%d')", i, 1650000000000+j, (float)j, j, 'a'+(int)j%25, rand(), - "涛思", 'z' - (int)j%25, rand(), + // "涛思", 'z' - (int)j%25, rand(), + "TAOS", 'z' - (int)j%25, rand(), 'b' - (int)j%25, rand() ); res = taos_query(taos, command); @@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %"PRId64"\n", f, c, - *(int64_t*)(row[f]+c*sizeof(int64_t))); + *(int64_t*)((char*)(row[f])+c*sizeof(int64_t))); } break; @@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %d\n", f, c, - *(int32_t*)(row[f]+c*sizeof(int32_t))); + *(int32_t*)((char*)(row[f])+c*sizeof(int32_t))); } break; @@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { printf("col%d, row: %"PRId64", " "value: %f\n", f, c, - *(float*)(row[f]+c*sizeof(float))); + *(float*)((char*)(row[f])+c*sizeof(float))); } break; @@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) { if (offsets) { for (int c = 0; c < rows; c++) { if (offsets[c] != -1) { - int length = *(int16_t*)(row[f] + offsets[c]); + int length = *(int16_t*)((char*)(row[f]) + offsets[c]); char *buf = calloc(1, length + 1); - strncpy(buf, (char *)(row[f] + offsets[c] + 2), length); + strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length); printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n", c, f, offsets[c], length, buf); free(buf); @@ -314,8 +317,9 @@ int main(int argc, char *argv[]) { const char* host = "127.0.0.1"; const char* user = "root"; const char* passwd = "taosdata"; - +#ifndef WINDOWS argp_parse(&argp, argc, argv, 0, 0, NULL); +#endif TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); diff --git a/include/client/taos.h b/include/client/taos.h index 526edae8af..55deee4fad 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -146,6 +146,7 @@ DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_affected_rows(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_affected_rows_once(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 6e344dfdb4..af8bda2593 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -78,13 +78,6 @@ typedef enum { TSDB_SMA_TYPE_ROLLUP = 2, // Rollup SMA } ETsdbSmaType; -typedef enum { - TSDB_BSMA_TYPE_NONE = 0, // no block-wise SMA - TSDB_BSMA_TYPE_I = 1, // sum/min/max(default) -} ETsdbBSmaType; - -#define TSDB_BSMA_TYPE_LATEST TSDB_BSMA_TYPE_I - extern char *qtypeStr[]; #define TSDB_PORT_HTTP 11 diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 809390a9d2..5ea53b5510 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -326,6 +326,13 @@ int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp); int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp); void* taosDecodeSEpSet(const void* buf, SEpSet* pEp); +typedef struct { + SEpSet epSet; +} SMEpSet; + +int32_t tSerializeSMEpSet(void* buf, int32_t bufLen, SMEpSet* pReq); +int32_t tDeserializeSMEpSet(void* buf, int32_t buflen, SMEpSet* pReq); + typedef struct { int8_t connType; int32_t pid; @@ -682,6 +689,7 @@ int32_t tDeserializeSDropFuncReq(void* buf, int32_t bufLen, SDropFuncReq* pReq); typedef struct { int32_t numOfFuncs; + bool ignoreCodeComment; SArray* pFuncNames; } SRetrieveFuncReq; @@ -710,6 +718,7 @@ typedef struct { int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); int32_t tDeserializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); +void tFreeSFuncInfo(SFuncInfo *pInfo); void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp* pRsp); typedef struct { @@ -982,7 +991,6 @@ int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { - int32_t type; char db[TSDB_DB_FNAME_LEN]; char tb[TSDB_TABLE_NAME_LEN]; int64_t showId; @@ -1275,11 +1283,16 @@ typedef struct { } SMVCreateStreamRsp, SMSCreateStreamRsp; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - int8_t igExists; - char* sql; - char* ast; - char subscribeDbName[TSDB_DB_NAME_LEN]; + char name[TSDB_TOPIC_FNAME_LEN]; + int8_t igExists; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; + char* sql; + char* ast; + int64_t subDbUid; + char subscribeDbName[TSDB_DB_NAME_LEN]; } SCMCreateTopicReq; int32_t tSerializeSCMCreateTopicReq(void* buf, int32_t bufLen, const SCMCreateTopicReq* pReq); @@ -1473,8 +1486,12 @@ typedef struct { typedef struct { float xFilesFactor; int32_t delay; - int8_t nFuncIds; + int32_t qmsg1Len; + int32_t qmsg2Len; func_id_t* pFuncIds; + char* qmsg1; // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1 + char* qmsg2; // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2 + int8_t nFuncIds; } SRSmaParam; typedef struct SVCreateTbReq { @@ -1934,12 +1951,22 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { return buf; } +enum { + TOPIC_SUB_TYPE__DB = 1, + TOPIC_SUB_TYPE__TABLE, +}; + typedef struct { int64_t leftForVer; int32_t vgId; int64_t oldConsumerId; int64_t newConsumerId; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int8_t subType; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; char* qmsg; } SMqRebVgReq; @@ -1950,7 +1977,14 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); tlen += taosEncodeString(buf, pReq->subKey); - tlen += taosEncodeString(buf, pReq->qmsg); + tlen += taosEncodeFixedI8(buf, pReq->subType); + tlen += taosEncodeFixedI8(buf, pReq->withTbName); + tlen += taosEncodeFixedI8(buf, pReq->withSchema); + tlen += taosEncodeFixedI8(buf, pReq->withTag); + tlen += taosEncodeFixedI8(buf, pReq->withTagSchema); + if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + tlen += taosEncodeString(buf, pReq->qmsg); + } return tlen; } @@ -1960,7 +1994,14 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); buf = taosDecodeStringTo(buf, pReq->subKey); - buf = taosDecodeString(buf, &pReq->qmsg); + buf = taosDecodeFixedI8(buf, &pReq->subType); + buf = taosDecodeFixedI8(buf, &pReq->withTbName); + buf = taosDecodeFixedI8(buf, &pReq->withSchema); + buf = taosDecodeFixedI8(buf, &pReq->withTag); + buf = taosDecodeFixedI8(buf, &pReq->withTagSchema); + if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + buf = taosDecodeString(buf, &pReq->qmsg); + } return (void*)buf; } @@ -2300,9 +2341,10 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW) { } } -static FORCE_INLINE void tdFreeTSmaWrapper(STSmaWrapper* pSW) { +static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW) { tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); + taosMemoryFree(pSW); + return NULL; } static FORCE_INLINE int32_t tEncodeTSma(void** buf, const STSma* pSma) { @@ -2690,6 +2732,7 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } return buf; } + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/trow.h b/include/common/trow.h index 7b0c009a11..bc5413f9ce 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -1359,7 +1359,7 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) { printf(">>>"); for (int i = 0; i < pSchema->numOfCols; ++i) { STColumn *stCol = pSchema->columns + i; - SCellVal sVal = {.valType = 255, .val = NULL}; + SCellVal sVal = { 255, NULL}; if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { break; } diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 528cea8828..d5958d432b 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -26,6 +26,9 @@ extern "C" { typedef struct SBnode SBnode; typedef struct { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SBnodeLoad; typedef struct { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 5abda69aa8..f61073c9a5 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -224,6 +224,8 @@ int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); + /** * Destroy catalog and relase all resources diff --git a/include/libs/function/function.h b/include/libs/function/function.h index eafe64c294..68fc3be617 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -166,6 +166,7 @@ typedef struct SInputColumnInfoData { SColumnInfoData *pPTS; // primary timestamp column SColumnInfoData **pData; SColumnDataAgg **pColumnDataAgg; + uint64_t uid; // table uid } SInputColumnInfoData; // sql function runtime context @@ -191,7 +192,7 @@ typedef struct SqlFunctionCtx { int16_t functionId; // function id char * pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; - SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param + SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param int64_t *ptsList; // corresponding timestamp array list SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ int32_t offset; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index cd49e88b2f..2fff819f54 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -119,12 +119,19 @@ typedef enum EFunctionType { struct SqlFunctionCtx; struct SResultRowEntryInfo; struct STimeWindow; +struct SCatalog; + +typedef struct SFmGetFuncInfoParam { + struct SCatalog* pCtg; + void *pRpc; + const SEpSet* pMgmtEps; +} SFmGetFuncInfoParam; int32_t fmFuncMgtInit(); void fmFuncMgtDestroy(); -int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); +int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len); diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 321ca13f0e..a47bf4caee 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -48,6 +48,9 @@ extern "C" { (NULL == cell1 ? (node1 = NULL, false) : (node1 = cell1->pNode, true)), (NULL == cell2 ? (node2 = NULL, false) : (node2 = cell2->pNode, true)), (node1 != NULL && node2 != NULL); \ cell1 = cell1->pNext, cell2 = cell2->pNext) +#define REPLACE_LIST1_NODE(newNode) cell1->pNode = (SNode*)(newNode) +#define REPLACE_LIST2_NODE(newNode) cell2->pNode = (SNode*)(newNode) + #define FOREACH_FOR_REWRITE(node, list) \ for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 01e03a983d..b4a290cbfc 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -155,7 +155,6 @@ typedef struct SLogicSubplan { typedef struct SQueryLogicPlan { ENodeType type; - int32_t totalLevel; SNodeList* pTopSubplans; } SQueryLogicPlan; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 93282f658a..143224d8f5 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -229,10 +229,10 @@ typedef struct SFillNode { typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; - SNodeList* pProjectionList; // SNode + SNodeList* pProjectionList; SNode* pFromTable; SNode* pWhere; - SNodeList* pPartitionByList; // SNode + SNodeList* pPartitionByList; SNode* pWindow; SNodeList* pGroupByList; // SGroupingSetNode SNode* pHaving; @@ -245,12 +245,14 @@ typedef struct SSelectStmt { } SSelectStmt; typedef enum ESetOperatorType { - SET_OP_TYPE_UNION_ALL = 1 + SET_OP_TYPE_UNION_ALL = 1, + SET_OP_TYPE_UNION } ESetOperatorType; typedef struct SSetOperator { ENodeType type; // QUERY_NODE_SET_OPERATOR ESetOperatorType opType; + SNodeList* pProjectionList; SNode* pLeft; SNode* pRight; SNodeList* pOrderByList; // SOrderByExprNode diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index bb550e75e8..2b86b61cdd 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -51,6 +51,10 @@ typedef struct STableComInfo { typedef struct SIndexMeta { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SIndexMeta; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 3e2f596784..ab26cfc155 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -54,12 +54,13 @@ typedef struct { uint16_t clientPort; SRpcMsg rpcMsg; int32_t rspLen; - void *pRsp; - void *pNode; + void * pRsp; + void * pNode; } SNodeMsg; typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); +typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *); typedef struct SRpcInit { uint16_t localPort; // local port @@ -80,22 +81,25 @@ typedef struct SRpcInit { RpcCfp cfp; // call back to retrieve the client auth info, for server app only - RpcAfp afp;; + RpcAfp afp; + + // user defined retry func + RpcRfp rfp; void *parent; } SRpcInit; typedef struct { - void *val; + void *val; int32_t (*clone)(void *src, void **dst); - void (*freeFunc)(const void *arg); + void (*freeFunc)(const void *arg); } SRpcCtxVal; typedef struct { - int32_t msgType; - void *val; + int32_t msgType; + void * val; int32_t (*clone)(void *src, void **dst); - void (*freeFunc)(const void *arg); + void (*freeFunc)(const void *arg); } SRpcBrokenlinkVal; typedef struct { diff --git a/include/os/os.h b/include/os/os.h index 3ea94d0094..86abcc15f5 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -22,7 +22,6 @@ extern "C" { #include #include -#include #include @@ -51,6 +50,15 @@ extern "C" { #include #include #endif +#else + +#include +#ifndef TD_USE_WINSOCK +#include +#else +#include +#endif + #endif diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index 5d45ced3ed..e2a122a0fe 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_add_fetch_ptr(void *ptr, int32_t val); +void* atomic_add_fetch_ptr(void *ptr, void *val); int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_add_ptr(void *ptr, int32_t val); +void* atomic_fetch_add_ptr(void *ptr, void *val); int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val); -void* atomic_sub_fetch_ptr(void *ptr, int32_t val); +void* atomic_sub_fetch_ptr(void *ptr, void *val); int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); -void* atomic_fetch_sub_ptr(void *ptr, int32_t val); +void* atomic_fetch_sub_ptr(void *ptr, void *val); int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); diff --git a/include/os/osDef.h b/include/os/osDef.h index 492df1047c..8da0c2c00c 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -191,7 +191,7 @@ extern "C" { #define threadlocal __declspec( thread ) #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define PRIzu "ld" #else #define PRIzu "zu" diff --git a/include/os/osDir.h b/include/os/osDir.h index e7da54bf54..b549acde37 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr; void taosRemoveDir(const char *dirname); -bool taosDirExist(char *dirname); +bool taosDirExist(const char *dirname); int32_t taosMkDir(const char *dirname); int32_t taosMulMkDir(const char *dirname); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); -int32_t taosRealPath(char *dirname, int32_t maxlen); +int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen); bool taosIsDir(const char *dirname); char* taosDirName(char *dirname); char* taosDirEntryBaseName(char *dirname); @@ -50,7 +50,7 @@ TdDirPtr taosOpenDir(const char *dirname); TdDirEntryPtr taosReadDir(TdDirPtr pDir); bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry); char* taosGetDirEntryName(TdDirEntryPtr pDirEntry); -int32_t taosCloseDir(TdDirPtr pDir); +int32_t taosCloseDir(TdDirPtr *ppDir); #ifdef __cplusplus } diff --git a/include/os/osEnv.h b/include/os/osEnv.h index 14d50858b7..a3f92a0b29 100644 --- a/include/os/osEnv.h +++ b/include/os/osEnv.h @@ -34,6 +34,7 @@ extern int64_t tsOpenMax; extern int64_t tsStreamMax; extern float tsNumOfCores; extern int64_t tsTotalMemoryKB; +extern char* tsProcPath; extern char configDir[]; extern char tsDataDir[]; diff --git a/include/os/osFile.h b/include/os/osFile.h index 36ca6fb8bb..b364d233ef 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -93,7 +93,6 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size); -void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length); bool taosValidFile(TdFilePtr pFile); int32_t taosGetErrorFile(TdFilePtr pFile); diff --git a/include/os/osLz4.h b/include/os/osLz4.h index 09aaa553ac..5bb33242d4 100644 --- a/include/os/osLz4.h +++ b/include/os/osLz4.h @@ -20,7 +20,7 @@ extern "C" { #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS int32_t BUILDIN_CLZL(uint64_t val); int32_t BUILDIN_CLZ(uint32_t val); int32_t BUILDIN_CTZL(uint64_t val); diff --git a/include/os/osMath.h b/include/os/osMath.h index 0b5e4bd93c..cabb821844 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -23,7 +23,7 @@ extern "C" { #define TPOW2(x) ((x) * (x)) #define TABS(x) ((x) > 0 ? (x) : -(x)) -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define TSWAP(a, b, c) \ do { \ @@ -33,6 +33,7 @@ extern "C" { } while (0) #define TMAX(a, b) (((a) > (b)) ? (a) : (b)) #define TMIN(a, b) (((a) < (b)) ? (a) : (b)) + #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc))) #else diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 7af8dd37bf..a47239089a 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -19,32 +19,53 @@ // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC - #define socket SOCKET_FUNC_TAOS_FORBID - #define bind BIND_FUNC_TAOS_FORBID - #define listen LISTEN_FUNC_TAOS_FORBID - #define accept ACCEPT_FUNC_TAOS_FORBID - #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID - #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID - #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID - #define inet_addr INET_ADDR_FUNC_TAOS_FORBID - #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID + #define socket SOCKET_FUNC_TAOS_FORBID + #define bind BIND_FUNC_TAOS_FORBID + #define listen LISTEN_FUNC_TAOS_FORBID + #define accept ACCEPT_FUNC_TAOS_FORBID + #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID + #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID + #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID + #define inet_addr INET_ADDR_FUNC_TAOS_FORBID + #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID #endif #if defined(WINDOWS) - #include "winsock2.h" - #include - #include - #include -#else - #include - #include + #if BYTE_ORDER == LITTLE_ENDIAN + #include + #define htobe16(x) _byteswap_ushort(x) + #define htole16(x) (x) + #define be16toh(x) _byteswap_ushort(x) + #define le16toh(x) (x) + + #define htobe32(x) _byteswap_ulong(x) + #define htole32(x) (x) + #define be32toh(x) _byteswap_ulong(x) + #define le32toh(x) (x) + + #define htobe64(x) _byteswap_uint64(x) + #define htole64(x) (x) + #define be64toh(x) _byteswap_uint64(x) + #define le64toh(x) (x) + #else + #error byte order not supported + #endif - #if defined(_TD_DARWIN_64) - #include - #else - #include - #include - #endif + #define __BYTE_ORDER BYTE_ORDER + #define __BIG_ENDIAN BIG_ENDIAN + #define __LITTLE_ENDIAN LITTLE_ENDIAN + #define __PDP_ENDIAN PDP_ENDIAN + +#else + #include + #include + + #if defined(_TD_DARWIN_64) + #include + #else + #include + #include + #endif #endif #ifdef __cplusplus @@ -52,14 +73,10 @@ extern "C" { #endif #if defined(WINDOWS) - #define htobe64 htonll -#endif - -#if defined(WINDOWS) + typedef int socklen_t; #define TAOS_EPOLL_WAIT_TIME 100 typedef SOCKET eventfd_t; #define eventfd(a, b) -1 - typedef SOCKET EpollFd; #define EpollClose(pollFd) epoll_close(pollFd) #ifndef EPOLLWAKEUP #define EPOLLWAKEUP (1u << 29) @@ -79,31 +96,29 @@ extern "C" { #if defined(_TD_DARWIN_64) // #define htobe64 htonll -# include +#include -# define htobe16(x) OSSwapHostToBigInt16(x) -# define htole16(x) OSSwapHostToLittleInt16(x) -# define be16toh(x) OSSwapBigToHostInt16(x) -# define le16toh(x) OSSwapLittleToHostInt16(x) +#define htobe16(x) OSSwapHostToBigInt16(x) +#define htole16(x) OSSwapHostToLittleInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define le16toh(x) OSSwapLittleToHostInt16(x) -# define htobe32(x) OSSwapHostToBigInt32(x) -# define htole32(x) OSSwapHostToLittleInt32(x) -# define be32toh(x) OSSwapBigToHostInt32(x) -# define le32toh(x) OSSwapLittleToHostInt32(x) +#define htobe32(x) OSSwapHostToBigInt32(x) +#define htole32(x) OSSwapHostToLittleInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define le32toh(x) OSSwapLittleToHostInt32(x) -# define htobe64(x) OSSwapHostToBigInt64(x) -# define htole64(x) OSSwapHostToLittleInt64(x) -# define be64toh(x) OSSwapBigToHostInt64(x) -# define le64toh(x) OSSwapLittleToHostInt64(x) +#define htobe64(x) OSSwapHostToBigInt64(x) +#define htole64(x) OSSwapHostToLittleInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) +#define le64toh(x) OSSwapLittleToHostInt64(x) -# define __BYTE_ORDER BYTE_ORDER -# define __BIG_ENDIAN BIG_ENDIAN -# define __LITTLE_ENDIAN LITTLE_ENDIAN -# define __PDP_ENDIAN PDP_ENDIAN +#define __BYTE_ORDER BYTE_ORDER +#define __BIG_ENDIAN BIG_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#define __PDP_ENDIAN PDP_ENDIAN #endif -#define TAOS_EPOLL_WAIT_TIME 500 - typedef int32_t SocketFd; typedef SocketFd EpollFd; @@ -122,7 +137,7 @@ typedef struct TdEpoll *TdEpollPtr; int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen); int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); @@ -139,14 +154,15 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); +void taosWinSocketInit(); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); -TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen); +TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); -int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen); +int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen); void taosBlockSIGPIPE(); uint32_t taosGetIpv4FromFqdn(const char *); diff --git a/include/os/osString.h b/include/os/osString.h index 66d69a849c..a4100652c3 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -39,7 +39,7 @@ typedef int32_t TdUcs4; #define wchar_t WCHAR_T_TYPE_TAOS_FORBID #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define tstrdup(str) _strdup(str) #else #define tstrdup(str) strdup(str) diff --git a/include/os/osThread.h b/include/os/osThread.h index 18c1b290ab..6252a0cb60 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -22,9 +22,12 @@ extern "C" { #endif +#ifndef WINDOWS #ifndef __USE_XOPEN2K +#define TD_USE_SPINLOCK_AS_MUTEX typedef pthread_mutex_t pthread_spinlock_t; #endif +#endif typedef pthread_t TdThread; typedef pthread_spinlock_t TdThreadSpinlock; @@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey; #define taosThreadCleanupPush pthread_cleanup_push #define taosThreadCleanupPop pthread_cleanup_pop + +#ifdef WINDOWS +#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1) +#else +#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +#endif + // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC @@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock); int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock); int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock); void taosThreadTestCancel(void); -int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset); -int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig); +void taosThreadClear(TdThread *thread); #ifdef __cplusplus } diff --git a/include/os/osTime.h b/include/os/osTime.h index fd431f6df8..b9e407cbcf 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -20,8 +20,6 @@ extern "C" { #endif -#include - // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC @@ -34,7 +32,7 @@ extern "C" { #define mktime MKTIME_FUNC_TAOS_FORBID #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define CLOCK_REALTIME 0 @@ -58,6 +56,8 @@ extern "C" { int32_t taosGetTimeOfDay(struct timeval *tv); +int32_t taosClockGetTime(int clock_id, struct timespec *pTS); + //@return timestamp in second int32_t taosGetTimestampSec(); @@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() { //@return timestamp in nanosecond static FORCE_INLINE int64_t taosGetTimestampNs() { struct timespec systemTime = {0}; - clock_gettime(CLOCK_REALTIME, &systemTime); + taosClockGetTime(CLOCK_REALTIME, &systemTime); return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec; } diff --git a/include/util/talgo.h b/include/util/talgo.h index ebddce62a8..3ce2652608 100644 --- a/include/util/talgo.h +++ b/include/util/talgo.h @@ -82,7 +82,7 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size * @return */ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); + __ext_compar_fn_t compar, char* buf, bool maxroot); /** * sort heap to make sure it is a max/min root heap @@ -98,7 +98,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const * @return */ void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, - const void *parswap, __ext_swap_fn_t swap, bool maxroot); + bool maxroot); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e04ff2730a..215e83c7d4 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -614,6 +614,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG TAOS_DEF_ERROR_CODE(0, 0x2631) #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632) #define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633) +#define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL TAOS_DEF_ERROR_CODE(0, 0x2634) //planner #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) diff --git a/include/util/tencode.h b/include/util/tencode.h index 7c877ae428..11357ba533 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -236,7 +236,8 @@ static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) { union { uint32_t ui; float f; - } v = {.f = val}; + } v; + v.f = val; return tEncodeU32(pEncoder, v.ui); } @@ -245,7 +246,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) { union { uint64_t ui; double d; - } v = {.d = val}; + } v; + v.d = val; return tEncodeU64(pEncoder, v.ui); } diff --git a/include/util/tjson.h b/include/util/tjson.h index 28bcb3cfc6..286a7e65fc 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -22,6 +22,9 @@ extern "C" { #endif +#ifdef WINDOWS +#define tjsonGetNumberValue(pJson, pName, val) -1 +#else #define tjsonGetNumberValue(pJson, pName, val) \ ({ \ uint64_t _tmp = 0; \ @@ -29,6 +32,7 @@ extern "C" { val = _tmp; \ _code; \ }) +#endif typedef void SJson; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 00a58e16a7..814caf330a 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -219,6 +219,7 @@ void doSetOneRowPtr(SReqResultInfo* pResultInfo); void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4); void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +void doFreeReqResultInfo(SReqResultInfo* pResInfo); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 219257ba74..04e9c3be9a 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -60,6 +60,7 @@ typedef struct SStmtBindInfo { } SStmtBindInfo; typedef struct SStmtExecInfo { + int32_t affectedRows; SRequestObj* pRequest; SHashObj* pVgHash; SHashObj* pBlockHash; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 0b75e3a30b..9a34334466 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -30,15 +30,15 @@ #define TSC_VAR_RELEASED 0 SAppInfo appInfo; -int32_t clientReqRefPool = -1; +int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; -static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; -volatile int32_t tscInitRes = 0; +static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; +volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id); - + assert(pTscObj != NULL); // connection has been released already, abort creating request. @@ -49,8 +49,8 @@ static void registerRequest(SRequestObj *pRequest) { if (pTscObj->pAppInfo) { SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary; - int32_t total = atomic_add_fetch_64((int64_t*)&pSummary->totalRequests, 1); - int32_t currentInst = atomic_add_fetch_64((int64_t*)&pSummary->currentRequests, 1); + int32_t total = atomic_add_fetch_64((int64_t *)&pSummary->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); @@ -60,16 +60,16 @@ static void registerRequest(SRequestObj *pRequest) { static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj * pTscObj = pRequest->pTscObj; + STscObj *pTscObj = pRequest->pTscObj; SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; - int32_t currentInst = atomic_sub_fetch_64((int64_t*)&pActivity->currentRequests, 1); + int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 " ms, current:%d, app current:%d", - pRequest->self, pTscObj->id, pRequest->requestId, duration/1000, num, currentInst); + pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); releaseTscObj(pTscObj->id); } @@ -109,12 +109,12 @@ void *openTransporter(const char *user, const char *auth, int32_t numOfThread) { } void closeAllRequests(SHashObj *pRequests) { - void *pIter = taosHashIterate(pRequests, NULL); + void *pIter = taosHashIterate(pRequests, NULL); while (pIter != NULL) { int64_t *rid = pIter; - releaseRequest(*rid); - + releaseRequest(*rid); + pIter = taosHashIterate(pRequests, pIter); } } @@ -144,7 +144,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } - + pObj->pAppInfo = pAppInfo; tstrncpy(pObj->user, user, sizeof(pObj->user)); memcpy(pObj->pass, auth, TSDB_PASSWORD_LEN); @@ -160,13 +160,9 @@ void *createTscObj(const char *user, const char *auth, const char *db, SAppInstI return pObj; } -STscObj *acquireTscObj(int64_t rid) { - return (STscObj *)taosAcquireRef(clientConnRefPool, rid); -} +STscObj *acquireTscObj(int64_t rid) { return (STscObj *)taosAcquireRef(clientConnRefPool, rid); } -int32_t releaseTscObj(int64_t rid) { - return taosReleaseRef(clientConnRefPool, rid); -} +int32_t releaseTscObj(int64_t rid) { return taosReleaseRef(clientConnRefPool, rid); } void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t type) { assert(pObj != NULL); @@ -190,11 +186,11 @@ void *createRequest(STscObj *pObj, __taos_async_fn_t fp, void *param, int32_t ty tsem_init(&pRequest->body.rspSem, 0, 0); registerRequest(pRequest); - + return pRequest; } -static void doFreeReqResultInfo(SReqResultInfo *pResInfo) { +void doFreeReqResultInfo(SReqResultInfo *pResInfo) { taosMemoryFreeClear(pResInfo->pRspMsg); taosMemoryFreeClear(pResInfo->length); taosMemoryFreeClear(pResInfo->row); @@ -217,7 +213,7 @@ static void doDestroyRequest(void *p) { assert(RID_VALID(pRequest->self)); taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); - + taosMemoryFreeClear(pRequest->msgBuf); taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFreeClear(pRequest->pDb); @@ -244,14 +240,9 @@ void destroyRequest(SRequestObj *pRequest) { taosRemoveRef(clientReqRefPool, pRequest->self); } -SRequestObj *acquireRequest(int64_t rid) { - return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); -} - -int32_t releaseRequest(int64_t rid) { - return taosReleaseRef(clientReqRefPool, rid); -} +SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); } +int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); } void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. @@ -380,7 +371,7 @@ uint64_t generateRequestId() { } uint64_t id = 0; - + while (true) { int64_t ts = taosGetTimestampMs(); uint64_t pid = taosGetPId(); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index e730f0a34a..27efcee76e 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -135,6 +135,16 @@ void taos_free_result(TAOS_RES *res) { if (TD_RES_QUERY(res)) { SRequestObj *pRequest = (SRequestObj *)res; destroyRequest(pRequest); + } else if (TD_RES_TMQ(res)) { + SMqRspObj *pRsp = (SMqRspObj *)res; + if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); + if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); + if (pRsp->rsp.blockSchema) taosArrayDestroy(pRsp->rsp.blockSchema); + if (pRsp->rsp.blockTbName) taosArrayDestroy(pRsp->rsp.blockTbName); + if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags); + if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema); + pRsp->resInfo.pRspMsg = NULL; + doFreeReqResultInfo(&pRsp->resInfo); } } @@ -622,6 +632,10 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) { return stmtSetTbName(stmt, name); } +int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { + return taos_stmt_set_tbname(stmt, name); +} + int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { if (stmt == NULL || bind == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 8a5134e02a..0972ff3477 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -486,7 +486,8 @@ int stmtExec(TAOS_STMT *stmt) { STMT_ERR_JRET(pStmt->exec.pRequest->code); - pStmt->affectedRows += taos_affected_rows(pStmt->exec.pRequest); + pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest); + pStmt->affectedRows += pStmt->exec.affectedRows; _return: diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 53b0fec8ff..8e8c62cd94 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -662,7 +662,7 @@ TEST(testCase, agg_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "select count(*) from tu"); + pRes = taos_query(pConn, "select now() from m1"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 63a04a8813..e01b113a04 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -5,8 +5,8 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include/common" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" IF(${TD_WINDOWS}) - PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32" - PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex" + PRIVATE "${TD_SOURCE_DIR}/contrib/pthread" + PRIVATE "${TD_SOURCE_DIR}/contrib/msvcregex" ENDIF () ) target_link_libraries( diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f2a5a98d96..2ed08ac81f 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -110,7 +110,7 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { for (int i = 0; i < numOfCols; i++) { col_type_t type = 0; - int8_t sma = TSDB_BSMA_TYPE_NONE; + int8_t sma = 0; col_id_t colId = 0; col_bytes_t bytes = 0; buf = taosDecodeFixedI8(buf, &type); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3622390f45..0ea196f1a1 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -434,6 +434,15 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { for (int8_t i = 0; i < param->nFuncIds; ++i) { tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]); } + tlen += taosEncodeFixedI32(buf, param->qmsg1Len); + if (param->qmsg1Len > 0) { + tlen += taosEncodeString(buf, param->qmsg1); + } + + tlen += taosEncodeFixedI32(buf, param->qmsg2Len); + if (param->qmsg2Len > 0) { + tlen += taosEncodeString(buf, param->qmsg2); + } } break; case TD_CHILD_TABLE: @@ -496,18 +505,25 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name); } if (pReq->rollup) { - pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam)); + pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam)); SRSmaParam *param = pReq->stbCfg.pRSmaParam; buf = taosDecodeBinaryTo(buf, (void *)¶m->xFilesFactor, sizeof(param->xFilesFactor)); buf = taosDecodeFixedI32(buf, ¶m->delay); buf = taosDecodeFixedI8(buf, ¶m->nFuncIds); if (param->nFuncIds > 0) { - param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t)); + param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t)); for (int8_t i = 0; i < param->nFuncIds; ++i) { buf = taosDecodeFixedI32(buf, param->pFuncIds + i); } - } else { - param->pFuncIds = NULL; + } + buf = taosDecodeFixedI32(buf, ¶m->qmsg1Len); + if (param->qmsg1Len > 0) { + buf = taosDecodeString(buf, ¶m->qmsg1); + } + + buf = taosDecodeFixedI32(buf, ¶m->qmsg2Len); + if (param->qmsg2Len > 0) { + buf = taosDecodeString(buf, ¶m->qmsg2); } } else { pReq->stbCfg.pRSmaParam = NULL; @@ -828,6 +844,27 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) { taosArrayDestroy(pReq->pFields); pReq->pFields = NULL; } +int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) return -1; + + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) { SCoder encoder = {0}; @@ -1624,6 +1661,7 @@ int32_t tSerializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq * if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1; + if (tEncodeI8(&encoder, pReq->ignoreCodeComment) < 0) return -1; if (pReq->numOfFuncs != (int32_t)taosArrayGetSize(pReq->pFuncNames)) return -1; for (int32_t i = 0; i < pReq->numOfFuncs; ++i) { @@ -1644,6 +1682,7 @@ int32_t tDeserializeSRetrieveFuncReq(void *buf, int32_t bufLen, SRetrieveFuncReq if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1; + if (tDecodeI8(&decoder, (int8_t *)&pReq->ignoreCodeComment) < 0) return -1; pReq->pFuncNames = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN); if (pReq->pFuncNames == NULL) return -1; @@ -1681,8 +1720,12 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * if (tEncodeI64(&encoder, pInfo->signature) < 0) return -1; if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; - if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; - if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; + if (pInfo->codeSize) { + if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + } + if (pInfo->commentSize) { + if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; + } } tEndEncode(&encoder); @@ -1713,15 +1756,23 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp if (tDecodeI64(&decoder, &fInfo.signature) < 0) return -1; if (tDecodeI32(&decoder, &fInfo.codeSize) < 0) return -1; if (tDecodeI32(&decoder, &fInfo.commentSize) < 0) return -1; - fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize); - fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize); - if (fInfo.pCode == NULL || fInfo.pComment == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + if (fInfo.codeSize) { + fInfo.pCode = taosMemoryCalloc(1, fInfo.codeSize); + if (fInfo.pCode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1; + } + if (fInfo.commentSize) { + fInfo.pComment = taosMemoryCalloc(1, fInfo.commentSize); + if (fInfo.pComment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; } - if (tDecodeCStrTo(&decoder, fInfo.pCode) < 0) return -1; - if (tDecodeCStrTo(&decoder, fInfo.pComment) < 0) return -1; taosArrayPush(pRsp->pFuncInfos, &fInfo); } tEndDecode(&decoder); @@ -1730,12 +1781,20 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp return 0; } +void tFreeSFuncInfo(SFuncInfo *pInfo) { + if (NULL == pInfo) { + return; + } + + taosMemoryFree(pInfo->pCode); + taosMemoryFree(pInfo->pComment); +} + void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) { int32_t size = taosArrayGetSize(pRsp->pFuncInfos); for (int32_t i = 0; i < size; ++i) { SFuncInfo *pInfo = taosArrayGet(pRsp->pFuncInfos, i); - taosMemoryFree(pInfo->pCode); - taosMemoryFree(pInfo->pComment); + tFreeSFuncInfo(pInfo); } taosArrayDestroy(pRsp->pFuncInfos); } @@ -2403,7 +2462,6 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tStartEncode(&encoder) < 0) return -1; if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; - if (tEncodeI32(&encoder, pReq->type) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; tEndEncode(&encoder); @@ -2419,7 +2477,6 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; - if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; tEndDecode(&decoder); @@ -2674,6 +2731,10 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tStartEncode(&encoder) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTagSchema) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; @@ -2696,6 +2757,10 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tStartDecode(&decoder) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTagSchema) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index d60261dfca..fea6c4c891 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -167,7 +167,7 @@ TEST(testCase, Datablock_test) { printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); - SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0}; + SBlockOrderInfo order = { true, TSDB_ORDER_ASC, 0, NULL }; taosArrayPush(pOrderInfo, &order); blockDataSort(b, pOrderInfo); diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 3f5c22dd84..7e4505ec95 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -40,10 +40,12 @@ static void dmSetSignalHandle() { taosSetSignal(SIGTERM, dmStopDnode); taosSetSignal(SIGHUP, dmStopDnode); taosSetSignal(SIGINT, dmStopDnode); - taosSetSignal(SIGTSTP, dmStopDnode); taosSetSignal(SIGABRT, dmStopDnode); taosSetSignal(SIGBREAK, dmStopDnode); +#ifndef WINDOWS + taosSetSignal(SIGTSTP, dmStopDnode); taosSetSignal(SIGQUIT, dmStopDnode); +#endif if (!tsMultiProcess) { } else if (global.ntype == DNODE || global.ntype == NODE_END) { diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index dd221f8404..376f589acd 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -216,6 +216,126 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) { dmStopStatusThread(pWrapper->pDnode); } +static int32_t dmSpawnUdfd(SDnode *pDnode); + +void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { + dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); + uv_close((uv_handle_t*)process, NULL); + SDnode *pDnode = process->data; + SUdfdData *pData = &pDnode->udfdData; + if (atomic_load_8(&pData->stopping) != 0) { + dDebug("udfd process exit due to stopping"); + } else { + uv_close((uv_handle_t*)&pData->ctrlPipe, NULL); + dmSpawnUdfd(pDnode); + } +} + +static int32_t dmSpawnUdfd(SDnode *pDnode) { + dInfo("dnode start spawning udfd"); + uv_process_options_t options = {0}; + + char path[PATH_MAX] = {0}; + if (tsProcPath == NULL) { + path[0] = '.'; + } else { + strncpy(path, tsProcPath, strlen(tsProcPath)); + taosDirName(path); + } + strcat(path, "/udfd"); + char* argsUdfd[] = {path, "-c", configDir, NULL}; + options.args = argsUdfd; + options.file = path; + + options.exit_cb = dmUdfdExit; + SUdfdData *pData = &pDnode->udfdData; + uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1); + + uv_stdio_container_t child_stdio[3]; + child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE; + child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe; + child_stdio[1].flags = UV_IGNORE; + child_stdio[2].flags = UV_INHERIT_FD; + child_stdio[2].data.fd = 2; + options.stdio_count = 3; + options.stdio = child_stdio; + + char dnodeIdEnvItem[32] = {0}; + char thrdPoolSizeEnvItem[32] = {0}; + snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId); + float numCpuCores = 4; + taosGetCpuCores(&numCpuCores); + snprintf(thrdPoolSizeEnvItem,32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2); + char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL}; + options.env = envUdfd; + + int err = uv_spawn(&pData->loop, &pData->process, &options); + pData->process.data = (void*)pDnode; + + if (err != 0) { + dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); + } + return err; +} + +static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) { + if (!uv_is_closing(handle)) { + uv_close(handle, NULL); + } +} + +void dmWatchUdfd(void *args) { + SDnode *pDnode = args; + SUdfdData *pData = &pDnode->udfdData; + uv_loop_init(&pData->loop); + int32_t err = dmSpawnUdfd(pDnode); + atomic_store_32(&pData->spawnErr, err); + uv_barrier_wait(&pData->barrier); + uv_run(&pData->loop, UV_RUN_DEFAULT); + err = uv_loop_close(&pData->loop); + while (err == UV_EBUSY) { + uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL); + uv_run(&pData->loop, UV_RUN_DEFAULT); + err = uv_loop_close(&pData->loop); + } + return; +} + +int32_t dmStartUdfd(SDnode *pDnode) { + SUdfdData *pData = &pDnode->udfdData; + if (pData->startCalled) { + dInfo("dnode-mgmt start udfd already called"); + return 0; + } + uv_barrier_init(&pData->barrier, 2); + pData->stopping = 0; + uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); + uv_barrier_wait(&pData->barrier); + pData->startCalled = true; + pData->needCleanUp = true; + return pData->spawnErr; +} + +int32_t dmStopUdfd(SDnode *pDnode) { + dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d", + pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr); + SUdfdData *pData = &pDnode->udfdData; + if (!pData->needCleanUp) { + return 0; + } + atomic_store_8(&pData->stopping, 1); + + uv_barrier_destroy(&pData->barrier); + if (pData->spawnErr == 0) { + uv_process_kill(&pData->process, SIGINT); + } + uv_stop(&pData->loop); + uv_thread_join(&pData->thread); + + atomic_store_8(&pData->stopping, 0); + return 0; +} + static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt start to init"); SDnode *pDnode = pWrapper->pDnode; @@ -247,6 +367,10 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "dnode-transport", "initialized"); + if (dmStartUdfd(pDnode) != 0) { + dError("failed to start udfd"); + } + dInfo("dnode-mgmt is initialized"); return 0; } @@ -254,6 +378,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { static void dmCleanupMgmt(SMgmtWrapper *pWrapper) { dInfo("dnode-mgmt start to clean up"); SDnode *pDnode = pWrapper->pDnode; + dmStopUdfd(pDnode); dmStopWorker(pDnode); taosWLockLatch(&pDnode->data.latch); diff --git a/source/dnode/mgmt/interface/inc/dmDef.h b/source/dnode/mgmt/interface/inc/dmDef.h index e6537dcf73..4f4a2ed349 100644 --- a/source/dnode/mgmt/interface/inc/dmDef.h +++ b/source/dnode/mgmt/interface/inc/dmDef.h @@ -16,6 +16,7 @@ #ifndef _TD_DM_DEF_H_ #define _TD_DM_DEF_H_ +#include "uv.h" #include "dmLog.h" #include "cJSON.h" @@ -142,6 +143,18 @@ typedef struct { char desc[TSDB_STEP_DESC_LEN]; } SStartupInfo; +typedef struct SUdfdData { + bool startCalled; + bool needCleanUp; + uv_loop_t loop; + uv_thread_t thread; + uv_barrier_t barrier; + uv_process_t process; + int spawnErr; + int8_t stopping; + uv_pipe_t ctrlPipe; +} SUdfdData; + typedef struct SDnode { EDndProcType ptype; EDndNodeType ntype; @@ -150,6 +163,7 @@ typedef struct SDnode { SStartupInfo startup; SDnodeTrans trans; SDnodeData data; + SUdfdData udfdData; TdThreadMutex mutex; SMgmtWrapper wrappers[NODE_END]; } SDnode; diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 9b4c479865..cc32f047af 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -89,7 +89,6 @@ int32_t Testbase::SendShowReq(int8_t showType, const char *tb, const char* db) { } SRetrieveTableReq retrieveReq = {0}; - retrieveReq.type = showType; strcpy(retrieveReq.db, db); strcpy(retrieveReq.tb, tb); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 12710f0d4c..59a00f0eaf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -391,7 +391,6 @@ typedef struct { int16_t numOfColumns; int32_t rowSize; int32_t numOfRows; - int32_t payloadLen; void* pIter; SMnode* pMnode; STableMetaRsp* pMeta; @@ -418,82 +417,6 @@ typedef struct { char payload[]; } SSysTableRetrieveObj; -typedef struct { - int32_t vgId; // -1 for unassigned - int32_t status; - int32_t epoch; - SEpSet epSet; - int64_t oldConsumerId; - int64_t consumerId; // -1 for unassigned - char* qmsg; -} SMqConsumerEp; - -static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pConsumerEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pConsumerEp->vgId); - tlen += taosEncodeFixedI32(buf, pConsumerEp->status); - tlen += taosEncodeFixedI32(buf, pConsumerEp->epoch); - tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); - tlen += taosEncodeFixedI64(buf, pConsumerEp->oldConsumerId); - tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); - tlen += taosEncodeString(buf, pConsumerEp->qmsg); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { - buf = taosDecodeFixedI32(buf, &pConsumerEp->vgId); - buf = taosDecodeFixedI32(buf, &pConsumerEp->status); - buf = taosDecodeFixedI32(buf, &pConsumerEp->epoch); - buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); - buf = taosDecodeFixedI64(buf, &pConsumerEp->oldConsumerId); - buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = taosDecodeString(buf, &pConsumerEp->qmsg); - return buf; -} - -static FORCE_INLINE void tDeleteSMqConsumerEp(SMqConsumerEp* pConsumerEp) { - if (pConsumerEp) { - taosMemoryFreeClear(pConsumerEp->qmsg); - } -} - -typedef struct { - int64_t consumerId; - SArray* vgInfo; // SArray -} SMqSubConsumer; - -static FORCE_INLINE int32_t tEncodeSMqSubConsumer(void** buf, const SMqSubConsumer* pConsumer) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); - int32_t sz = taosArrayGetSize(pConsumer->vgInfo); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp* pCEp = taosArrayGet(pConsumer->vgInfo, i); - tlen += tEncodeSMqConsumerEp(buf, pCEp); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubConsumer(void** buf, SMqSubConsumer* pConsumer) { - int32_t sz; - buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->vgInfo = taosArrayInit(sz, sizeof(SMqConsumerEp)); - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp consumerEp; - buf = tDecodeSMqConsumerEp(buf, &consumerEp); - taosArrayPush(pConsumer->vgInfo, &consumerEp); - } - return buf; -} - -static FORCE_INLINE void tDeleteSMqSubConsumer(SMqSubConsumer* pSubConsumer) { - if (pSubConsumer->vgInfo) { - taosArrayDestroyEx(pSubConsumer->vgInfo, (void (*)(void*))tDeleteSMqConsumerEp); - pSubConsumer->vgInfo = NULL; - } -} - typedef struct { char key[TSDB_PARTITION_KEY_LEN]; int64_t offset; @@ -512,147 +435,21 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset) return buf; } -#if 0 typedef struct { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - int32_t status; - int32_t vgNum; - SArray* consumers; // SArray - SArray* lostConsumers; // SArray - SArray* unassignedVg; // SArray -} SMqSubscribeObj; - -static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { - SMqSubscribeObj* pSub = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); - if (pSub == NULL) { - return NULL; - } - - pSub->consumers = taosArrayInit(0, sizeof(SMqSubConsumer)); - if (pSub->consumers == NULL) { - goto _err; - } - - pSub->lostConsumers = taosArrayInit(0, sizeof(SMqSubConsumer)); - if (pSub->lostConsumers == NULL) { - goto _err; - } - - pSub->unassignedVg = taosArrayInit(0, sizeof(SMqConsumerEp)); - if (pSub->unassignedVg == NULL) { - goto _err; - } - - pSub->key[0] = 0; - pSub->vgNum = 0; - pSub->status = 0; - - return pSub; - -_err: - taosMemoryFreeClear(pSub->consumers); - taosMemoryFreeClear(pSub->lostConsumers); - taosMemoryFreeClear(pSub->unassignedVg); - taosMemoryFreeClear(pSub); - return NULL; -} - -static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pSub->key); - tlen += taosEncodeFixedI32(buf, pSub->vgNum); - tlen += taosEncodeFixedI32(buf, pSub->status); - int32_t sz; - - sz = taosArrayGetSize(pSub->consumers); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->consumers, i); - tlen += tEncodeSMqSubConsumer(buf, pSubConsumer); - } - - sz = taosArrayGetSize(pSub->lostConsumers); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer* pSubConsumer = taosArrayGet(pSub->lostConsumers, i); - tlen += tEncodeSMqSubConsumer(buf, pSubConsumer); - } - - sz = taosArrayGetSize(pSub->unassignedVg); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp* pCEp = taosArrayGet(pSub->unassignedVg, i); - tlen += tEncodeSMqConsumerEp(buf, pCEp); - } - - return tlen; -} - -static FORCE_INLINE void* tDecodeSubscribeObj(void* buf, SMqSubscribeObj* pSub) { - buf = taosDecodeStringTo(buf, pSub->key); - buf = taosDecodeFixedI32(buf, &pSub->vgNum); - buf = taosDecodeFixedI32(buf, &pSub->status); - - int32_t sz; - - buf = taosDecodeFixedI32(buf, &sz); - pSub->consumers = taosArrayInit(sz, sizeof(SMqSubConsumer)); - if (pSub->consumers == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer subConsumer = {0}; - buf = tDecodeSMqSubConsumer(buf, &subConsumer); - taosArrayPush(pSub->consumers, &subConsumer); - } - - buf = taosDecodeFixedI32(buf, &sz); - pSub->lostConsumers = taosArrayInit(sz, sizeof(SMqSubConsumer)); - if (pSub->lostConsumers == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubConsumer subConsumer = {0}; - buf = tDecodeSMqSubConsumer(buf, &subConsumer); - taosArrayPush(pSub->lostConsumers, &subConsumer); - } - - buf = taosDecodeFixedI32(buf, &sz); - pSub->unassignedVg = taosArrayInit(sz, sizeof(SMqConsumerEp)); - if (pSub->unassignedVg == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp consumerEp = {0}; - buf = tDecodeSMqConsumerEp(buf, &consumerEp); - taosArrayPush(pSub->unassignedVg, &consumerEp); - } - return buf; -} - -static FORCE_INLINE void tDeleteSMqSubscribeObj(SMqSubscribeObj* pSub) { - if (pSub->consumers) { - // taosArrayDestroyEx(pSub->consumers, (void (*)(void*))tDeleteSMqSubConsumer); - // taosArrayDestroy(pSub->consumers); - pSub->consumers = NULL; - } - - if (pSub->unassignedVg) { - // taosArrayDestroyEx(pSub->unassignedVg, (void (*)(void*))tDeleteSMqConsumerEp); - // taosArrayDestroy(pSub->unassignedVg); - pSub->unassignedVg = NULL; - } -} -#endif - -typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + // TODO: use subDbUid int64_t dbUid; + int64_t subDbUid; int32_t version; + int8_t subType; // db or table + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; SRWLatch lock; int32_t sqlLen; int32_t astLen; @@ -662,79 +459,6 @@ typedef struct { SSchemaWrapper schema; } SMqTopicObj; -#if 0 -typedef struct { - int64_t consumerId; - int64_t connId; - SRWLatch lock; - char cgroup[TSDB_CGROUP_LEN]; - SArray* currentTopics; // SArray - SArray* recentRemovedTopics; // SArray - int32_t epoch; - // stat - int64_t pollCnt; - // status - int32_t status; - // heartbeat from the consumer reset hbStatus to 0 - // each checkConsumerAlive msg add hbStatus by 1 - // if checkConsumerAlive > CONSUMER_REBALANCE_CNT, mask to lost - int32_t hbStatus; -} SMqConsumerObj; - -static FORCE_INLINE int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer) { - int32_t sz; - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); - tlen += taosEncodeFixedI64(buf, pConsumer->connId); - tlen += taosEncodeFixedI32(buf, pConsumer->epoch); - tlen += taosEncodeFixedI64(buf, pConsumer->pollCnt); - tlen += taosEncodeFixedI32(buf, pConsumer->status); - tlen += taosEncodeString(buf, pConsumer->cgroup); - - sz = taosArrayGetSize(pConsumer->currentTopics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - char* topic = taosArrayGetP(pConsumer->currentTopics, i); - tlen += taosEncodeString(buf, topic); - } - - sz = taosArrayGetSize(pConsumer->recentRemovedTopics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - char* topic = taosArrayGetP(pConsumer->recentRemovedTopics, i); - tlen += taosEncodeString(buf, topic); - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqConsumerObj(void* buf, SMqConsumerObj* pConsumer) { - int32_t sz; - buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); - buf = taosDecodeFixedI64(buf, &pConsumer->connId); - buf = taosDecodeFixedI32(buf, &pConsumer->epoch); - buf = taosDecodeFixedI64(buf, &pConsumer->pollCnt); - buf = taosDecodeFixedI32(buf, &pConsumer->status); - buf = taosDecodeStringTo(buf, pConsumer->cgroup); - - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->currentTopics = taosArrayInit(sz, sizeof(void*)); - for (int32_t i = 0; i < sz; i++) { - char* topic; - buf = taosDecodeString(buf, &topic); - taosArrayPush(pConsumer->currentTopics, &topic); - } - - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->recentRemovedTopics = taosArrayInit(sz, sizeof(void*)); - for (int32_t i = 0; i < sz; i++) { - char* topic; - buf = taosDecodeString(buf, &topic); - taosArrayPush(pConsumer->recentRemovedTopics, &topic); - } - return buf; -} -#endif - enum { CONSUMER_UPDATE__TOUCH = 1, CONSUMER_UPDATE__ADD, @@ -753,12 +477,9 @@ typedef struct { int32_t hbStatus; // lock is used for topics update SRWLatch lock; - SArray* currentTopics; // SArray -#if 0 - SArray* waitingRebTopics; // SArray -#endif - SArray* rebNewTopics; // SArray - SArray* rebRemovedTopics; // SArray + SArray* currentTopics; // SArray + SArray* rebNewTopics; // SArray + SArray* rebRemovedTopics; // SArray } SMqConsumerObj; SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); @@ -769,8 +490,7 @@ void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer typedef struct { int32_t vgId; char* qmsg; - // char topic[TSDB_TOPIC_FNAME_LEN]; - SEpSet epSet; + SEpSet epSet; } SMqVgEp; SMqVgEp* tCloneSMqVgEp(const SMqVgEp* pVgEp); @@ -792,7 +512,14 @@ typedef struct { char key[TSDB_SUBSCRIBE_KEY_LEN]; SRWLatch lock; int32_t vgNum; + int8_t subType; + int8_t withTbName; + int8_t withSchema; + int8_t withTag; + int8_t withTagSchema; SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub + // TODO put -1 into unassignVgs + // SArray* unassignedVgs; } SMqSubscribeObj; SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); @@ -821,18 +548,6 @@ void tDeleteSMqSubActionLogObj(SMqSubActionLogObj* pLog); int32_t tEncodeSMqSubActionLogObj(void** buf, const SMqSubActionLogObj* pLog); void* tDecodeSMqSubActionLogObj(const void* buf, SMqSubActionLogObj* pLog); -typedef struct { - int64_t consumerId; - char cgroup[TSDB_CGROUP_LEN]; - SRWLatch lock; - SArray* vgs; // SArray -} SMqConsumerEpObj; - -SMqConsumerEpObj* tCloneSMqConsumerEpObj(const SMqConsumerEpObj* pConsumerEp); -void tDeleteSMqConsumerEpObj(SMqConsumerEpObj* pConsumerEp); -int32_t tEncodeSMqConsumerEpObj(void** buf, const SMqConsumerEpObj* pConsumerEp); -void* tDecodeSMqConsumerEpObj(const void* buf, SMqConsumerEpObj* pConsumerEp); - typedef struct { const SMqSubscribeObj* pOldSub; const SMqTopicObj* pTopic; @@ -845,12 +560,6 @@ typedef struct { SMqVgEp* pVgEp; } SMqRebOutputVg; -#if 0 -typedef struct { - int64_t consumerId; -} SMqRebOutputConsumer; -#endif - typedef struct { SArray* rebVgs; // SArray SArray* newConsumers; // SArray diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index c67c6862e7..dab587432e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -851,6 +851,8 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow; cfgRsp.streamMode = pDb->cfg.streamMode; cfgRsp.singleSTable = pDb->cfg.singleSTable; + cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; + cfgRsp.pRetensions = pDb->cfg.pRetensions; int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp); void *pRsp = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 37c819ae60..78d54d273d 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -32,9 +32,6 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L taosInitRWLatch(&pConsumer->lock); pConsumer->currentTopics = taosArrayInit(0, sizeof(void *)); -#if 0 - pConsumer->waitingRebTopics = NULL; -#endif pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); @@ -53,11 +50,6 @@ void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) { if (pConsumer->currentTopics) { taosArrayDestroyP(pConsumer->currentTopics, (FDelete)taosMemoryFree); } -#if 0 - if (pConsumer->waitingRebTopics) { - taosArrayDestroyP(pConsumer->waitingRebTopics, taosMemoryFree); - } -#endif if (pConsumer->rebNewTopics) { taosArrayDestroyP(pConsumer->rebNewTopics, (FDelete)taosMemoryFree); } @@ -87,20 +79,6 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { tlen += taosEncodeFixedI32(buf, 0); } -#if 0 - // waiting reb topics - if (pConsumer->waitingRebTopics) { - sz = taosArrayGetSize(pConsumer->waitingRebTopics); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - char *topic = taosArrayGetP(pConsumer->waitingRebTopics, i); - tlen += taosEncodeString(buf, topic); - } - } else { - tlen += taosEncodeFixedI32(buf, 0); - } -#endif - // reb new topics if (pConsumer->rebNewTopics) { sz = taosArrayGetSize(pConsumer->rebNewTopics); @@ -145,17 +123,6 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { taosArrayPush(pConsumer->currentTopics, &topic); } -#if 0 - // waiting reb topics - buf = taosDecodeFixedI32(buf, &sz); - pConsumer->waitingRebTopics = taosArrayInit(sz, sizeof(void *)); - for (int32_t i = 0; i < sz; i++) { - char *topic; - buf = taosDecodeString(buf, &topic); - taosArrayPush(pConsumer->waitingRebTopics, &topic); - } -#endif - // reb new topics buf = taosDecodeFixedI32(buf, &sz); pConsumer->rebNewTopics = taosArrayInit(sz, sizeof(void *)); @@ -182,7 +149,6 @@ SMqVgEp *tCloneSMqVgEp(const SMqVgEp *pVgEp) { if (pVgEpNew == NULL) return NULL; pVgEpNew->vgId = pVgEp->vgId; pVgEpNew->qmsg = strdup(pVgEp->qmsg); - /*memcpy(pVgEpNew->topic, pVgEp->topic, TSDB_TOPIC_FNAME_LEN);*/ pVgEpNew->epSet = pVgEp->epSet; return pVgEpNew; } @@ -193,7 +159,6 @@ int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgEp->vgId); tlen += taosEncodeString(buf, pVgEp->qmsg); - /*tlen += taosEncodeString(buf, pVgEp->topic);*/ tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); return tlen; } @@ -201,40 +166,10 @@ int32_t tEncodeSMqVgEp(void **buf, const SMqVgEp *pVgEp) { void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) { buf = taosDecodeFixedI32(buf, &pVgEp->vgId); buf = taosDecodeString(buf, &pVgEp->qmsg); - /*buf = taosDecodeStringTo(buf, pVgEp->topic);*/ buf = taosDecodeSEpSet(buf, &pVgEp->epSet); return (void *)buf; } -SMqConsumerEpObj *tCloneSMqConsumerEpObj(const SMqConsumerEpObj *pConsumerEp) { - SMqConsumerEpObj *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEpObj)); - if (pConsumerEpNew == NULL) return NULL; - pConsumerEpNew->consumerId = pConsumerEp->consumerId; - memcpy(pConsumerEpNew->cgroup, pConsumerEp->cgroup, TSDB_CGROUP_LEN); - taosInitRWLatch(&pConsumerEpNew->lock); - pConsumerEpNew->vgs = taosArrayDeepCopy(pConsumerEpNew->vgs, (FCopy)tCloneSMqVgEp); - return pConsumerEpNew; -} - -void tDeleteSMqConsumerEpObj(SMqConsumerEpObj *pConsumerEp) { - taosArrayDestroyEx(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp); -} - -int32_t tEncodeSMqConsumerEpObj(void **buf, const SMqConsumerEpObj *pConsumerEp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); - tlen += taosEncodeString(buf, pConsumerEp->cgroup); - tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp); - return tlen; -} - -void *tDecodeSMqConsumerEpObj(const void *buf, SMqConsumerEpObj *pConsumerEp) { - buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = taosDecodeStringTo(buf, pConsumerEp->cgroup); - buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp)); - return (void *)buf; -} - SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) { SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub)); if (pEpInSubNew == NULL) return NULL; @@ -276,7 +211,7 @@ void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) { } SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) { - SMqSubscribeObj *pSubNew = taosMemoryMalloc(sizeof(SMqSubscribeObj)); + SMqSubscribeObj *pSubNew = taosMemoryCalloc(1, sizeof(SMqSubscribeObj)); if (pSubNew == NULL) return NULL; memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubNew->lock); @@ -297,8 +232,14 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { if (pSubNew == NULL) return NULL; memcpy(pSubNew->key, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); taosInitRWLatch(&pSubNew->lock); + + pSubNew->subType = pSub->subType; + pSubNew->withTbName = pSub->withTbName; + pSubNew->withSchema = pSub->withSchema; + pSubNew->withTag = pSub->withTag; + pSubNew->withTagSchema = pSub->withTagSchema; + pSubNew->vgNum = pSub->vgNum; - /*pSubNew->consumerEps = taosArrayDeepCopy(pSub->consumerEps, (FCopy)tCloneSMqConsumerEpInSub);*/ pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); /*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/ void *pIter = NULL; @@ -325,6 +266,11 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { int32_t tlen = 0; tlen += taosEncodeString(buf, pSub->key); tlen += taosEncodeFixedI32(buf, pSub->vgNum); + tlen += taosEncodeFixedI8(buf, pSub->subType); + tlen += taosEncodeFixedI8(buf, pSub->withTbName); + tlen += taosEncodeFixedI8(buf, pSub->withSchema); + tlen += taosEncodeFixedI8(buf, pSub->withTag); + tlen += taosEncodeFixedI8(buf, pSub->withTagSchema); void *pIter = NULL; int32_t sz = taosHashGetSize(pSub->consumerHash); @@ -347,6 +293,11 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { // buf = taosDecodeStringTo(buf, pSub->key); buf = taosDecodeFixedI32(buf, &pSub->vgNum); + buf = taosDecodeFixedI8(buf, &pSub->subType); + buf = taosDecodeFixedI8(buf, &pSub->withTbName); + buf = taosDecodeFixedI8(buf, &pSub->withSchema); + buf = taosDecodeFixedI8(buf, &pSub->withTag); + buf = taosDecodeFixedI8(buf, &pSub->withTagSchema); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index c25561814b..156d894a44 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -427,7 +427,6 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { - terrno = TSDB_CODE_MND_INVALID_FUNC; goto RETRIEVE_FUNC_OVER; } @@ -439,21 +438,26 @@ static int32_t mndProcessRetrieveFuncReq(SNodeMsg *pReq) { funcInfo.outputLen = pFunc->outputLen; funcInfo.bufSize = pFunc->bufSize; funcInfo.signature = pFunc->signature; - funcInfo.commentSize = pFunc->commentSize; - funcInfo.codeSize = pFunc->codeSize; - funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize); - if (funcInfo.pCode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto RETRIEVE_FUNC_OVER; - } - memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); - if (funcInfo.commentSize > 0) { - funcInfo.pComment = taosMemoryCalloc(1, funcInfo.commentSize); - if (funcInfo.pComment == NULL) { + if (retrieveReq.ignoreCodeComment) { + funcInfo.commentSize = 0; + funcInfo.codeSize = 0; + } else { + funcInfo.commentSize = pFunc->commentSize; + funcInfo.codeSize = pFunc->codeSize; + funcInfo.pCode = taosMemoryCalloc(1, funcInfo.codeSize); + if (funcInfo.pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto RETRIEVE_FUNC_OVER; } - memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); + memcpy(funcInfo.pCode, pFunc->pCode, pFunc->codeSize); + if (funcInfo.commentSize > 0) { + funcInfo.pComment = taosMemoryCalloc(1, funcInfo.commentSize); + if (funcInfo.pComment == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } + memcpy(funcInfo.pComment, pFunc->pComment, pFunc->commentSize); + } } taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo); mndReleaseFunc(pMnode, pFunc); @@ -518,11 +522,16 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b1, false); - char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]); - STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]); + if (pFunc->pComment) { + char *b2 = taosMemoryCalloc(1, pShow->bytes[cols]); + STR_WITH_MAXSIZE_TO_VARSTR(b2, pFunc->pComment, pShow->bytes[cols]); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + } else { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, NULL, true); + } int32_t isAgg = (pFunc->funcType == TSDB_FUNC_TYPE_AGGREGATE) ? 1 : 0; @@ -556,4 +565,4 @@ static int32_t mndRetrieveFuncs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 7c115c2e24..32c667faf9 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -98,7 +98,7 @@ static const SInfosTableSchema userFuncSchema[] = { {.name = "name", .bytes = TSDB_FUNC_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "comment", .bytes = PATH_MAX - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "aggregate", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, - {.name = "comment", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "output_type", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "code_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index dad912b4e6..f5433e8f9e 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -133,9 +133,9 @@ OFFSET_DECODE_OVER: int32_t mndCreateOffsets(STrans *pTrans, const char *cgroup, const char *topicName, const SArray *vgs) { int32_t sz = taosArrayGetSize(vgs); for (int32_t i = 0; i < sz; i++) { - SMqConsumerEp *pConsumerEp = taosArrayGet(vgs, i); - SMqOffsetObj offsetObj; - if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, pConsumerEp->vgId) < 0) { + int32_t vgId = *(int32_t *)taosArrayGet(vgs, i); + SMqOffsetObj offsetObj; + if (mndMakePartitionKey(offsetObj.key, cgroup, topicName, vgId) < 0) { return -1; } offsetObj.offset = -1; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1a14c94640..f81179201c 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -18,7 +18,7 @@ #define SHOW_STEP_SIZE 100 -static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq); +static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq); static void mndFreeShowObj(SShowObj *pShow); static SShowObj *mndAcquireShowObj(SMnode *pMnode, int64_t showId); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); @@ -47,18 +47,80 @@ void mndCleanupShow(SMnode *pMnode) { } } -static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowReq *pReq) { +static int32_t convertToRetrieveType(char* name, int32_t len) { + int32_t type = -1; + + if (strncasecmp(name, TSDB_INS_TABLE_DNODES, len) == 0) { + type = TSDB_MGMT_TABLE_DNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, len) == 0) { + type = TSDB_MGMT_TABLE_MNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, len) == 0) { + type = TSDB_MGMT_TABLE_MODULE; + } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, len) == 0) { + type = TSDB_MGMT_TABLE_QNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, len) == 0) { + type = TSDB_MGMT_TABLE_BNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, len) == 0) { + type = TSDB_MGMT_TABLE_SNODE; + } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, len) == 0) { + type = TSDB_MGMT_TABLE_CLUSTER; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, len) == 0) { + type = TSDB_MGMT_TABLE_DB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) { + type = TSDB_MGMT_TABLE_FUNC; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) { + // type = TSDB_MGMT_TABLE_INDEX; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) { + type = TSDB_MGMT_TABLE_STB; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, len) == 0) { + type = TSDB_MGMT_TABLE_STREAMS; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { + type = TSDB_MGMT_TABLE_TABLE; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) { + // type = TSDB_MGMT_TABLE_DIST; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) { + type = TSDB_MGMT_TABLE_USER; + } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, len) == 0) { + type = TSDB_MGMT_TABLE_GRANTS; + } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) { + type = TSDB_MGMT_TABLE_VGROUP; + } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) { + type = TSDB_MGMT_TABLE_TOPICS; + } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) { + type = TSDB_MGMT_TABLE_CONSUMERS; + } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) { + type = TSDB_MGMT_TABLE_SUBSCRIBES; + } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, len) == 0) { + type = TSDB_MGMT_TABLE_TRANS; + } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, len) == 0) { + type = TSDB_MGMT_TABLE_SMAS; + } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, len) == 0) { + type = TSDB_MGMT_TABLE_CONFIGS; + } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, len) == 0) { + type = TSDB_MGMT_TABLE_CONNS; + } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, len) == 0) { + type = TSDB_MGMT_TABLE_QUERIES; + } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) { + type = TSDB_MGMT_TABLE_VNODES; + } else { +// ASSERT(0); + } + + return type; +} + +static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { SShowMgmt *pMgmt = &pMnode->showMgmt; int64_t showId = atomic_add_fetch_64(&pMgmt->showId, 1); if (showId == 0) atomic_add_fetch_64(&pMgmt->showId, 1); - int32_t size = sizeof(SShowObj) + pReq->payloadLen; + int32_t size = sizeof(SShowObj); + SShowObj showObj = {0}; - showObj.id = showId; + showObj.id = showId; showObj.pMnode = pMnode; - showObj.type = pReq->type; - showObj.payloadLen = pReq->payloadLen; + showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); memcpy(showObj.db, pReq->db, TSDB_DB_FNAME_LEN); int32_t keepTime = tsShellActivityTimer * 6 * 1000; @@ -127,10 +189,6 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { } if (retrieveReq.showId == 0) { - SShowReq req = {0}; - req.type = retrieveReq.type; - strncpy(req.db, retrieveReq.db, tListLen(req.db)); - STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1); if (pMeta == NULL) { terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; @@ -138,7 +196,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { return -1; } - pShow = mndCreateShowObj(pMnode, &req); + pShow = mndCreateShowObj(pMnode, &retrieveReq); if (pShow == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to process show-meta req since %s", terrstr()); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 6bdf4575a8..f304e3153d 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -20,6 +20,7 @@ #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPerfSchema.h" +#include "mndScheduler.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" @@ -443,6 +444,25 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) { *(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod; } + if (pStb->ast1Len > 0) { + if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(req.stbCfg.pRSmaParam); + taosMemoryFreeClear(req.stbCfg.pSchema); + return NULL; + } + } + if (pStb->ast2Len > 0) { + int32_t qmsgLen2 = 0; + if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam->qmsg1); + taosMemoryFreeClear(req.stbCfg.pRSmaParam); + taosMemoryFreeClear(req.stbCfg.pSchema); + return NULL; + } + } + req.stbCfg.pRSmaParam = pRSmaParam; } @@ -451,6 +471,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt if (pHead == NULL) { if (pRSmaParam) { taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam->qmsg1); + taosMemoryFreeClear(pRSmaParam->qmsg2); taosMemoryFreeClear(pRSmaParam); } taosMemoryFreeClear(req.stbCfg.pSchema); @@ -467,6 +489,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt *pContLen = contLen; if (pRSmaParam) { taosMemoryFreeClear(pRSmaParam->pFuncIds); + taosMemoryFreeClear(pRSmaParam->qmsg1); + taosMemoryFreeClear(pRSmaParam->qmsg2); taosMemoryFreeClear(pRSmaParam); } taosMemoryFreeClear(req.stbCfg.pSchema); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index f708d4ffc1..e37bd60e12 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -46,16 +46,8 @@ static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionDelete(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubscribeObj *pNewSub); -/*static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);*/ -/*static int32_t mndProcessSubscribeRsp(SNodeMsg *pMsg);*/ -static int32_t mndProcessSubscribeInternalReq(SNodeMsg *pMsg); -static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg); -/*static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);*/ -/*static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg);*/ -/*static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg);*/ -/*static int32_t mndProcessResetOffsetReq(SNodeMsg *pMsg);*/ - static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg); +static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pMsg); static int32_t mndSetSubRedoLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub) { SSdbRaw *pRedoRaw = mndSubActionEncode(pSub); @@ -73,15 +65,6 @@ static int32_t mndSetSubCommitLogs(SMnode *pMnode, STrans *pTrans, SMqSubscribeO return 0; } -/*static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup,*/ -/*const SMqConsumerEp *pConsumerEp);*/ - -/*static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/ -/*const char *topicName);*/ - -/*static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp,*/ -/*const char *oldTopicName);*/ - int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, .keyType = SDB_KEY_BINARY, @@ -91,13 +74,6 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndSubActionUpdate, .deleteFp = (SdbDeleteFp)mndSubActionDelete}; - /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);*/ - /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp);*/ - /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_REB_RSP, mndProcessSubscribeInternalRsp);*/ - /*mndSetMsgHandle(pMnode, TDMT_VND_MQ_CANCEL_CONN_RSP, mndProcessSubscribeInternalRsp);*/ - /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);*/ - /*mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq);*/ - /*mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessDoRebalanceMsg);*/ mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); return sdbSetTable(pMnode->pSdb, table); @@ -109,6 +85,12 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pSub->subType = pTopic->subType; + pSub->withTbName = pTopic->withTbName; + pSub->withSchema = pTopic->withSchema; + pSub->withTag = pTopic->withTag; + pSub->withTagSchema = pTopic->withTagSchema; + ASSERT(taosHashGetSize(pSub->consumerHash) == 1); if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { @@ -122,144 +104,19 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, return pSub; } -#if 0 -static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj *pTopic, const char *cgroup) { - SMqSubscribeObj *pSub = tNewSubscribeObj(); - if (pSub == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, cgroup, pTopic->name); - strcpy(pSub->key, key); - - if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { - tDeleteSMqSubscribeObj(pSub); - taosMemoryFree(pSub); - return NULL; - } - - // TODO: disable alter subscribed table - return pSub; -} - -static int32_t mndBuildRebalanceMsg(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, - const char *topicName) { - SMqMVRebReq req = { - .vgId = pConsumerEp->vgId, - .oldConsumerId = pConsumerEp->oldConsumerId, - .newConsumerId = pConsumerEp->consumerId, - }; - req.topic = strdup(topicName); - - int32_t tlen = tEncodeSMqMVRebReq(NULL, &req); - void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - SMsgHead *pMsgHead = (SMsgHead *)buf; - - pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); - pMsgHead->vgId = htonl(pConsumerEp->vgId); - - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqMVRebReq(&abuf, &req); - taosMemoryFree(req.topic); - - *pBuf = buf; - *pLen = tlen; - - return 0; -} - -static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *topicName) { - ASSERT(pConsumerEp->oldConsumerId != -1); - - void *buf; - int32_t tlen; - if (mndBuildRebalanceMsg(&buf, &tlen, pConsumerEp, topicName) < 0) { - return -1; - } - - int32_t vgId = pConsumerEp->vgId; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); - - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = buf; - action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_REB; - - mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - - return 0; -} - -static int32_t mndBuildCancelConnReq(void **pBuf, int32_t *pLen, const SMqConsumerEp *pConsumerEp, - const char *oldTopicName) { - SMqCancelConnReq req = {0}; - req.consumerId = pConsumerEp->consumerId; - req.vgId = pConsumerEp->vgId; - req.epoch = pConsumerEp->epoch; - strcpy(req.topicName, oldTopicName); - - int32_t tlen = tEncodeSMqCancelConnReq(NULL, &req); - void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - SMsgHead *pMsgHead = (SMsgHead *)buf; - - pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); - pMsgHead->vgId = htonl(pConsumerEp->vgId); - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqCancelConnReq(&abuf, &req); - *pBuf = buf; - *pLen = tlen; - return 0; -} - -static int32_t mndPersistCancelConnReq(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp, - const char *oldTopicName) { - void *buf; - int32_t tlen; - if (mndBuildCancelConnReq(&buf, &tlen, pConsumerEp, oldTopicName) < 0) { - return -1; - } - - int32_t vgId = pConsumerEp->vgId; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); - - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = buf; - action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_CANCEL_CONN; - - mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - - return 0; -} -#endif - -static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const char *subKey, const SMqRebOutputVg *pRebVg) { +static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscribeObj *pSub, + const SMqRebOutputVg *pRebVg) { SMqRebVgReq req = {0}; req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; req.vgId = pRebVg->pVgEp->vgId; req.qmsg = pRebVg->pVgEp->qmsg; - strncpy(req.subKey, subKey, TSDB_SUBSCRIBE_KEY_LEN); + req.subType = pSub->subType; + req.withTbName = pSub->withTbName; + req.withSchema = pSub->withSchema; + req.withTag = pSub->withTag; + req.withTagSchema = pSub->withTagSchema; + strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); void *buf = taosMemoryMalloc(tlen); @@ -280,13 +137,13 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const char *subK return 0; } -static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const char *subKey, +static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg) { ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId); void *buf; int32_t tlen; - if (mndBuildSubChangeReq(&buf, &tlen, subKey, pRebVg) < 0) { + if (mndBuildSubChangeReq(&buf, &tlen, pSub, pRebVg) < 0) { return -1; } @@ -307,108 +164,6 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const ch return 0; } -#if 0 -static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; - SMqCMGetSubEpRsp rsp = {0}; - int64_t consumerId = be64toh(pReq->consumerId); - int32_t epoch = ntohl(pReq->epoch); - - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); - if (pConsumer == NULL) { - terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - return -1; - } - // TODO add lock - ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); - int32_t serverEpoch = pConsumer->epoch; - - // TODO - int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus); - mDebug("consumer %ld epoch(%d) try to get sub ep, server epoch %d, old val: %d", consumerId, epoch, serverEpoch, - hbStatus); - atomic_store_32(&pConsumer->hbStatus, 0); - /*SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);*/ - /*sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);*/ - /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ - - strcpy(rsp.cgroup, pReq->cgroup); - if (epoch != serverEpoch) { - mInfo("send new assignment to consumer %ld, consumer epoch %d, server epoch %d", pConsumer->consumerId, epoch, - serverEpoch); - mDebug("consumer %ld try r lock", consumerId); - taosRLockLatch(&pConsumer->lock); - mDebug("consumer %ld r locked", consumerId); - SArray *pTopics = pConsumer->currentTopics; - int32_t sz = taosArrayGetSize(pTopics); - rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); - for (int32_t i = 0; i < sz; i++) { - char *topicName = taosArrayGetP(pTopics, i); - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topicName); - ASSERT(pSub); - int32_t csz = taosArrayGetSize(pSub->consumers); - // TODO: change to bsearch - for (int32_t j = 0; j < csz; j++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); - if (consumerId == pSubConsumer->consumerId) { - int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - mInfo("topic %s has %d vg", topicName, serverEpoch); - - SMqSubTopicEp topicEp; - strcpy(topicEp.topic, topicName); - - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topicName); - ASSERT(pTopic != NULL); - topicEp.schema = pTopic->schema; - mndReleaseTopic(pMnode, pTopic); - - topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); - for (int32_t k = 0; k < vgsz; k++) { - char offsetKey[TSDB_PARTITION_KEY_LEN]; - SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); - SMqSubVgEp vgEp = { - .epSet = pConsumerEp->epSet, - .vgId = pConsumerEp->vgId, - .offset = -1, - }; - mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId); - SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey); - if (pOffsetObj != NULL) { - vgEp.offset = pOffsetObj->offset; - mndReleaseOffset(pMnode, pOffsetObj); - } - taosArrayPush(topicEp.vgs, &vgEp); - } - taosArrayPush(rsp.topics, &topicEp); - break; - } - } - mndReleaseSubscribe(pMnode, pSub); - } - taosRUnLockLatch(&pConsumer->lock); - mDebug("consumer %ld r unlock", consumerId); - } - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); - void *buf = rpcMallocCont(tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; - ((SMqRspHead *)buf)->epoch = serverEpoch; - ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; - - void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); - tDeleteSMqCMGetSubEpRsp(&rsp); - mndReleaseConsumer(pMnode, pConsumer); - pMsg->pRsp = buf; - pMsg->rspLen = tlen; - return 0; -} -#endif - static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) { int32_t i = 0; while (key[i] != TMQ_SEPARATOR) { @@ -433,235 +188,6 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } -#if 0 -static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - SSdb *pSdb = pMnode->pSdb; - SMqConsumerObj *pConsumer; - void *pIter = NULL; - SMqDoRebalanceMsg *pRebMsg = rpcMallocCont(sizeof(SMqDoRebalanceMsg)); - pRebMsg->rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); - - while (1) { - pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); - if (pIter == NULL) break; - int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); - if (hbStatus > MND_SUBSCRIBE_REBALANCE_CNT) { - int32_t old = - atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST); - if (old == MQ_CONSUMER_STATUS__ACTIVE) { - // get all topics of that topic - int32_t sz = taosArrayGetSize(pConsumer->currentTopics); - for (int32_t i = 0; i < sz; i++) { - char *topic = taosArrayGetP(pConsumer->currentTopics, i); - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, pConsumer->cgroup, topic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - taosArrayPush(pRebSub->lostConsumers, &pConsumer->consumerId); - } - } - } - int32_t status = atomic_load_32(&pConsumer->status); - if (status == MQ_CONSUMER_STATUS__INIT || status == MQ_CONSUMER_STATUS__MODIFY) { - SArray *rebSubs; - if (status == MQ_CONSUMER_STATUS__INIT) { - rebSubs = pConsumer->currentTopics; - } else { - rebSubs = pConsumer->recentRemovedTopics; - } - int32_t sz = taosArrayGetSize(rebSubs); - for (int32_t i = 0; i < sz; i++) { - char *topic = taosArrayGetP(rebSubs, i); - char key[TSDB_SUBSCRIBE_KEY_LEN]; - mndMakeSubscribeKey(key, pConsumer->cgroup, topic); - SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); - if (status == MQ_CONSUMER_STATUS__INIT) { - taosArrayPush(pRebSub->newConsumers, &pConsumer->consumerId); - } else if (status == MQ_CONSUMER_STATUS__MODIFY) { - taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); - } - } - if (status == MQ_CONSUMER_STATUS__MODIFY) { - int32_t removeSz = taosArrayGetSize(pConsumer->recentRemovedTopics); - for (int32_t i = 0; i < removeSz; i++) { - char *topicName = taosArrayGetP(pConsumer->recentRemovedTopics, i); - taosMemoryFree(topicName); - } - taosArrayClear(pConsumer->recentRemovedTopics); - } - } - } - if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { - mInfo("mq rebalance will be triggered"); - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_MQ_DO_REBALANCE, - .pCont = pRebMsg, - .contLen = sizeof(SMqDoRebalanceMsg), - }; - tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - } else { - taosHashCleanup(pRebMsg->rebSubHash); - rpcFreeCont(pRebMsg); - } - return 0; -} -#endif - -#if 0 -static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - SMqDoRebalanceMsg *pReq = pMsg->rpcMsg.pCont; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_REBALANCE, &pMsg->rpcMsg); - void *pIter = NULL; - - mInfo("mq rebalance start"); - - while (1) { - pIter = taosHashIterate(pReq->rebSubHash, pIter); - if (pIter == NULL) break; - SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; - SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); - - mInfo("mq rebalance subscription: %s, vgNum: %d, unassignedVg: %d", pSub->key, pSub->vgNum, - (int32_t)taosArrayGetSize(pSub->unassignedVg)); - - // remove lost consumer - for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { - int64_t lostConsumerId = *(int64_t *)taosArrayGet(pRebSub->lostConsumers, i); - - mInfo("mq remove lost consumer %" PRId64 "", lostConsumerId); - - for (int32_t j = 0; j < taosArrayGetSize(pSub->consumers); j++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); - if (pSubConsumer->consumerId == lostConsumerId) { - taosArrayAddAll(pSub->unassignedVg, pSubConsumer->vgInfo); - taosArrayPush(pSub->lostConsumers, pSubConsumer); - taosArrayRemove(pSub->consumers, j); - break; - } - } - } - - // calculate rebalance - int32_t consumerNum = taosArrayGetSize(pSub->consumers); - if (consumerNum != 0) { - int32_t vgNum = pSub->vgNum; - int32_t vgEachConsumer = vgNum / consumerNum; - int32_t imbalanceVg = vgNum % consumerNum; - - // iterate all consumers, set unassignedVgStash - for (int32_t i = 0; i < consumerNum; i++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int32_t vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int32_t vgThisConsumerAfterRb; - if (i < imbalanceVg) - vgThisConsumerAfterRb = vgEachConsumer + 1; - else - vgThisConsumerAfterRb = vgEachConsumer; - - mInfo("mq consumer:%" PRId64 ", connectted vgroup number change from %d to %d", pSubConsumer->consumerId, - vgThisConsumerBeforeRb, vgThisConsumerAfterRb); - - while (taosArrayGetSize(pSubConsumer->vgInfo) > vgThisConsumerAfterRb) { - SMqConsumerEp *pConsumerEp = taosArrayPop(pSubConsumer->vgInfo); - ASSERT(pConsumerEp != NULL); - ASSERT(pConsumerEp->consumerId == pSubConsumer->consumerId); - taosArrayPush(pSub->unassignedVg, pConsumerEp); - mDebug("mq rebalance: vg %d push to unassignedVg", pConsumerEp->vgId); - } - - SMqConsumerObj *pRebConsumer = mndAcquireConsumer(pMnode, pSubConsumer->consumerId); - mDebug("consumer %ld try w lock", pRebConsumer->consumerId); - taosWLockLatch(&pRebConsumer->lock); - mDebug("consumer %ld w locked", pRebConsumer->consumerId); - int32_t status = atomic_load_32(&pRebConsumer->status); - if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || - (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || - (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) { - /*if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) {*/ - /*pRebConsumer->epoch++;*/ - /*}*/ - if (vgThisConsumerAfterRb != 0) { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); - } else { - atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); - } - - mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status, - pRebConsumer->status); - - SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); - sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); - mndTransAppendCommitlog(pTrans, pConsumerRaw); - } - taosWUnLockLatch(&pRebConsumer->lock); - mDebug("consumer %ld w unlock", pRebConsumer->consumerId); - mndReleaseConsumer(pMnode, pRebConsumer); - } - - // assign to vgroup - if (taosArrayGetSize(pSub->unassignedVg) != 0) { - for (int32_t i = 0; i < consumerNum; i++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int32_t vgThisConsumerAfterRb; - if (i < imbalanceVg) - vgThisConsumerAfterRb = vgEachConsumer + 1; - else - vgThisConsumerAfterRb = vgEachConsumer; - - while (taosArrayGetSize(pSubConsumer->vgInfo) < vgThisConsumerAfterRb) { - SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg); - mDebug("mq rebalance: vg %d pop from unassignedVg", pConsumerEp->vgId); - ASSERT(pConsumerEp != NULL); - - pConsumerEp->oldConsumerId = pConsumerEp->consumerId; - pConsumerEp->consumerId = pSubConsumer->consumerId; - // TODO - pConsumerEp->epoch = 0; - taosArrayPush(pSubConsumer->vgInfo, pConsumerEp); - - char topic[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CGROUP_LEN]; - mndSplitSubscribeKey(pSub->key, topic, cgroup); - if (pConsumerEp->oldConsumerId == -1) { - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId, - topic, pConsumerEp->consumerId, cgroup); - - mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); - mndReleaseTopic(pMnode, pTopic); - } else { - mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", - pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId); - - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, topic); - } - } - } - } - ASSERT(taosArrayGetSize(pSub->unassignedVg) == 0); - - // TODO: log rebalance statistics - SSdbRaw *pSubRaw = mndSubActionEncode(pSub); - sdbSetRawStatus(pSubRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pSubRaw); - } - mndReleaseSubscribe(pMnode, pSub); - } - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("mq-rebalance-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - taosHashCleanup(pReq->rebSubHash); - mndTransDrop(pTrans); - return -1; - } - - taosHashCleanup(pReq->rebSubHash); - mndTransDrop(pTrans); - return 0; -} -#endif - static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { if (pInput->pTopic != NULL) { // create subscribe @@ -825,36 +351,6 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR pRebVg->newConsumerId = pEpInSub->consumerId; taosArrayPush(pOutput->rebVgs, pRebVg); } - -#if 0 - /*int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);*/ - if (imbCnt < imbConsumerNum) { - imbCnt++; - // push until equal minVg + 1 - while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt + 1) { - // iter hash and find one vg - pRemovedIter = taosHashIterate(pHash, pRemovedIter); - ASSERT(pRemovedIter); - pRebVg = (SMqRebOutputVg *)pRemovedIter; - // push - taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); - pRebVg->newConsumerId = pEpInSub->consumerId; - taosArrayPush(pOutput->rebVgs, pRebVg); - } - } else { - // push until equal minVg - while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) { - // iter hash and find one vg - pRemovedIter = taosHashIterate(pHash, pRemovedIter); - ASSERT(pRemovedIter); - pRebVg = (SMqRebOutputVg *)pRemovedIter; - // push - taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); - pRebVg->newConsumerId = pEpInSub->consumerId; - taosArrayPush(pOutput->rebVgs, pRebVg); - } - } -#endif } // 7. handle unassigned vg @@ -911,7 +407,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SNodeMsg *pMsg, const SMqRebO int32_t vgNum = taosArrayGetSize(rebVgs); for (int32_t i = 0; i < vgNum; i++) { SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i); - if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub->key, pRebVg) < 0) { + if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg) < 0) { goto REB_FAIL; } } @@ -1040,52 +536,6 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { return 0; } -static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pConsumerEp) { - ASSERT(pConsumerEp->oldConsumerId == -1); - int32_t vgId = pConsumerEp->vgId; - - SMqSetCVgReq req = { - .vgId = vgId, - .consumerId = pConsumerEp->consumerId, - .sql = pTopic->sql, - .physicalPlan = pTopic->physicalPlan, - .qmsg = pConsumerEp->qmsg, - }; - - strcpy(req.cgroup, cgroup); - strcpy(req.topicName, pTopic->name); - int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *buf = taosMemoryMalloc(sizeof(SMsgHead) + tlen); - if (buf == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - SMsgHead *pMsgHead = (SMsgHead *)buf; - - pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); - pMsgHead->vgId = htonl(vgId); - - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - tEncodeSMqSetCVgReq(&abuf, &req); - - SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); - - STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = buf; - action.contLen = sizeof(SMsgHead) + tlen; - action.msgType = TDMT_VND_MQ_SET_CONN; - - mndReleaseVgroup(pMnode, pVgObj); - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - return 0; -} - void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b9c42fe899..22b1b404bb 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -76,7 +76,13 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); + SDB_SET_INT64(pRaw, dataPos, pTopic->subDbUid, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withTagSchema, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -134,7 +140,13 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); + SDB_GET_INT64(pRaw, dataPos, &pTopic->subDbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withTagSchema, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); @@ -254,34 +266,14 @@ static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; return -1; } + + if ((pCreate->ast == NULL || pCreate->ast[0] == 0) && pCreate->subscribeDbName[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + return -1; + } return 0; } -#if 0 -static int32_t mndGetPlanString(const SCMCreateTopicReq *pCreate, char **pStr) { - if (NULL == pCreate->ast) { - return TSDB_CODE_SUCCESS; - } - - SNode *pAst = NULL; - int32_t code = nodesStringToNode(pCreate->ast, &pAst); - - SQueryPlan *pPlan = NULL; - if (TSDB_CODE_SUCCESS == code) { - SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; - code = qCreateQueryPlan(&cxt, &pPlan, NULL); - } - - if (TSDB_CODE_SUCCESS == code) { - code = nodesNodeToString(pPlan, false, pStr, NULL); - } - nodesDestroyNode(pAst); - nodesDestroyNode(pPlan); - terrno = code; - return code; -} -#endif - static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; @@ -297,28 +289,38 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq topicObj.ast = strdup(pCreate->ast); topicObj.astLen = strlen(pCreate->ast) + 1; - SNode *pAst = NULL; - if (nodesStringToNode(pCreate->ast, &pAst) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } + if (pCreate->ast && pCreate->ast[0]) { + topicObj.subType = TOPIC_SUB_TYPE__TABLE; + topicObj.withTbName = 0; + topicObj.withSchema = 0; - SQueryPlan *pPlan = NULL; + SNode *pAst = NULL; + if (nodesStringToNode(pCreate->ast, &pAst) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } - SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; - if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } + SQueryPlan *pPlan = NULL; - if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; - } + SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; + if (qCreateQueryPlan(&cxt, &pPlan, NULL) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } - if (nodesNodeToString(pPlan, false, &topicObj.physicalPlan, NULL) != 0) { - mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; + if (qExtractResultSchema(pAst, &topicObj.schema.nCols, &topicObj.schema.pSchema) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + + if (nodesNodeToString(pPlan, false, &topicObj.physicalPlan, NULL) != 0) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } else { + topicObj.subType = TOPIC_SUB_TYPE__DB; + topicObj.withTbName = 1; + topicObj.withSchema = 1; } STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_TOPIC, &pReq->rpcMsg); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 1569976105..4db9411e87 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -240,7 +240,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); } { @@ -371,7 +371,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); } } diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index 2b0eaa0bec..5c431f65d3 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -36,7 +36,7 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG); + ASSERT_NE(pRsp->code, 0); } TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { @@ -50,7 +50,7 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen); ASSERT_NE(pRsp, nullptr); - ASSERT_EQ(pRsp->code, TSDB_CODE_INVALID_MSG); + ASSERT_NE(pRsp->code, 0); } TEST_F(MndTestShow, 03_ShowMsg_Conn) { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ce9549af56..a84776abfd 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -391,6 +391,7 @@ struct SReadH { #define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh)) #define TSDB_READ_FSET(rh) (&((rh)->rSet)) #define TSDB_READ_TABLE(rh) ((rh)->pTable) +#define TSDB_READ_TABLE_UID(rh) ((rh)->pTable->uid) #define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD) #define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA) #define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST) diff --git a/source/dnode/vnode/src/meta/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c index 9a566f788c..36057ae806 100644 --- a/source/dnode/vnode/src/meta/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -23,6 +23,9 @@ struct SMetaIdx { SIndex *pIdx; #endif /* data */ +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif }; int metaOpenIdx(SMeta *pMeta) { diff --git a/source/dnode/vnode/src/meta/metaTDBImpl.c b/source/dnode/vnode/src/meta/metaTDBImpl.c index 9fd11222bf..8389286596 100644 --- a/source/dnode/vnode/src/meta/metaTDBImpl.c +++ b/source/dnode/vnode/src/meta/metaTDBImpl.c @@ -45,10 +45,12 @@ struct SMetaDB { #endif }; -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { tb_uid_t uid; int32_t sver; } SSchemaDbKey; +#pragma pack(pop) typedef struct { char *name; @@ -626,14 +628,8 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { #ifdef META_TDB_SMA_TEST STSmaWrapper *pSW = NULL; - pSW = taosMemoryCalloc(1, sizeof(*pSW)); - if (pSW == NULL) { - return NULL; - } - SMSmaCursor *pCur = metaOpenSmaCursor(pMeta, uid); if (pCur == NULL) { - taosMemoryFree(pSW); return NULL; } @@ -653,6 +649,12 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid) { continue; } + if ((pSW == NULL) && ((pSW = taosMemoryCalloc(1, sizeof(*pSW))) == NULL)) { + TDB_FREE(pSmaVal); + metaCloseSmaCursor(pCur); + return NULL; + } + ++pSW->number; STSma *tptr = (STSma *)taosMemoryRealloc(pSW->tSma, pSW->number * sizeof(STSma)); if (tptr == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c7c8054120..510dd32459 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -618,127 +618,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { } } -#if 0 -int32_t tqProcessRebReq(STQ* pTq, char* msg) { - SMqMVRebReq req = {0}; - terrno = TSDB_CODE_SUCCESS; - tDecodeSMqMVRebReq(msg, &req); - - vDebug("vg %d set from consumer %ld to consumer %ld", req.vgId, req.oldConsumerId, req.newConsumerId); - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); - ASSERT(pConsumer); - ASSERT(pConsumer->consumerId == req.oldConsumerId); - int32_t numOfTopics = taosArrayGetSize(pConsumer->topics); - if (numOfTopics == 1) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0); - ASSERT(strcmp(pTopic->topicName, req.topic) == 0); - STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId); - if (pNewConsumer == NULL) { - pConsumer->consumerId = req.newConsumerId; - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); - tqHandlePurge(pTq->tqMeta, req.oldConsumerId); - return 0; - } else { - taosArrayPush(pNewConsumer->topics, pTopic); - } - } else { - for (int32_t i = 0; i < numOfTopics; i++) { - STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); - if (strcmp(pTopic->topicName, req.topic) == 0) { - STqConsumer* pNewConsumer = tqHandleGet(pTq->tqMeta, req.newConsumerId); - if (pNewConsumer == NULL) { - pNewConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); - if (pNewConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - strcpy(pNewConsumer->cgroup, pConsumer->cgroup); - pNewConsumer->topics = taosArrayInit(0, sizeof(STqTopic)); - pNewConsumer->consumerId = req.newConsumerId; - pNewConsumer->epoch = 0; - - taosArrayPush(pNewConsumer->topics, pTopic); - tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.newConsumerId); - return 0; - } - ASSERT(pNewConsumer->consumerId == req.newConsumerId); - taosArrayPush(pNewConsumer->topics, pTopic); - break; - } - } - // - } - return 0; -} - -int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { - SMqSetCVgReq req = {0}; - tDecodeSMqSetCVgReq(msg, &req); - bool create = false; - - vDebug("vg %d set to consumer %ld", req.vgId, req.consumerId); - STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.consumerId); - if (pConsumer == NULL) { - pConsumer = taosMemoryCalloc(1, sizeof(STqConsumer)); - if (pConsumer == NULL) { - terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; - return -1; - } - strcpy(pConsumer->cgroup, req.cgroup); - pConsumer->topics = taosArrayInit(0, sizeof(STqTopic)); - pConsumer->consumerId = req.consumerId; - pConsumer->epoch = 0; - create = true; - } - - STqTopic* pTopic = taosMemoryCalloc(1, sizeof(STqTopic)); - if (pTopic == NULL) { - taosArrayDestroy(pConsumer->topics); - taosMemoryFree(pConsumer); - return -1; - } - strcpy(pTopic->topicName, req.topicName); - pTopic->sql = req.sql; - pTopic->physicalPlan = req.physicalPlan; - pTopic->qmsg = req.qmsg; - /*pTopic->committedOffset = -1;*/ - /*pTopic->currentOffset = -1;*/ - - pTopic->buffer.firstOffset = -1; - pTopic->buffer.lastOffset = -1; - pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); - if (pTopic->pReadhandle == NULL) { - ASSERT(false); - } - for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - pTopic->buffer.output[i].status = 0; - STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta); - SReadHandle handle = { - .reader = pReadHandle, - .meta = pTq->pVnodeMeta, - }; - pTopic->buffer.output[i].pReadHandle = pReadHandle; - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, &handle); - ASSERT(pTopic->buffer.output[i].task); - } - vDebug("set topic %s to consumer %ld on vg %d", pTopic->topicName, req.consumerId, TD_VID(pTq->pVnode)); - taosArrayPush(pConsumer->topics, pTopic); - if (create) { - tqHandleMovePut(pTq->tqMeta, req.consumerId, pConsumer); - tqHandleCommit(pTq->tqMeta, req.consumerId); - } - terrno = TSDB_CODE_SUCCESS; - return 0; -} - -int32_t tqProcessCancelConnReq(STQ* pTq, char* msg) { - terrno = TSDB_CODE_SUCCESS; - return 0; -} -#endif - int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) { if (pTask->execType == TASK_EXEC__NONE) return 0; diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 809fd7cb06..72469b74ac 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F } strcpy(pMeta->dirPath, path); - char name[pathLen + 10]; + char *name = taosMemoryMalloc(pathLen + 10) ; strcpy(name, path); if (!taosDirExist(name) && taosMkDir(name) != 0) { @@ -99,6 +99,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s ", name, terrstr()); // free memory + taosMemoryFree(name); return NULL; } @@ -106,6 +107,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + taosMemoryFree(name); return NULL; } pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; @@ -116,8 +118,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); tqError("failed to open file:%s since %s", name, terrstr()); + taosMemoryFree(name); return NULL; } + taosMemoryFree(name); pMeta->pFile = pFile; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index fd640471bd..f70a4478b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3260,6 +3260,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT* pTsdbReadHandle, SColumnDat return TSDB_CODE_SUCCESS; } + tsdbDebug("vgId:%d succeed to load block statis part for uid %" PRIu64, REPO_ID(pHandle->pTsdb), + TSDB_READ_TABLE_UID(&pHandle->rhelper)); + int16_t* colIds = pHandle->defaultLoadColumn->pData; size_t numOfCols = QH_GET_NUM_OF_COLS(pHandle); diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 0cf8a0d359..f18f36e07b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -322,15 +322,18 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); if (!pBlock->aggrStat) { + tsdbDebug("vgId:%d no need to load block statis part for uid %" PRIu64 " since not exist", REPO_ID(pReadh->pRepo), + TSDB_READ_TABLE_UID(pReadh)); return TSDB_STATIS_NONE; } SDFile *pDFileAggr = pBlock->last ? TSDB_READ_SMAL_FILE(pReadh) : TSDB_READ_SMAD_FILE(pReadh); if (tsdbSeekDFile(pDFileAggr, pBlock->aggrOffset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block aggr part while seek file %s to offset %" PRIu64 " since %s", - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, - tstrerror(terrno)); + tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 " while seek file %s to offset %" PRIu64 + " since %s", + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + (uint64_t)pBlock->aggrOffset, tstrerror(terrno)); return -1; } @@ -339,25 +342,28 @@ int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) { int64_t nreadAggr = tsdbReadDFile(pDFileAggr, (void *)(pReadh->pAggrBlkData), sizeAggr); if (nreadAggr < 0) { - tsdbError("vgId:%d failed to load block aggr part while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), tstrerror(terrno), - (uint64_t)pBlock->aggrOffset, sizeAggr); + tsdbError("vgId:%d failed to load block statis part for uid %" PRIu64 + " while read file %s since %s, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + tstrerror(terrno), (uint64_t)pBlock->aggrOffset, sizeAggr); return -1; } if (nreadAggr < sizeAggr) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block aggr part in file %s is corrupted, offset:%" PRIu64 " expected bytes:%" PRIzu - " read bytes: %" PRId64, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr, - nreadAggr); + tsdbError("vgId:%d block statis part for uid %" PRIu64 " in file %s is corrupted, offset:%" PRIu64 + " expected bytes:%" PRIzu " read bytes: %" PRId64, + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + (uint64_t)pBlock->aggrOffset, sizeAggr, nreadAggr); return -1; } if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pAggrBlkData), (uint32_t)sizeAggr)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block aggr part in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu, - TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), (uint64_t)pBlock->aggrOffset, sizeAggr); + tsdbError("vgId:%d block statis part for uid %" PRIu64 + "in file %s is corrupted since wrong checksum, offset:%" PRIu64 " len :%" PRIzu, + TSDB_READ_REPO_ID(pReadh), TSDB_READ_TABLE_UID(pReadh), TSDB_FILE_FULL_NAME(pDFileAggr), + (uint64_t)pBlock->aggrOffset, sizeAggr); return -1; } return 0; @@ -367,7 +373,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { ASSERT(pBlock->numOfSubBlocks <= 1); SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh); if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) { - tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s", + tsdbError("vgId:%d failed to load block head part while seek file %s to offset %" PRId64 " since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno)); return -1; } @@ -377,14 +383,14 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size); if (nread < 0) { - tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, + tsdbError("vgId:%d failed to load block head part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size); return -1; } if (nread < size) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu + tsdbError("vgId:%d block head part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu " read bytes: %" PRId64, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread); return -1; @@ -392,7 +398,7 @@ static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock) { if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, + tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu, TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size); return -1; } @@ -546,7 +552,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat int32_t tsize = (int32_t)tsdbBlockStatisSize(pBlock->numOfCols, (uint32_t)pBlock->blkVer); if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; - tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", + tsdbError("vgId:%d block head part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 4b1c213cdd..273b7447ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -643,6 +643,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SSubmitMsgIter msgIter = {0}; SSubmitBlk *pBlock = NULL; SInterval interval = {0}; + TSKEY lastWinSKey = INT64_MIN; if (tInitSubmitMsgIter(pMsg, &msgIter) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; @@ -657,7 +658,7 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SSubmitBlkIter blkIter = {0}; if (tInitSubmitBlkIter(pBlock, &blkIter) != TSDB_CODE_SUCCESS) { - tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW); break; } @@ -667,31 +668,37 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers tdFreeTSmaWrapper(pSW); break; } - if (pSW == NULL) { + if (!pSW || (pTSma->tableUid != pBlock->suid)) { + if (pSW) { + pSW = tdFreeTSmaWrapper(pSW); + } if ((pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) { break; } if ((pSW->number) <= 0 || (pSW->tSma == NULL)) { - tdFreeTSmaWrapper(pSW); + pSW = tdFreeTSmaWrapper(pSW); break; } - pTSma = pSW->tSma; - } - interval.interval = pTSma->interval; - interval.intervalUnit = pTSma->intervalUnit; - interval.offset = pTSma->offset; - interval.precision = REPO_CFG(pTsdb)->precision; - interval.sliding = pTSma->sliding; - interval.slidingUnit = pTSma->slidingUnit; + pTSma = pSW->tSma; + + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = REPO_CFG(pTsdb)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + } TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); - tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version); - - // TODO: release only when suid changes. - tdDestroyTSmaWrapper(pSW); - taosMemoryFreeClear(pSW); + if (lastWinSKey != winSKey) { + lastWinSKey = winSKey; + tsdbSetExpiredWindow(pTsdb, pItemsHash, pTSma->indexUid, winSKey, version); + } else { + tsdbDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", + REPO_ID(pTsdb), pTSma->indexUid, winSKey); + } } } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index fb75ffe6b6..1c3a3787f3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -215,10 +215,21 @@ static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq) { return -1; } + // TODO: remove the debug log + SRSmaParam *param = vCreateTbReq.stbCfg.pRSmaParam; + if (param) { + printf("qmsg1 len = %d, body = %s\n", param->qmsg1 ? (int32_t)strlen(param->qmsg1) : 0, + param->qmsg1 ? param->qmsg1 : ""); + printf("qmsg1 len = %d, body = %s\n", param->qmsg2 ? (int32_t)strlen(param->qmsg2) : 0, + param->qmsg2 ? param->qmsg2 : ""); + } + taosMemoryFree(vCreateTbReq.stbCfg.pSchema); taosMemoryFree(vCreateTbReq.stbCfg.pTagSchema); if (vCreateTbReq.stbCfg.pRSmaParam) { taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->pFuncIds); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg1); + taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam->qmsg2); taosMemoryFree(vCreateTbReq.stbCfg.pRSmaParam); } taosMemoryFree(vCreateTbReq.name); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 8938084724..918892b786 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -104,6 +104,10 @@ typedef struct SCatalog { typedef struct SCtgApiStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SCtgApiStat; typedef struct SCtgRuntimeStat { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 21e0be40a4..410527c9e6 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -643,6 +643,49 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt return TSDB_CODE_SUCCESS; } +int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *funcName, SFuncInfo **out) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get udf info from mnode, funcName:%s", funcName); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)]((void *)funcName, &msg, 0, &msgLen); + if (code) { + ctgError("Build get udf msg failed, code:%x, db:%s", code, funcName); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_RETRIEVE_FUNC, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + if (TSDB_CODE_MND_FUNC_NOT_EXIST == rpcRsp.code) { + ctgDebug("funcName %s not exist in mnode", funcName); + taosMemoryFreeClear(*out); + CTG_RET(TSDB_CODE_SUCCESS); + } + + ctgError("error rsp for get udf, error:%s, funcName:%s", tstrerror(rpcRsp.code), funcName); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)](*out, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get udf rsp failed, code:%x, funcName:%s", code, funcName); + CTG_ERR_RET(code); + } + + ctgDebug("Got udf from mnode, funcName:%s", funcName); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -2811,6 +2854,30 @@ int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, CTG_API_LEAVE(ctgGetIndexInfoFromMnode(pCtg, pRpc, pMgmtEps, indexName, pInfo)); } +int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == funcName || NULL == pInfo) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + *pInfo = taosMemoryMalloc(sizeof(SFuncInfo)); + if (NULL == *pInfo) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_JRET(ctgGetUdfInfoFromMnode(pCtg, pRpc, pMgmtEps, funcName, pInfo)); + +_return: + + if (code) { + taosMemoryFreeClear(*pInfo); + } + + CTG_API_LEAVE(code); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 9a6356acbd..1045acbe93 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -24,6 +24,9 @@ #pragma GCC diagnostic ignored "-Wformat" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" #include "catalog.h" @@ -155,11 +158,11 @@ int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) { } void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { - SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); - SName sn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName sn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(sn.dbname, "db1"); strcpy(sn.tname, ctgTestSTablename); @@ -577,9 +580,16 @@ void ctgTestSetRspDbVgroups() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroups); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroups); } @@ -590,9 +600,16 @@ void ctgTestSetRspTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspTableMeta); } @@ -603,9 +620,16 @@ void ctgTestSetRspCTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspCTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspCTableMeta); } @@ -616,9 +640,16 @@ void ctgTestSetRspSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspSTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspSTableMeta); } @@ -629,9 +660,16 @@ void ctgTestSetRspMultiSTableMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspMultiSTableMeta); } @@ -642,9 +680,16 @@ void ctgTestSetRspByIdx() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspByIdx); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspByIdx); } @@ -656,9 +701,16 @@ void ctgTestSetRspDbVgroupsAndNormalMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta); } @@ -669,9 +721,16 @@ void ctgTestSetRspDbVgroupsAndChildMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta); } @@ -682,9 +741,16 @@ void ctgTestSetRspDbVgroupsAndSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta); } @@ -695,9 +761,16 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { static Stub stub; stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRecv", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRecv$", result); +#endif for (const auto &f : result) { stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta); } @@ -789,7 +862,7 @@ void *ctgTestGetCtableMetaThread(void *param) { STableMeta *tbMeta = NULL; bool inCache = false; - SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(cn.dbname, "db1"); strcpy(cn.tname, ctgTestCTablename); @@ -871,7 +944,7 @@ TEST(tableMeta, normalTable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -978,7 +1051,7 @@ TEST(tableMeta, childTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -1085,7 +1158,7 @@ TEST(tableMeta, superTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1210,7 +1283,7 @@ TEST(tableMeta, rmStbMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1280,7 +1353,7 @@ TEST(tableMeta, updateStbMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -1378,7 +1451,7 @@ TEST(refreshGetMeta, normal2normal) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1457,7 +1530,7 @@ TEST(refreshGetMeta, normal2notexist) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1529,7 +1602,7 @@ TEST(refreshGetMeta, normal2child) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; @@ -1613,7 +1686,7 @@ TEST(refreshGetMeta, stable2child) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; @@ -1699,7 +1772,7 @@ TEST(refreshGetMeta, stable2stable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentSTableName = ctgTestTablename; @@ -1786,7 +1859,7 @@ TEST(refreshGetMeta, child2stable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); ctgTestCurrentCTableName = ctgTestTablename; @@ -1871,7 +1944,7 @@ TEST(tableDistVgroup, normalTable) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -1913,7 +1986,7 @@ TEST(tableDistVgroup, childTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestCTablename); @@ -1956,7 +2029,7 @@ TEST(tableDistVgroup, superTableCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestSTablename); @@ -2006,7 +2079,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2091,7 +2164,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2143,7 +2216,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2195,7 +2268,7 @@ TEST(multiThread, ctableMeta) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); strcpy(n.tname, ctgTestTablename); @@ -2247,7 +2320,7 @@ TEST(rentTest, allRent) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1}; + SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} }; strcpy(n.dbname, "db1"); for (int32_t i = 1; i <= 10; ++i) { diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9e48e12cb5..5adfb7caca 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -382,7 +382,7 @@ typedef struct SSysTableScanInfo { void* pCur; // cursor for iterate the local table meta store. SArray* scanCols; // SArray scan column id list - int32_t type; // show type, TODO remove it +// int32_t type; // show type, TODO remove it SName name; SSDataBlock* pRes; int32_t capacity; @@ -479,7 +479,7 @@ typedef struct { typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; - SArray* pGroupCols; + SArray* pGroupCols; // group by columns, SArray SArray* pGroupColVals; // current group column values, SArray SNode* pCondition; bool isInit; // denote if current val is initialized or not @@ -600,7 +600,6 @@ typedef struct SJoinOperatorInfo { int32_t rightPos; SColumnInfo rightCol; SNode *pOnCondition; -// SRspResultInfo resultInfo; } SJoinOperatorInfo; int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7791a345ed..7a7cb68424 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1087,6 +1087,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pCtx[i].currentStage = MAIN_SCAN; SInputColumnInfoData* pInput = &pCtx[i].input; + pInput->uid = pBlock->info.uid; SExprInfo* pOneExpr = &pOperator->pExpr[i]; for (int32_t j = 0; j < pOneExpr->base.numOfParams; ++j) { @@ -1101,7 +1102,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->pPTS = taosArrayGet(pBlock->pDataBlock, 0); // todo set the correct timestamp column ASSERT(pInput->pData[j] != NULL); } else if (pFuncParam->type == FUNC_PARAM_TYPE_VALUE) { - if (createDummyCol) { + // todo avoid case: top(k, 12), 12 is the value parameter. + // sum(11), 11 is also the value parameter. + if (createDummyCol && pOneExpr->base.numOfParams == 1) { code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return code; @@ -1876,67 +1879,58 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, } } pCtx->resDataInfo.interBufSize = env.calcMemSize; - } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR || pExpr->pExpr->nodeType == QUERY_NODE_VALUE) { - pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; // for simple column, the intermediate buffer needs to hold one element. + } else if (pExpr->pExpr->nodeType == QUERY_NODE_COLUMN || pExpr->pExpr->nodeType == QUERY_NODE_OPERATOR || + pExpr->pExpr->nodeType == QUERY_NODE_VALUE) { + // for simple column, the intermediate buffer needs to hold one element. + pCtx->resDataInfo.interBufSize = pFunct->resSchema.bytes; } pCtx->input.numOfInputCols = pFunct->numOfParams; pCtx->input.pData = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); pCtx->input.pColumnDataAgg = taosMemoryCalloc(pFunct->numOfParams, POINTER_BYTES); - pCtx->pTsOutput = NULL; + pCtx->pTsOutput = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; - pCtx->resDataInfo.type = pFunct->resSchema.type; - pCtx->order = TSDB_ORDER_ASC; - pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; -#if 0 - for (int32_t j = 0; j < pCtx->numOfParams; ++j) { -// int16_t type = pFunct->param[j].nType; -// int16_t bytes = pFunct->param[j].nLen; + pCtx->resDataInfo.type = pFunct->resSchema.type; + pCtx->order = TSDB_ORDER_ASC; + pCtx->start.key = INT64_MIN; + pCtx->end.key = INT64_MIN; + pCtx->numOfParams = pExpr->base.numOfParams; -// if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { -// taosVariantCreateFromBinary(&pCtx->param[j], pFunct->param[j].pz, bytes, type); -// } else { -// taosVariantCreateFromBinary(&pCtx->param[j], (char *)&pFunct->param[j].i, bytes, type); + pCtx->param = pFunct->pParam; +// for (int32_t j = 0; j < pCtx->numOfParams; ++j) { +// // set the order information for top/bottom query +// int32_t functionId = pCtx->functionId; +// if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { +// int32_t f = getExprFunctionId(&pExpr[0]); +// assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); +// +// // pCtx->param[2].i = pQueryAttr->order.order; +// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; +// // pCtx->param[3].i = functionId; +// // pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; +// +// // pCtx->param[1].i = pQueryAttr->order.col.info.colId; +// } else if (functionId == FUNCTION_INTERP) { +// // pCtx->param[2].i = (int8_t)pQueryAttr->fillType; +// // if (pQueryAttr->fillVal != NULL) { +// // if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { +// // pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; +// // } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value +// // if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { +// // taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); +// // } +// // } +// // } +// } else if (functionId == FUNCTION_TWA) { +// // pCtx->param[1].i = pQueryAttr->window.skey; +// // pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; +// // pCtx->param[2].i = pQueryAttr->window.ekey; +// // pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; +// } else if (functionId == FUNCTION_ARITHM) { +// // pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); // } - } - - // set the order information for top/bottom query - int32_t functionId = pCtx->functionId; - if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { - int32_t f = getExprFunctionId(&pExpr[0]); - assert(f == FUNCTION_TS || f == FUNCTION_TS_DUMMY); - -// pCtx->param[2].i = pQueryAttr->order.order; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - pCtx->param[3].i = functionId; - pCtx->param[3].nType = TSDB_DATA_TYPE_BIGINT; - -// pCtx->param[1].i = pQueryAttr->order.col.info.colId; - } else if (functionId == FUNCTION_INTERP) { -// pCtx->param[2].i = (int8_t)pQueryAttr->fillType; -// if (pQueryAttr->fillVal != NULL) { -// if (isNull((const char *)&pQueryAttr->fillVal[i], pCtx->inputType)) { -// pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; -// } else { // todo refactor, taosVariantCreateFromBinary should handle the NULL value -// if (pCtx->inputType != TSDB_DATA_TYPE_BINARY && pCtx->inputType != TSDB_DATA_TYPE_NCHAR) { -// taosVariantCreateFromBinary(&pCtx->param[1], (char *)&pQueryAttr->fillVal[i], pCtx->inputBytes, pCtx->inputType); -// } -// } -// } - } else if (functionId == FUNCTION_TS_COMP) { -// pCtx->param[0].i = pQueryAttr->vgId; //TODO this should be the parameter from client - pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == FUNCTION_TWA) { -// pCtx->param[1].i = pQueryAttr->window.skey; - pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; -// pCtx->param[2].i = pQueryAttr->window.ekey; - pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; - } else if (functionId == FUNCTION_ARITHM) { -// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); - } -#endif +// } } for (int32_t i = 1; i < numOfOutput; ++i) { @@ -1955,7 +1949,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t j = 0; j < pCtx[i].numOfParams; ++j) { - taosVariantDestroy(&pCtx[i].param[j]); + taosVariantDestroy(&pCtx[i].param[j].param); } taosVariantDestroy(&pCtx[i].tag); @@ -6487,9 +6481,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); - if (pDataReader == NULL) { - return NULL; - } SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9f75f97632..e51284bfef 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -410,8 +410,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t contentLen = 0; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - int32_t* offset = pPage + startOffset; - columnLen = pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; + int32_t* offset = (int32_t*)((char*)pPage + startOffset); + columnLen = (char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity; char* data = (char*)(columnLen + sizeof(int32_t)); if (colDataIsNull_s(pColInfoData, j)) { @@ -424,8 +424,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { contentLen = varDataTLen(src); } } else { - char* bitmap = pPage + startOffset; - columnLen = pPage + startOffset + BitmapLen(pInfo->rowCapacity); + char* bitmap = (char*)pPage + startOffset; + columnLen = (char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity); char* data = (char*) columnLen + sizeof(int32_t); bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b054bbfcb6..ce3de02c9f 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -319,7 +319,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; // The read handle is not initialized yet, since no qualified tables exists - if (pTableScanInfo->dataReader == NULL) { + if (pTableScanInfo->dataReader == NULL || pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -375,10 +375,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) { return p; } -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, - int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime, - SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, - SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int32_t numOfOutput, int32_t dataLoadFlag, + int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, + SNode* pCondition, SInterval* pInterval, double sampleRatio, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); @@ -391,19 +390,19 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, return NULL; } - pInfo->interval = *pInterval; - pInfo->sampleRatio = sampleRatio; - pInfo->dataBlockLoadFlag = dataLoadFlag; - pInfo->pResBlock = pResBlock; - pInfo->pFilterNode = pCondition; - pInfo->dataReader = pTsdbReadHandle; - pInfo->times = repeatTime; - pInfo->reverseTimes = reverseTime; - pInfo->order = order; - pInfo->current = 0; - pInfo->scanFlag = MAIN_SCAN; - pInfo->pColMatchInfo = pColMatchInfo; - pOperator->name = "TableScanOperator"; + pInfo->interval = *pInterval; + pInfo->sampleRatio = sampleRatio; + pInfo->dataBlockLoadFlag= dataLoadFlag; + pInfo->pResBlock = pResBlock; + pInfo->pFilterNode = pCondition; + pInfo->dataReader = pDataReader; + pInfo->times = repeatTime; + pInfo->reverseTimes = reverseTime; + pInfo->order = order; + pInfo->current = 0; + pInfo->scanFlag = MAIN_SCAN; + pInfo->pColMatchInfo = pColMatchInfo; + pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; @@ -677,7 +676,8 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { tsem_destroy(&pInfo->ready); blockDataDestroy(pInfo->pRes); - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { metaCloseTbCursor(pInfo->pCur); } } @@ -812,7 +812,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { SSysTableScanInfo* pInfo = pOperator->info; // retrieve local table list info from vnode - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { if (pInfo->pCur == NULL) { pInfo->pCur = metaOpenTbCursor(pInfo->readHandle); } @@ -864,8 +865,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { while (1) { int64_t startTs = taosGetTimestampUs(); - - pInfo->req.type = pInfo->type; strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); if (pInfo->showRewrite) { @@ -947,68 +946,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB pInfo->pCondition = pCondition; pInfo->scanCols = colList; - // TODO remove it - int32_t tableType = 0; - const char* name = tNameGetTableName(pName); - if (strncasecmp(name, TSDB_INS_TABLE_DNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_DNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_MNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_MNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_MODULES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_MODULE; - } else if (strncasecmp(name, TSDB_INS_TABLE_QNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_QNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_BNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_BNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_SNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_SNODE; - } else if (strncasecmp(name, TSDB_INS_TABLE_CLUSTER, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CLUSTER; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_DATABASES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_DB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_FUNC; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, tListLen(pName->tname)) == 0) { - // tableType = TSDB_MGMT_TABLE_INDEX; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_STB; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_STREAMS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_STREAMS; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_TABLE; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, tListLen(pName->tname)) == 0) { - // tableType = TSDB_MGMT_TABLE_DIST; - } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_USER; - } else if (strncasecmp(name, TSDB_INS_TABLE_LICENCES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_GRANTS; - } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_VGROUP; - } else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_TOPICS; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CONSUMERS; - } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_SUBSCRIBES; - } else if (strncasecmp(name, TSDB_INS_TABLE_TRANS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_TRANS; - } else if (strncasecmp(name, TSDB_INS_TABLE_SMAS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_SMAS; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONFIGS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CONFIGS; - } else if (strncasecmp(name, TSDB_INS_TABLE_CONNS, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_CONNS; - } else if (strncasecmp(name, TSDB_INS_TABLE_QUERIES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_QUERIES; - } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, tListLen(pName->tname)) == 0) { - tableType = TSDB_MGMT_TABLE_VNODES; - } else { - ASSERT(0); - } - tNameAssign(&pInfo->name, pName); - pInfo->type = tableType; - if (pInfo->type == TSDB_MGMT_TABLE_TABLE) { + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = pSysTableReadHandle; blockDataEnsureCapacity(pInfo->pRes, pInfo->capacity); } else { diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index fe370d93b2..8f7ff04d92 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -48,7 +48,7 @@ struct SSHashObj { }; static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = (length < HASH_MAX_CAPACITY ? length : HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); @@ -127,7 +127,7 @@ static void taosHashTableResize(SSHashObj *pHashObj) { } size_t inc = newCapacity - pHashObj->capacity; - memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); + memset((char*)pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc); pHashObj->hashList = pNewEntryList; pHashObj->capacity = newCapacity; @@ -301,7 +301,7 @@ size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) { void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) { int32_t offset = offsetof(SHNode, data); - SHNode *node = data - offset; + SHNode *node = ((SHNode*)(char*)data - offset); if (keyLen != NULL) { *keyLen = pHashObj->keyLen; } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 870a0b71fc..543df30686 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -102,7 +102,7 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } else if (pInfo->type == data_asc) { v = ++pInfo->startVal; } else if (pInfo->type == data_rand) { - v = random(); + v = taosRand(); } colDataAppend(pColInfo, i, reinterpret_cast(&v), false); @@ -172,7 +172,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator, bool* newgroup) { } else if (pInfo->type == data_asc) { v = ++pInfo->startVal; } else if (pInfo->type == data_rand) { - v = random(); + v = taosRand(); } colDataAppend(pColInfo1, i, reinterpret_cast(&v), false); @@ -938,7 +938,7 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - SReadHandle handle = {.reader = reinterpret_cast(0x1), .meta = reinterpret_cast(0x1)}; + SReadHandle handle = { reinterpret_cast(0x1), reinterpret_cast(0x1), NULL }; struct SSubplan *plan = NULL; int32_t code = qStringToSubplan(msg, &plan); diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index eddc094285..98f1209ad9 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -47,6 +47,9 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +target_link_libraries( + udf1 PUBLIC os +) #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) add_executable(udfd src/udfd.c) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 11c89f1568..8473138e4a 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -58,6 +58,9 @@ bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t lastFunction(SqlFunctionCtx *pCtx); +bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); +int32_t topFunction(SqlFunctionCtx *pCtx); + #ifdef __cplusplus } #endif diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index c51b6e1264..b5c839c811 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -27,41 +27,31 @@ extern "C" { #endif #define UDF_LISTEN_PIPE_NAME_LEN 32 -#define UDF_LISTEN_PIPE_NAME_PREFIX "udf.sock." +#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." //====================================================================================== //begin API to taosd and qworker enum { UDFC_CODE_STOPPING = -1, - UDFC_CODE_RESTARTING = -2, UDFC_CODE_PIPE_READ_ERR = -3, }; -/*TODO: no api for dnode startudfd/stopudfd*/ -/** - * start udfd dameon service - */ -int32_t startUdfd(int32_t dnodeId); - -/** - * stop udfd dameon service - */ -int32_t stopUdfd(int32_t dnodeId); +typedef void *UdfcHandle; +typedef void *UdfcFuncHandle; /** * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf * @return error code */ -int32_t createUdfdProxy(int32_t dnodeId); +int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle); /** * destroy udfd proxy * @return error code */ -int32_t destroyUdfdProxy(int32_t dnodeId); +int32_t udfcClose(UdfcHandle proxyhandle); -typedef void *UdfHandle; /** * setup udf @@ -69,7 +59,7 @@ typedef void *UdfHandle; * @param handle, out * @return error code */ -int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle); +int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle); typedef struct SUdfColumnMeta { int16_t type; @@ -116,26 +106,26 @@ typedef struct SUdfInterBuf { } SUdfInterBuf; // output: interBuf -int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf); +int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf); // input: block, state // output: newState -int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); +int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState); // input: interBuf // output: resultData -int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); +int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData); // input: interbuf1, interbuf2 // output: resultBuf -int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); +int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf); // input: block // output: resultData -int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData); +int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData); /** * tearn down udf * @param handle * @return */ -int32_t teardownUdf(UdfHandle handle); +int32_t teardownUdf(UdfcFuncHandle handle); // end API to taosd and qworker //============================================================================================================================= diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 496f486e05..2c16afbd0d 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -69,6 +69,9 @@ typedef struct SUdfTeardownRequest { typedef struct SUdfTeardownResponse { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SUdfTeardownResponse; typedef struct SUdfRequest { diff --git a/source/libs/function/inc/udfc.h b/source/libs/function/inc/udfc.h index fed2818ced..a693e476e8 100644 --- a/source/libs/function/inc/udfc.h +++ b/source/libs/function/inc/udfc.h @@ -30,20 +30,20 @@ typedef struct SUdfInfo { char *path; } SUdfInfo; -typedef void *UdfHandle; +typedef void *UdfcFuncHandle; int32_t createUdfdProxy(); int32_t destroyUdfdProxy(); -//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfHandle *handles); +//int32_t setupUdf(SUdfInfo *udf, int32_t numOfUdfs, UdfcFuncHandle *handles); -int32_t setupUdf(SUdfInfo* udf, UdfHandle* handle); +int32_t setupUdf(SUdfInfo* udf, UdfcFuncHandle* handle); -int32_t callUdf(UdfHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, +int32_t callUdf(UdfcFuncHandle handle, int8_t step, char *state, int32_t stateSize, SSDataBlock input, char **newstate, int32_t *newStateSize, SSDataBlock *output); -int32_t teardownUdf(UdfHandle handle); +int32_t teardownUdf(UdfcFuncHandle handle); typedef struct SUdfSetupRequest { char udfName[16]; // @@ -84,6 +84,9 @@ typedef struct SUdfTeardownRequest { typedef struct SUdfTeardownResponse { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SUdfTeardownResponse; typedef struct SUdfRequest { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index ce7384cbbe..efd8de84c4 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -193,8 +193,15 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return TSDB_CODE_SUCCESS; } +static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + // pseudo column do not need to check parameters + pFunc->node.resType = (SDataType){.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - // todo + SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type}; return TSDB_CODE_SUCCESS; } @@ -497,9 +504,9 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_TOP, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateTop, - .getEnvFunc = getMinmaxFuncEnv, - .initFunc = maxFunctionSetup, - .processFunc = maxFunction, + .getEnvFunc = getTopBotFuncEnv, + .initFunc = functionSetup, + .processFunc = topFunction, .finalizeFunc = functionFinalize }, { @@ -876,7 +883,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "tbname", .type = FUNCTION_TYPE_TBNAME, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC, - .translateFunc = NULL, + .translateFunc = translateTbnameColumn, .getEnvFunc = NULL, .initFunc = NULL, .sprocessFunc = NULL, @@ -914,7 +921,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { }, { .name = "_wendts", - .type = FUNCTION_TYPE_QENDTS, + .type = FUNCTION_TYPE_WENDTS, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3d796515a0..0942726588 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,10 +14,11 @@ */ #include "builtinsimpl.h" -#include "tpercentile.h" +#include #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" +#include "tpercentile.h" #define SET_VAL(_info, numOfElem, res) \ do { \ @@ -472,17 +473,6 @@ int32_t maxFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -typedef struct STopBotRes { - int32_t num; -} STopBotRes; - -bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0); - int32_t bytes = pColNode->node.resType.bytes; - SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); - return true; -} - typedef struct SStddevRes { double result; int64_t count; @@ -523,7 +513,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { switch (type) { case TSDB_DATA_TYPE_TINYINT: { int8_t* plist = (int8_t*)pCol->pData; - for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + for (int32_t i = start; i < numOfRows + start; ++i) { if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { continue; } @@ -749,9 +739,9 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -// TODO set the correct parameter. void percentileFinalize(SqlFunctionCtx* pCtx) { - double v = 50;//pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64 : pCtx->param[0].dKey; + SVariant* pVal = &pCtx->param[1].param; + double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); @@ -1173,3 +1163,130 @@ int32_t diffFunction(SqlFunctionCtx *pCtx) { } } +typedef struct STopBotResItem { + SVariant v; + uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data + struct { + int32_t pageId; + int32_t offset; + } tuplePos; // tuple data of this chosen row +} STopBotResItem; + +typedef struct STopBotRes { + int32_t num; + STopBotResItem *pItems; +} STopBotRes; + +bool getTopBotFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + SColumnNode* pColNode = (SColumnNode*) nodesListGetNode(pFunc->pParameterList, 0); + int32_t bytes = pColNode->node.resType.bytes; + SValueNode* pkNode = (SValueNode*) nodesListGetNode(pFunc->pParameterList, 1); + + pEnv->calcMemSize = sizeof(STopBotRes) + pkNode->datum.i * bytes; + return true; +} + +static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + STopBotRes* pRes = GET_ROWCELL_INTERBUF(pResInfo); + pRes->pItems = (STopBotResItem*)((char*) pRes + sizeof(STopBotRes)); + + return pRes; +} + +static void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid); + +int32_t topFunction(SqlFunctionCtx *pCtx) { + int32_t numOfElems = 0; + + STopBotRes *pRes = getTopBotOutputInfo(pCtx); + assert(pRes->num >= 0); + +// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { +// buildTopBotStruct(pRes, pCtx); +// } + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pCol = pInput->pData[0]; + + int32_t type = pInput->pData[0]->info.type; + + int32_t start = pInput->startRowIndex; + int32_t numOfRows = pInput->numOfRows; + + for (int32_t i = start; i < numOfRows + start; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + numOfElems++; + + char* data = colDataGetData(pCol, i); + doAddIntoResult(pRes, pCtx->param[1].param.i, data, type, pInput->uid); + } + + // treat the result as only one result + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + return TSDB_CODE_SUCCESS; +} + +static int32_t topBotResComparFn(const void *p1, const void *p2, const void *param) { + uint16_t type = *(uint16_t *) param; + + STopBotResItem *val1 = (STopBotResItem *) p1; + STopBotResItem *val2 = (STopBotResItem *) p2; + + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (val1->v.i == val2->v.i) { + return 0; + } + + return (val1->v.i > val2->v.i) ? 1 : -1; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (val1->v.u == val2->v.u) { + return 0; + } + + return (val1->v.u > val2->v.u) ? 1 : -1; + } + + if (val1->v.d == val2->v.d) { + return 0; + } + + return (val1->v.d > val2->v.d) ? 1 : -1; +} + +void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, uint16_t type, uint64_t uid) { + SVariant val = {0}; + taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); + + STopBotResItem *pItems = pRes->pItems; + assert(pItems != NULL); + + // not full yet + if (pRes->num < maxSize) { + STopBotResItem* pItem = &pItems[pRes->num]; + pItem->v = val; + pItem->uid = uid; + pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer + + pRes->num++; + taosheapsort((void *) pItem, sizeof(STopBotResItem), pRes->num, (const void *) &type, topBotResComparFn, false); + } else { // replace the minimum value in the result + if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || + (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || + (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { + STopBotResItem* pItem = &pItems[pRes->num]; + pItem->v = val; + pItem->uid = uid; + pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer + + taosheapadjust((void *) pItem, sizeof(STopBotResItem), 0, pRes->num - 1, (const void *) &type, topBotResComparFn, NULL, false); + } + } +} + +void topBotFinalize(SqlFunctionCtx* pCtx) { + functionFinalize(pCtx); + +} \ No newline at end of file diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 130b81a609..d44e3e251b 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -29,6 +29,7 @@ typedef struct SFuncMgtService { typedef struct SUdfInfo { SDataType outputDt; + int8_t funcType; } SUdfInfo; static SFuncMgtService gFunMgtService; @@ -52,30 +53,41 @@ static void doInitFunctionTable() { gFunMgtService.pUdfTable = NULL; } +static int8_t getUdfType(int32_t funcId) { + SUdfInfo* pUdf = taosArrayGet(gFunMgtService.pUdfTable, funcId - FUNC_UDF_ID_START_OFFSET_VAL - 1); + return pUdf->funcType; +} + static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) { + if (fmIsUserDefinedFunc(funcId)) { + return getUdfType(funcId); + } if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; } return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification); } -static int32_t getUdfId(const char* pFuncName) { - // todo: udf by call catalog - if (1) { +static int32_t getUdfId(SFmGetFuncInfoParam* pParam, const char* pFuncName) { + SFuncInfo* pInfo = NULL; + int32_t code = catalogGetUdfInfo(pParam->pCtg, pParam->pRpc, pParam->pMgmtEps, pFuncName, &pInfo); + if (TSDB_CODE_SUCCESS != code || NULL == pInfo) { return -1; } if (NULL == gFunMgtService.pUdfTable) { gFunMgtService.pUdfTable = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SUdfInfo)); } - SUdfInfo info = {0}; //todo + SUdfInfo info = { .outputDt.type = pInfo->outputType, .outputDt.bytes = pInfo->outputLen, .funcType = pInfo->funcType }; taosArrayPush(gFunMgtService.pUdfTable, &info); + tFreeSFuncInfo(pInfo); + taosMemoryFree(pInfo); return taosArrayGetSize(gFunMgtService.pUdfTable) + FUNC_UDF_ID_START_OFFSET_VAL; } -static int32_t getFuncId(const char* pFuncName) { +static int32_t getFuncId(SFmGetFuncInfoParam* pParam, const char* pFuncName) { void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName)); if (NULL == pVal) { - return getUdfId(pFuncName); + return getUdfId(pParam, pFuncName); } return *(int32_t*)pVal; } @@ -91,8 +103,8 @@ int32_t fmFuncMgtInit() { return initFunctionCode; } -int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { - *pFuncId = getFuncId(pFuncName); +int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { + *pFuncId = getFuncId(pParam, pFuncName); if (*pFuncId < 0) { return TSDB_CODE_FAILED; } diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 56ee9bc9ae..c26342bfa8 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -765,9 +765,9 @@ static int32_t firstFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t c } static int32_t lastFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i) { - return BLK_DATA_NOT_LOAD; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return BLK_DATA_NOT_LOAD; +// } if (GET_RES_INFO(pCtx) == NULL || GET_RES_INFO(pCtx)->numOfRes <= 0) { return BLK_DATA_DATA_LOAD; @@ -797,9 +797,9 @@ static int32_t firstDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32 } static int32_t lastDistFuncRequired(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { - if (pCtx->order != pCtx->param[0].i) { - return BLK_DATA_NOT_LOAD; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return BLK_DATA_NOT_LOAD; +// } // not initialized yet, it is the first block, load it. if (pCtx->pOutput == NULL) { @@ -1261,128 +1261,6 @@ int32_t tsCompare(const void* p1, const void* p2) { } } -static void stddev_dst_function(SqlFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - // the second stage to calculate standard deviation - double *retVal = &pStd->res; - - // all data are null, no need to proceed - SArray* resList = (SArray*) pCtx->param[0].pz; - if (resList == NULL) { - return; - } - - // find the correct group average results according to the tag value - int32_t len = (int32_t) taosArrayGetSize(resList); - assert(len > 0); - - double avg = 0; - if (len == 1) { - SResPair* p = taosArrayGet(resList, 0); - avg = p->avg; - } else { // todo opt performance by using iterator since the timestamp lsit is matched with the output result - SResPair* p = bsearch(&pCtx->startTs, resList->pData, len, sizeof(SResPair), tsCompare); - if (p == NULL) { - return; - } - - avg = p->avg; - } - - void *pData = GET_INPUT_DATA_LIST(pCtx); - int32_t num = 0; - - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: { - for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { - continue; - } - num += 1; - *retVal += TPOW2(((int32_t *)pData)[i] - avg); - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - LOOP_STDDEV_IMPL(float, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - LOOP_STDDEV_IMPL(double, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_TINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UTINYINT: { - LOOP_STDDEV_IMPL(int8_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - LOOP_STDDEV_IMPL(int16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - LOOP_STDDEV_IMPL(uint16_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UINT: { - LOOP_STDDEV_IMPL(uint32_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_BIGINT: { - LOOP_STDDEV_IMPL(int64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - LOOP_STDDEV_IMPL(uint64_t, *retVal, pData, pCtx, avg, pCtx->inputType, num); - break; - } - default: - assert(0); -// qError("stddev function not support data type:%d", pCtx->inputType); - } - - pStd->num += num; - SET_VAL(pCtx, num, 1); - - // copy to the final output buffer for super table - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)), sizeof(SAvgInfo)); -} - -static void stddev_dst_merge(SqlFunctionCtx *pCtx) { - SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevdstInfo* pRes = GET_ROWCELL_INTERBUF(pResInfo); - - char *input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SStddevdstInfo *pInput = (SStddevdstInfo *)input; - if (pInput->num == 0) { // current input is null - continue; - } - - pRes->num += pInput->num; - pRes->res += pInput->res; - } -} - -static void stddev_dst_finalizer(SqlFunctionCtx *pCtx) { - SStddevdstInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - - if (pStd->num <= 0) { - setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); - } else { - double *retValue = (double *)pCtx->pOutput; - SET_DOUBLE_VAL(retValue, sqrt(pStd->res / pStd->num)); - SET_VAL(pCtx, 1, 1); - } - - doFinalizer(pCtx); -} - ////////////////////////////////////////////////////////////////////////////////////// static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo) { if (!function_setup(pCtx, pResInfo)) { @@ -1390,8 +1268,8 @@ static bool first_last_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* } // used to keep the timestamp for comparison - pCtx->param[1].nType = 0; - pCtx->param[1].i = 0; +// pCtx->param[1].param.nType = 0; +// pCtx->param[1].param.i = 0; return true; } @@ -1487,13 +1365,13 @@ static void first_dist_func_merge(SqlFunctionCtx *pCtx) { } // The param[1] is used to keep the initial value of max ts value - if (pCtx->param[1].nType != pCtx->resDataInfo.type || pCtx->param[1].i > pInput->ts) { - memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); - pCtx->param[1].i = pInput->ts; - pCtx->param[1].nType = pCtx->resDataInfo.type; - -// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); - } +// if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i > pInput->ts) { +// memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); +// pCtx->param[1].param.i = pInput->ts; +// pCtx->param[1].param.nType = pCtx->resDataInfo.type; +// +//// DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); +// } SET_VAL(pCtx, 1, 1); // GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; @@ -1508,9 +1386,9 @@ static void first_dist_func_merge(SqlFunctionCtx *pCtx) { * least one data in this block that is not null.(TODO opt for this case) */ static void last_function(SqlFunctionCtx *pCtx) { - if (pCtx->order != pCtx->param[0].i) { - return; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return; +// } SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -1582,9 +1460,9 @@ static void last_dist_function(SqlFunctionCtx *pCtx) { * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data */ - if (pCtx->order != pCtx->param[0].i) { - return; - } +// if (pCtx->order != pCtx->param[0].param.i) { +// return; +// } int32_t notNullElems = 0; for (int32_t i = pCtx->size - 1; i >= 0; --i) { @@ -1624,10 +1502,10 @@ static void last_dist_func_merge(SqlFunctionCtx *pCtx) { * param[1] used to keep the corresponding timestamp to decide if current result is * the true last result */ - if (pCtx->param[1].nType != pCtx->resDataInfo.type || pCtx->param[1].i < pInput->ts) { + if (pCtx->param[1].param.nType != pCtx->resDataInfo.type || pCtx->param[1].param.i < pInput->ts) { memcpy(pCtx->pOutput, pData, pCtx->resDataInfo.bytes); - pCtx->param[1].i = pInput->ts; - pCtx->param[1].nType = pCtx->resDataInfo.type; + pCtx->param[1].param.i = pInput->ts; + pCtx->param[1].param.nType = pCtx->resDataInfo.type; // DO_UPDATE_TAG_COLUMNS(pCtx, pInput->ts); } @@ -1955,11 +1833,11 @@ static STopBotInfo *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SqlFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); pTopBotInfo->res = (tValuePair**) tmp; - tmp += POINTER_BYTES * pCtx->param[0].i; +// tmp += POINTER_BYTES * pCtx->param[0].param.i; // size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; -// for (int32_t i = 0; i < pCtx->param[0].i; ++i) { +// for (int32_t i = 0; i < pCtx->param[0].param.i; ++i) { // pTopBotInfo->res[i] = (tValuePair*) tmp; // pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); // tmp += size; @@ -1975,13 +1853,13 @@ bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const cha STopBotInfo *pTopBotInfo = getTopBotOutputInfo(pCtx); // required number of results are not reached, continue load data block - if (pTopBotInfo->num < pCtx->param[0].i) { - return true; - } +// if (pTopBotInfo->num < pCtx->param[0].param.i) { +// return true; +// } - if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i)) { - buildTopBotStruct(pTopBotInfo, pCtx); - } +// if ((void *)pTopBotInfo->res[0] != (void *)((char *)pTopBotInfo + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { +// buildTopBotStruct(pTopBotInfo, pCtx); +// } tValuePair **pRes = (tValuePair**) pTopBotInfo->res; @@ -2038,9 +1916,9 @@ static void top_function(SqlFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); assert(pRes->num >= 0); - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i)) { - buildTopBotStruct(pRes, pCtx); - } +// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { +// buildTopBotStruct(pRes, pCtx); +// } for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2052,7 +1930,7 @@ static void top_function(SqlFunctionCtx *pCtx) { // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_top_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_top_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2079,7 +1957,7 @@ static void top_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->resDataInfo.type; -// do_top_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, +// do_top_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, // type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2096,9 +1974,9 @@ static void bottom_function(SqlFunctionCtx *pCtx) { STopBotInfo *pRes = getTopBotOutputInfo(pCtx); - if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].i)) { - buildTopBotStruct(pRes, pCtx); - } +// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotInfo) + POINTER_BYTES * pCtx->param[0].param.i)) { +// buildTopBotStruct(pRes, pCtx); +// } for (int32_t i = 0; i < pCtx->size; ++i) { char *data = GET_INPUT_DATA(pCtx, i); @@ -2109,7 +1987,7 @@ static void bottom_function(SqlFunctionCtx *pCtx) { notNullElems++; // NOTE: Set the default timestamp if it is missing [todo refactor] TSKEY ts = (pCtx->ptsList != NULL)? GET_TS_DATA(pCtx, i):0; -// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); +// do_bottom_function_add(pRes, (int32_t)pCtx->param[0].param.i, data, ts, pCtx->inputType, &pCtx->tagInfo, NULL, 0); } if (!pCtx->hasNull) { @@ -2136,7 +2014,7 @@ static void bottom_func_merge(SqlFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { int16_t type = (pCtx->resDataInfo.type == TSDB_DATA_TYPE_FLOAT) ? TSDB_DATA_TYPE_DOUBLE : pCtx->resDataInfo.type; -// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, +// do_bottom_function_add(pOutput, (int32_t)pCtx->param[0].param.i, &pInput->res[i]->v.i, pInput->res[i]->timestamp, type, // &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } @@ -2162,11 +2040,11 @@ static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) { tValuePair **tvp = pRes->res; // user specify the order of output by sort the result according to timestamp - if (pCtx->param[1].i == PRIMARYKEY_TIMESTAMP_COL_ID) { - __compar_fn_t comparator = (pCtx->param[2].i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; + if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) { + __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); - } else /*if (pCtx->param[1].i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { - __compar_fn_t comparator = (pCtx->param[2].i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; + } else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ { + __compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } @@ -2277,7 +2155,7 @@ static void percentile_function(SqlFunctionCtx *pCtx) { } static void percentile_finalizer(SqlFunctionCtx *pCtx) { - double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i : pCtx->param[0].d; +// double v = pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].param.i : pCtx->param[0].param.d; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); @@ -2287,7 +2165,7 @@ static void percentile_finalizer(SqlFunctionCtx *pCtx) { assert(ppInfo->numOfElems == 0); setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else { - SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); +// SET_DOUBLE_VAL((double *)pCtx->pOutput, getPercentile(pMemBucket, v)); } tMemBucketDestroy(pMemBucket); @@ -2389,7 +2267,7 @@ static void apercentile_func_merge(SqlFunctionCtx *pCtx) { } static void apercentile_finalizer(SqlFunctionCtx *pCtx) { - double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i : pCtx->param[0].d; + double v = (pCtx->param[0].param.nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].param.i : pCtx->param[0].param.d; SResultRowEntryInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); @@ -2432,7 +2310,7 @@ static bool leastsquares_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInf SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // 2*3 matrix - pInfo->startVal = pCtx->param[0].d; +// pInfo->startVal = pCtx->param[0].param.d; return true; } @@ -2478,54 +2356,54 @@ static void leastsquares_function(SqlFunctionCtx *pCtx) { param[0][2] += x * p[i]; param[1][2] += p[i]; - x += pCtx->param[1].d; + x += pCtx->param[1].param.d; numOfElem++; } break; } case TSDB_DATA_TYPE_BIGINT: { int64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_DOUBLE: { double *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_FLOAT: { float *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; }; case TSDB_DATA_TYPE_SMALLINT: { int16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_TINYINT: { int8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_UTINYINT: { uint8_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_USMALLINT: { uint16_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_UINT: { uint32_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } case TSDB_DATA_TYPE_UBIGINT: { uint64_t *p = pData; - LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].d); + LEASTSQR_CAL_LOOP(pCtx, param, x, p, pCtx->inputType, numOfElem, pCtx->param[1].param.d); break; } } @@ -2584,16 +2462,16 @@ static void col_project_function(SqlFunctionCtx *pCtx) { } // only one row is required. - if (pCtx->param[0].i == 1) { - SET_VAL(pCtx, pCtx->size, 1); - } else { - INC_INIT_VAL(pCtx, pCtx->size); - } +// if (pCtx->param[0].param.i == 1) { +// SET_VAL(pCtx, pCtx->size, 1); +// } else { +// INC_INIT_VAL(pCtx, pCtx->size); +// } char *pData = GET_INPUT_DATA_LIST(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - int32_t numOfRows = (pCtx->param[0].i == 1)? 1:pCtx->size; - memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); +// int32_t numOfRows = (pCtx->param[0].param.i == 1)? 1:pCtx->size; +// memcpy(pCtx->pOutput, pData, (size_t) numOfRows * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->pOutput + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, @@ -2658,7 +2536,7 @@ static bool diff_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResI } // diff function require the value is set to -1 - pCtx->param[1].nType = INITIAL_VALUE_NOT_ASSIGNED; + pCtx->param[1].param.nType = INITIAL_VALUE_NOT_ASSIGNED; return false; } @@ -2670,9 +2548,9 @@ static bool deriv_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pRes // diff function require the value is set to -1 SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo); - pDerivInfo->ignoreNegative = pCtx->param[1].i; +// pDerivInfo->ignoreNegative = pCtx->param[1].param.i; pDerivInfo->prevTs = -1; - pDerivInfo->tsWindow = pCtx->param[0].i; +// pDerivInfo->tsWindow = pCtx->param[0].param.i; pDerivInfo->valueSet = false; return false; } @@ -2861,12 +2739,12 @@ static void deriv_function(SqlFunctionCtx *pCtx) { #define DIFF_IMPL(ctx, d, type) \ do { \ - if ((ctx)->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED) { \ - (ctx)->param[1].nType = (ctx)->inputType; \ - *(type *)&(ctx)->param[1].i = *(type *)(d); \ + if ((ctx)->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED) { \ + (ctx)->param[1].param.nType = (ctx)->inputType; \ + *(type *)&(ctx)->param[1].param.i = *(type *)(d); \ } else { \ - *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].i)); \ - *(type *)(&(ctx)->param[1].i) = *(type *)(d); \ + *(type *)(ctx)->pOutput = *(type *)(d) - (*(type *)(&(ctx)->param[1].param.i)); \ + *(type *)(&(ctx)->param[1].param.i) = *(type *)(d); \ *(int64_t *)(ctx)->pTsOutput = GET_TS_DATA(ctx, index); \ } \ } while (0); @@ -2874,7 +2752,7 @@ static void deriv_function(SqlFunctionCtx *pCtx) { // TODO difference in date column static void diff_function(SqlFunctionCtx *pCtx) { void *data = GET_INPUT_DATA_LIST(pCtx); - bool isFirstBlock = (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED); + bool isFirstBlock = (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED); int32_t notNullElems = 0; @@ -2894,15 +2772,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int32_t)(pData[i] - pCtx->param[1].i); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (int32_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2916,15 +2794,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = pData[i] - pCtx->param[1].i; // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = pData[i] - pCtx->param[1].param.i; // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2938,15 +2816,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].d); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + SET_DOUBLE_VAL(pOutput, pData[i] - pCtx->param[1].param.d); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].d = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.d = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2960,15 +2838,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (float)(pData[i] - pCtx->param[1].d); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (float)(pData[i] - pCtx->param[1].param.d); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].d = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.d = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -2982,15 +2860,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int16_t)(pData[i] - pCtx->param[1].i); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (int16_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -3005,15 +2883,15 @@ static void diff_function(SqlFunctionCtx *pCtx) { continue; } - if (pCtx->param[1].nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet - *pOutput = (int8_t)(pData[i] - pCtx->param[1].i); // direct previous may be null + if (pCtx->param[1].param.nType != INITIAL_VALUE_NOT_ASSIGNED) { // initial value is not set yet + *pOutput = (int8_t)(pData[i] - pCtx->param[1].param.i); // direct previous may be null *pTimestamp = (tsList != NULL)? tsList[i]:0; pOutput += 1; pTimestamp += 1; } - pCtx->param[1].i = pData[i]; - pCtx->param[1].nType = pCtx->inputType; + pCtx->param[1].param.i = pData[i]; + pCtx->param[1].param.nType = pCtx->inputType; notNullElems++; } break; @@ -3024,7 +2902,7 @@ static void diff_function(SqlFunctionCtx *pCtx) { } // initial value is not set yet - if (pCtx->param[1].nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { + if (pCtx->param[1].param.nType == INITIAL_VALUE_NOT_ASSIGNED || notNullElems <= 0) { /* * 1. current block and blocks before are full of null * 2. current block may be null value @@ -3091,8 +2969,8 @@ static bool spread_function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pRe // this is the server-side setup function in client-side, the secondary merge do not need this procedure if (pCtx->currentStage == MERGE_STAGE) { - pCtx->param[0].d = DBL_MAX; - pCtx->param[3].d = -DBL_MAX; +// pCtx->param[0].param.d = DBL_MAX; +// pCtx->param[3].param.d = -DBL_MAX; } else { pInfo->min = DBL_MAX; pInfo->max = -DBL_MAX; @@ -3192,13 +3070,13 @@ void spread_func_merge(SqlFunctionCtx *pCtx) { return; } - if (pCtx->param[0].d > pData->min) { - pCtx->param[0].d = pData->min; - } +// if (pCtx->param[0].param.d > pData->min) { +// pCtx->param[0].param.d = pData->min; +// } - if (pCtx->param[3].d < pData->max) { - pCtx->param[3].d = pData->max; - } +// if (pCtx->param[3].param.d < pData->max) { +// pCtx->param[3].param.d = pData->max; +// } // GET_RES_INFO(pCtx)->hasResult = DATA_SET_FLAG; } @@ -3218,7 +3096,7 @@ void spread_function_finalizer(SqlFunctionCtx *pCtx) { // return; // } - SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].d - pCtx->param[0].d); +// SET_DOUBLE_VAL((double *)pCtx->pOutput, pCtx->param[3].param.d - pCtx->param[0].param.d); } else { assert(IS_NUMERIC_TYPE(pCtx->inputType) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); @@ -3571,7 +3449,7 @@ void twa_function_finalizer(SqlFunctionCtx *pCtx) { */ static void interp_function_impl(SqlFunctionCtx *pCtx) { - int32_t type = (int32_t) pCtx->param[2].i; + int32_t type = (int32_t) pCtx->param[2].param.i; if (type == TSDB_FILL_NONE) { return; } @@ -3583,7 +3461,7 @@ static void interp_function_impl(SqlFunctionCtx *pCtx) { } else if (type == TSDB_FILL_NULL) { setNull(pCtx->pOutput, pCtx->resDataInfo.type, pCtx->resDataInfo.bytes); } else if (type == TSDB_FILL_SET_VALUE) { - taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); +// taosVariantDump(&pCtx->param[1], pCtx->pOutput, pCtx->inputType, true); } else { if (pCtx->start.key != INT64_MIN && ((ascQuery && pCtx->start.key <= pCtx->startTs && pCtx->end.key >= pCtx->startTs) || ((!ascQuery) && pCtx->start.key >= pCtx->startTs && pCtx->end.key <= pCtx->startTs))) { if (type == TSDB_FILL_PREV) { @@ -3755,11 +3633,11 @@ static void ts_comp_function(SqlFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); +// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_DATA(pCtx, i); - tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); +// tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].param.i, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -3911,7 +3789,7 @@ static void rate_finalizer(SqlFunctionCtx *pCtx) { return; } - SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].i))); +// SET_DOUBLE_VAL((double*) pCtx->pOutput, do_calc_rate(pRateInfo, (double) TSDB_TICK_PER_SECOND(pCtx->param[0].param.i))); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; @@ -4008,7 +3886,7 @@ static void blockInfo_func(SqlFunctionCtx* pCtx) { int32_t len = *(int32_t*) pCtx->pInput; blockDistInfoFromBinary((char*)pCtx->pInput + sizeof(int32_t), len, pDist); - pDist->rowSize = (uint16_t)pCtx->param[0].i; +// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; memcpy(pCtx->pOutput, pCtx->pInput, sizeof(int32_t) + len); @@ -4160,7 +4038,7 @@ void blockinfo_func_finalizer(SqlFunctionCtx* pCtx) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDistInfo* pDist = (STableBlockDistInfo*) GET_ROWCELL_INTERBUF(pResInfo); - pDist->rowSize = (uint16_t)pCtx->param[0].i; +// pDist->rowSize = (uint16_t)pCtx->param[0].param.i; generateBlockDistResult(pDist, pCtx->pOutput); if (pDist->dataBlockInfos != NULL) { @@ -4557,9 +4435,9 @@ SAggFunctionInfo aggFunc[35] = {{ FUNCTION_AVG, FUNCSTATE_SO | FUNCSTATE_STABLE, function_setup, - stddev_dst_function, - stddev_dst_finalizer, - stddev_dst_merge, + NULL, + NULL, + NULL, dataBlockRequired, }, { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ad74daddc6..e31a860e85 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -14,19 +14,15 @@ */ #include "uv.h" #include "os.h" -#include "tlog.h" #include "tudf.h" #include "tudfInt.h" #include "tarray.h" #include "tdatablock.h" -//TODO: when startup, set thread poll size. add it to cfg -//TODO: test for udfd restart -//TODO: udfd restart when exist or aborts -//TODO: deal with uv task that has been started and then udfd core dumped //TODO: network error processing. //TODO: add unit test //TODO: include all global variable under context struct + /* Copyright (c) 2013, Ben Noordhuis * The QUEUE is copied from queue.h under libuv * */ @@ -125,12 +121,35 @@ enum { UV_TASK_DISCONNECT = 2 }; +int64_t gUdfTaskSeqNum = 0; +typedef struct SUdfdProxy { + int32_t dnodeId; + uv_barrier_t gUdfInitBarrier; + + uv_loop_t gUdfdLoop; + uv_thread_t gUdfLoopThread; + uv_async_t gUdfLoopTaskAync; + + uv_async_t gUdfLoopStopAsync; + + uv_mutex_t gUdfTaskQueueMutex; + int8_t gUdfcState; + QUEUE gUdfTaskQueue; + QUEUE gUvProcTaskQueue; + // int8_t gUdfcState = UDFC_STATE_INITAL; + // QUEUE gUdfTaskQueue = {0}; + // QUEUE gUvProcTaskQueue = {0}; +} SUdfdProxy; + + typedef struct SUdfUvSession { + SUdfdProxy *udfc; int64_t severHandle; uv_pipe_t *udfSvcPipe; } SUdfUvSession; typedef struct SClientUvTaskNode { + SUdfdProxy *udfc; int8_t type; int errCode; @@ -169,7 +188,6 @@ typedef struct SClientUdfTask { } _teardown; }; - } SClientUdfTask; typedef struct SClientConnBuf { @@ -185,34 +203,13 @@ typedef struct SClientUvConn { SClientConnBuf readBuf; } SClientUvConn; -uv_process_t gUdfdProcess; - -uv_barrier_t gUdfInitBarrier; - -uv_loop_t gUdfdLoop; -uv_thread_t gUdfLoopThread; -uv_async_t gUdfLoopTaskAync; - -uv_async_t gUdfLoopStopAsync; - -uv_mutex_t gUdfTaskQueueMutex; -int64_t gUdfTaskSeqNum = 0; - enum { UDFC_STATE_INITAL = 0, // initial state - UDFC_STATE_STARTNG, // starting after createUdfdProxy + UDFC_STATE_STARTNG, // starting after udfcOpen UDFC_STATE_READY, // started and begin to receive quests - UDFC_STATE_RESTARTING, // udfd abnormal exit. cleaning up and restart. - UDFC_STATE_STOPPING, // stopping after destroyUdfdProxy + UDFC_STATE_STOPPING, // stopping after udfcClose UDFC_STATUS_FINAL, // stopped }; -int8_t gUdfcState = UDFC_STATE_INITAL; - -//double circular linked list - -QUEUE gUdfTaskQueue = {0}; - -QUEUE gUvProcTaskQueue = {0}; int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) { int32_t len = 0; @@ -777,13 +774,14 @@ void onUdfClientConnect(uv_connect_t *connect, int status) { int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskNode **pUvTask) { SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode)); uvTask->type = uvTaskType; + uvTask->udfc = task->session->udfc; if (uvTaskType == UV_TASK_CONNECT) { } else if (uvTaskType == UV_TASK_REQ_RSP) { uvTask->pipe = task->session->udfSvcPipe; SUdfRequest request; request.type = task->type; - request.seqNum = gUdfTaskSeqNum++; + request.seqNum = atomic_fetch_add_64(&gUdfTaskSeqNum, 1); if (task->type == UDF_TASK_SETUP) { request.setup = task->_setup.req; @@ -815,11 +813,11 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { debugPrint("%s, %d", "queue uv task", uvTask->type); - - uv_mutex_lock(&gUdfTaskQueueMutex); - QUEUE_INSERT_TAIL(&gUdfTaskQueue, &uvTask->recvTaskQueue); - uv_mutex_unlock(&gUdfTaskQueueMutex); - uv_async_send(&gUdfLoopTaskAync); + SUdfdProxy *udfc = uvTask->udfc; + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); + uv_async_send(&udfc->gUdfLoopTaskAync); uv_sem_wait(&uvTask->taskSem); uv_sem_destroy(&uvTask->taskSem); @@ -832,7 +830,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); - uv_pipe_init(&gUdfdLoop, pipe, 0); + uv_pipe_init(&uvTask->udfc->gUdfdLoop, pipe, 0); uvTask->pipe = pipe; SClientUvConn *conn = taosMemoryMalloc(sizeof(SClientUvConn)); @@ -873,142 +871,98 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { } void udfClientAsyncCb(uv_async_t *async) { + SUdfdProxy *udfc = async->data; QUEUE wq; - uv_mutex_lock(&gUdfTaskQueueMutex); - QUEUE_MOVE(&gUdfTaskQueue, &wq); - uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); while (!QUEUE_EMPTY(&wq)) { QUEUE* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); startUvUdfTask(task); - QUEUE_INSERT_TAIL(&gUvProcTaskQueue, &task->procTaskQueue); + QUEUE_INSERT_TAIL(&udfc->gUvProcTaskQueue, &task->procTaskQueue); } } -void cleanUpUvTasks() { +void cleanUpUvTasks(SUdfdProxy *udfc) { QUEUE wq; - uv_mutex_lock(&gUdfTaskQueueMutex); - QUEUE_MOVE(&gUdfTaskQueue, &wq); - uv_mutex_unlock(&gUdfTaskQueueMutex); + uv_mutex_lock(&udfc->gUdfTaskQueueMutex); + QUEUE_MOVE(&udfc->gUdfTaskQueue, &wq); + uv_mutex_unlock(&udfc->gUdfTaskQueueMutex); while (!QUEUE_EMPTY(&wq)) { QUEUE* h = QUEUE_HEAD(&wq); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue); - if (gUdfcState == UDFC_STATE_STOPPING) { + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { task->errCode = UDFC_CODE_STOPPING; - } else if (gUdfcState == UDFC_STATE_RESTARTING) { - task->errCode = UDFC_CODE_RESTARTING; } uv_sem_post(&task->taskSem); } // TODO: deal with tasks that are waiting result. - while (!QUEUE_EMPTY(&gUvProcTaskQueue)) { - QUEUE* h = QUEUE_HEAD(&gUvProcTaskQueue); + while (!QUEUE_EMPTY(&udfc->gUvProcTaskQueue)) { + QUEUE* h = QUEUE_HEAD(&udfc->gUvProcTaskQueue); QUEUE_REMOVE(h); SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue); - if (gUdfcState == UDFC_STATE_STOPPING) { + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { task->errCode = UDFC_CODE_STOPPING; - } else if (gUdfcState == UDFC_STATE_RESTARTING) { - task->errCode = UDFC_CODE_RESTARTING; } uv_sem_post(&task->taskSem); } } void udfStopAsyncCb(uv_async_t *async) { - cleanUpUvTasks(); - if (gUdfcState == UDFC_STATE_STOPPING) { - uv_stop(&gUdfdLoop); + SUdfdProxy *udfc = async->data; + cleanUpUvTasks(udfc); + if (udfc->gUdfcState == UDFC_STATE_STOPPING) { + uv_stop(&udfc->gUdfdLoop); } } -int32_t udfcSpawnUdfd(); - -void onUdfdExit(uv_process_t *req, int64_t exit_status, int term_signal) { - //TODO: pipe close will be first received - debugPrint("Process exited with status %" PRId64 ", signal %d", exit_status, term_signal); - uv_close((uv_handle_t *) req, NULL); - //TODO: restart the udfd process - if (gUdfcState == UDFC_STATE_STOPPING) { - if (term_signal != SIGINT) { - //TODO: log error - } - } - if (gUdfcState == UDFC_STATE_READY) { - gUdfcState = UDFC_STATE_RESTARTING; - //TODO: asynchronous without blocking. how to do it - //cleanUpUvTasks(); - udfcSpawnUdfd(); - } -} - -int32_t udfcSpawnUdfd() { - //TODO: path - uv_process_options_t options = {0}; - static char path[256] = {0}; - size_t cwdSize; - uv_cwd(path, &cwdSize); - strcat(path, "/udfd"); - char* args[2] = {path, NULL}; - options.args = args; - options.file = path; - options.exit_cb = onUdfdExit; - options.stdio_count = 3; - uv_stdio_container_t child_stdio[3]; - child_stdio[0].flags = UV_IGNORE; - child_stdio[1].flags = UV_INHERIT_FD; - child_stdio[1].data.fd = 1; - child_stdio[2].flags = UV_INHERIT_FD; - child_stdio[2].data.fd = 2; - options.stdio = child_stdio; - //TODO spawn error - int err = uv_spawn(&gUdfdLoop, &gUdfdProcess, &options); - if (err != 0) { - debugPrint("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err)); - } - return err; -} - void constructUdfService(void *argsThread) { - uv_loop_init(&gUdfdLoop); + SUdfdProxy *udfc = (SUdfdProxy*)argsThread; + uv_loop_init(&udfc->gUdfdLoop); - uv_async_init(&gUdfdLoop, &gUdfLoopTaskAync, udfClientAsyncCb); - uv_async_init(&gUdfdLoop, &gUdfLoopStopAsync, udfStopAsyncCb); - uv_mutex_init(&gUdfTaskQueueMutex); - QUEUE_INIT(&gUdfTaskQueue); - QUEUE_INIT(&gUvProcTaskQueue); - uv_barrier_wait(&gUdfInitBarrier); + uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopTaskAync, udfClientAsyncCb); + udfc->gUdfLoopTaskAync.data = udfc; + uv_async_init(&udfc->gUdfdLoop, &udfc->gUdfLoopStopAsync, udfStopAsyncCb); + udfc->gUdfLoopStopAsync.data = udfc; + uv_mutex_init(&udfc->gUdfTaskQueueMutex); + QUEUE_INIT(&udfc->gUdfTaskQueue); + QUEUE_INIT(&udfc->gUvProcTaskQueue); + uv_barrier_wait(&udfc->gUdfInitBarrier); //TODO return value of uv_run - uv_run(&gUdfdLoop, UV_RUN_DEFAULT); - uv_loop_close(&gUdfdLoop); + uv_run(&udfc->gUdfdLoop, UV_RUN_DEFAULT); + uv_loop_close(&udfc->gUdfdLoop); } - -int32_t createUdfdProxy(int32_t dnodeId) { - gUdfcState = UDFC_STATE_STARTNG; - uv_barrier_init(&gUdfInitBarrier, 2); - uv_thread_create(&gUdfLoopThread, constructUdfService, 0); - uv_barrier_wait(&gUdfInitBarrier); gUdfcState = UDFC_STATE_READY; +int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) { + SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy)); + proxy->dnodeId = dnodeId; + proxy->gUdfcState = UDFC_STATE_STARTNG; + uv_barrier_init(&proxy->gUdfInitBarrier, 2); + uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); + uv_barrier_wait(&proxy->gUdfInitBarrier); + proxy->gUdfcState = UDFC_STATE_READY; + *udfc = proxy; return 0; } -int32_t destroyUdfdProxy(int32_t dnodeId) { - gUdfcState = UDFC_STATE_STOPPING; - uv_barrier_destroy(&gUdfInitBarrier); -// if (gUdfcState == UDFC_STATE_STOPPING) { -// uv_process_kill(&gUdfdProcess, SIGINT); -// } - uv_async_send(&gUdfLoopStopAsync); - uv_thread_join(&gUdfLoopThread); - uv_mutex_destroy(&gUdfTaskQueueMutex); - gUdfcState = UDFC_STATUS_FINAL; +int32_t udfcClose(UdfcHandle udfcHandle) { + SUdfdProxy *udfc = udfcHandle; + udfc->gUdfcState = UDFC_STATE_STOPPING; + uv_async_send(&udfc->gUdfLoopStopAsync); + uv_thread_join(&udfc->gUdfLoopThread); + uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); + uv_barrier_destroy(&udfc->gUdfInitBarrier); + udfc->gUdfcState = UDFC_STATUS_FINAL; + taosMemoryFree(udfc); return 0; } @@ -1026,11 +980,12 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { return task->errCode; } -int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) { +int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) { debugPrint("%s", "client setup udf"); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); + task->session->udfc = udfc; task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; @@ -1046,13 +1001,13 @@ int32_t setupUdf(char udfName[], SEpSet *epSet, UdfHandle *handle) { SUdfSetupResponse *rsp = &task->_setup.rsp; task->session->severHandle = rsp->udfHandle; - *handle = task->session; + *funcHandle = task->session; int32_t err = task->errCode; taosMemoryFree(task); return err; } -int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, +int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { debugPrint("%s", "client call udf"); @@ -1121,7 +1076,7 @@ int32_t callUdf(UdfHandle handle, int8_t callType, SSDataBlock *input, SUdfInter } //TODO: translate these calls to callUdf -int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) { +int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { int8_t callType = TSDB_UDF_CALL_AGG_INIT; int32_t err = callUdf(handle, callType, NULL, NULL, NULL, NULL, interBuf); @@ -1131,7 +1086,7 @@ int32_t callUdfAggInit(UdfHandle handle, SUdfInterBuf *interBuf) { // input: block, state // output: interbuf, -int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { +int32_t callUdfAggProcess(UdfcFuncHandle handle, SSDataBlock *block, SUdfInterBuf *state, SUdfInterBuf *newState) { int8_t callType = TSDB_UDF_CALL_AGG_PROC; int32_t err = callUdf(handle, callType, block, state, NULL, NULL, newState); return err; @@ -1139,7 +1094,7 @@ int32_t callUdfAggProcess(UdfHandle handle, SSDataBlock *block, SUdfInterBuf *st // input: interbuf1, interbuf2 // output: resultBuf -int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { +int32_t callUdfAggMerge(UdfcFuncHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf *interBuf2, SUdfInterBuf *resultBuf) { int8_t callType = TSDB_UDF_CALL_AGG_MERGE; int32_t err = callUdf(handle, callType, NULL, interBuf1, interBuf2, NULL, resultBuf); return err; @@ -1147,7 +1102,7 @@ int32_t callUdfAggMerge(UdfHandle handle, SUdfInterBuf *interBuf1, SUdfInterBuf // input: interBuf // output: resultData -int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { +int32_t callUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdfInterBuf *resultData) { int8_t callType = TSDB_UDF_CALL_AGG_PROC; int32_t err = callUdf(handle, callType, NULL, interBuf, NULL, NULL, resultData); return err; @@ -1155,13 +1110,13 @@ int32_t callUdfAggFinalize(UdfHandle handle, SUdfInterBuf *interBuf, SUdfInterBu // input: block // output: resultData -int32_t callUdfScalaProcess(UdfHandle handle, SSDataBlock *block, SSDataBlock *resultData) { +int32_t callUdfScalaProcess(UdfcFuncHandle handle, SSDataBlock *block, SSDataBlock *resultData) { int8_t callType = TSDB_UDF_CALL_SCALA_PROC; int32_t err = callUdf(handle, callType, block, NULL, NULL, resultData, NULL); return err; } -int32_t teardownUdf(UdfHandle handle) { +int32_t teardownUdf(UdfcFuncHandle handle) { debugPrint("%s", "client teardown udf"); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 71434c695f..d6e7a43666 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -27,7 +27,10 @@ typedef struct SUdfdContext { uv_loop_t *loop; + uv_pipe_t ctrlPipe; + uv_signal_t intrSignal; char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; + uv_pipe_t listeningPipe; void *clientRpc; uv_mutex_t udfsMutex; @@ -76,9 +79,9 @@ typedef struct SUdf { // TODO: low priority: change name onxxx to xxxCb, and udfc or udfd as prefix // TODO: add private udf structure. -typedef struct SUdfHandle { +typedef struct SUdfcFuncHandle { SUdf *udf; -} SUdfHandle; +} SUdfcFuncHandle; int32_t udfdLoadUdf(char* udfName, SUdf* udf) { strcpy(udf->name, udfName); @@ -143,7 +146,7 @@ void udfdProcessRequest(uv_work_t *req) { } uv_mutex_unlock(&udf->lock); } - SUdfHandle *handle = taosMemoryMalloc(sizeof(SUdfHandle)); + SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle)); handle->udf = udf; // TODO: allocate private structure and call init function and set it to handle SUdfResponse rsp; @@ -166,7 +169,7 @@ void udfdProcessRequest(uv_work_t *req) { case UDF_TASK_CALL: { SUdfCallRequest *call = &request.call; fnDebug("%"PRId64 "call request. call type %d, handle: %"PRIx64, request.seqNum, call->callType, call->udfHandle); - SUdfHandle *handle = (SUdfHandle *)(call->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle); SUdf *udf = handle->udf; SUdfDataBlock input = {0}; @@ -204,7 +207,7 @@ void udfdProcessRequest(uv_work_t *req) { case UDF_TASK_TEARDOWN: { SUdfTeardownRequest *teardown = &request.teardown; fnInfo("teardown. %"PRId64"handle:%"PRIx64, request.seqNum, teardown->udfHandle) - SUdfHandle *handle = (SUdfHandle *)(teardown->udfHandle); + SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle); SUdf *udf = handle->udf; bool unloadUdf = false; uv_mutex_lock(&global.udfsMutex); @@ -380,10 +383,12 @@ void udfdOnNewConnection(uv_stream_t *server, int status) { } } -void removeListeningPipe(int sig) { +void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { + fnInfo("udfd signal received: %d\n", signum); uv_fs_t req; - uv_fs_unlink(global.loop, &req, "udf.sock", NULL); - exit(0); + uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + uv_signal_stop(handle); + uv_stop(global.loop); } void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } @@ -492,37 +497,67 @@ static int32_t udfdInitLog() { return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0); } +void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { + buf->base = taosMemoryMalloc(suggested_size); + buf->len = suggested_size; +} + +void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) { + if (nread < 0) { + fnError("udfd ctrl pipe read error. %s", uv_err_name(nread)); + uv_close((uv_handle_t*)q, NULL); + uv_stop(global.loop); + return; + } + fnError("udfd ctrl pipe read %zu bytes", nread); + taosMemoryFree(buf->base); +} + +static int32_t removeListeningPipe() { + uv_fs_t req; + int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + uv_fs_req_cleanup(&req); + return err; +} + static int32_t udfdUvInit() { uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t)); if (loop) { uv_loop_init(loop); } global.loop = loop; + + uv_pipe_init(global.loop, &global.ctrlPipe, 1); + uv_pipe_open(&global.ctrlPipe, 0); + uv_read_start((uv_stream_t*)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); + char dnodeId[8] = {0}; size_t dnodeIdSize; - uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); + int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); + if (err != 0) { + dnodeId[0] = '1'; + } char listenPipeName[32] = {0}; snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); strcpy(global.listenPipeName, listenPipeName); - uv_fs_t req; - uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL); + removeListeningPipe(); - uv_pipe_t server; - uv_pipe_init(global.loop, &server, 0); + uv_pipe_init(global.loop, &global.listeningPipe, 0); - signal(SIGINT, removeListeningPipe); + uv_signal_init(global.loop, &global.intrSignal); + uv_signal_start(&global.intrSignal, udfdIntrSignalHandler, SIGINT); int r; fnInfo("bind to pipe %s", global.listenPipeName); - if ((r = uv_pipe_bind(&server, listenPipeName))) { + if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) { fnError("Bind error %s", uv_err_name(r)); - removeListeningPipe(0); + removeListeningPipe(); return -1; } - if ((r = uv_listen((uv_stream_t *)&server, 128, udfdOnNewConnection))) { + if ((r = uv_listen((uv_stream_t *)&global.listeningPipe, 128, udfdOnNewConnection))) { fnError("Listen error %s", uv_err_name(r)); - removeListeningPipe(0); + removeListeningPipe(); return -2; } return 0; diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 41c7f65e0b..fb9c3c678a 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -8,7 +8,8 @@ #include "tdatablock.h" int main(int argc, char *argv[]) { - createUdfdProxy(1); + UdfcHandle udfc; + udfcOpen(1, &udfc); uv_sleep(1000); char path[256] = {0}; size_t cwdSize = 256; @@ -20,9 +21,9 @@ int main(int argc, char *argv[]) { fprintf(stdout, "current working directory:%s\n", path); strcat(path, "/libudf1.so"); - UdfHandle handle; + UdfcFuncHandle handle; SEpSet epSet; - setupUdf("udf1", &epSet, &handle); + setupUdf(udfc, "udf1", &epSet, &handle); SSDataBlock block = {0}; SSDataBlock* pBlock = █ @@ -53,5 +54,5 @@ int main(int argc, char *argv[]) { } teardownUdf(handle); - destroyUdfdProxy(1); + udfcClose(udfc); } diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h index 8fb5455336..2bf9c9b791 100644 --- a/source/libs/index/inc/indexFstRegex.h +++ b/source/libs/index/inc/indexFstRegex.h @@ -31,6 +31,9 @@ extern "C" { typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType; typedef struct MatchValue { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } MatchValue; typedef struct JumpValue { uint32_t step; diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 058c33b725..554b8092a2 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -748,7 +748,7 @@ static SArray* tfileGetFileList(const char* path) { sprintf(buf, "%s/%s", path, file); taosArrayPush(files, &buf); } - taosCloseDir(pDir); + taosCloseDir(&pDir); taosArraySort(files, tfileCompare); tfileRmExpireFile(files); diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index ee9da50ab2..0162b754ee 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -485,7 +485,7 @@ TEST_F(IndexTFileEnv, test_tfile_write) { std::string colVal("ab"); SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM}; SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t)); fObj->Get(&query, result); @@ -625,7 +625,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v3"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM }; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; @@ -640,7 +640,7 @@ TEST_F(IndexCacheEnv, cache_test) { std::string colVal("v2"); SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); - SIndexTermQuery query = {.term = term, .qType = QUERY_TERM}; + SIndexTermQuery query = { term, QUERY_TERM }; SArray* ret = (SArray*)taosArrayInit(4, sizeof(suid)); STermValueType valType; diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 9b0452425a..f789d23136 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -108,7 +108,7 @@ TEST_F(JsonEnv, testWriteMillonData) { { std::string colName("voltagefdadfa"); std::string colVal("abxxxxxxxxxxxx"); - for (uint i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { colVal[i % colVal.size()] = '0' + i % 128; SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size()); diff --git a/source/libs/parser/inc/parToken.h b/source/libs/parser/inc/parToken.h index 1122bafd12..d8a2c22fd8 100644 --- a/source/libs/parser/inc/parToken.h +++ b/source/libs/parser/inc/parToken.h @@ -20,7 +20,6 @@ extern "C" { #endif -#include "os.h" #include "ttokendef.h" // used to denote the minimum unite in sql parsing diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index e68a4389fe..9ff36c702f 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -837,6 +837,8 @@ query_expression(A) ::= query_expression_body(A) ::= query_primary(B). { A = B; } query_expression_body(A) ::= query_expression_body(B) UNION ALL query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, B, D); } +query_expression_body(A) ::= + query_expression_body(B) UNION query_expression_body(D). { A = createSetOperator(pCxt, SET_OP_TYPE_UNION, B, D); } query_primary(A) ::= query_specification(B). { A = B; } //query_primary(A) ::= diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 2eeff04ea2..fcc15b197f 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parInt.h" #include "parAst.h" diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index f21a738032..045f54cdb1 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parInsertData.h" #include "parInt.h" #include "parToken.h" diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7a8db2df50..40860aaaf8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -49,12 +49,14 @@ static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_B static bool beforeHaving(ESqlClause clause) { return clause < SQL_CLAUSE_HAVING; } -#define generateDealNodeErrMsg(pCxt, code, ...) \ - ({ \ - generateSyntaxErrMsg(&pCxt->msgBuf, code, ##__VA_ARGS__); \ - pCxt->errCode = code; \ - DEAL_RES_ERROR; \ - }) +enum EDealRes generateDealNodeErrMsg(STranslateContext* pCxt, int32_t code, ...) { + va_list ap; + va_start(ap, code); + generateSyntaxErrMsg(&pCxt->msgBuf, code, ap); + va_end(ap); + pCxt->errCode = code; + return DEAL_RES_ERROR; +} static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { size_t currTotalLevel = taosArrayGetSize(pCxt->pNsLevel); @@ -230,6 +232,21 @@ static int32_t initTranslateContext(SParseContext* pParseCxt, STranslateContext* return TSDB_CODE_SUCCESS; } +static int32_t resetTranslateNamespace(STranslateContext* pCxt) { + if (NULL != pCxt->pNsLevel) { + size_t size = taosArrayGetSize(pCxt->pNsLevel); + for (size_t i = 0; i < size; ++i) { + taosArrayDestroy(taosArrayGetP(pCxt->pNsLevel, i)); + } + taosArrayDestroy(pCxt->pNsLevel); + } + pCxt->pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + if (NULL == pCxt->pNsLevel) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + static void destroyTranslateContext(STranslateContext* pCxt) { if (NULL != pCxt->pNsLevel) { size_t size = taosArrayGetSize(pCxt->pNsLevel); @@ -261,9 +278,11 @@ static bool belongTable(const char* currentDb, const SColumnNode* pCol, const ST return (0 == cmp); } -static SNodeList* getProjectList(SNode* pNode) { +static SNodeList* getProjectList(const SNode* pNode) { if (QUERY_NODE_SELECT_STMT == nodeType(pNode)) { return ((SSelectStmt*)pNode)->pProjectionList; + } else if (QUERY_NODE_SET_OPERATOR == nodeType(pNode)) { + return ((SSetOperator*)pNode)->pProjectionList; } return NULL; } @@ -568,7 +587,8 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) { } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { + SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); } pCxt->errCode = fmGetFuncResultType(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); @@ -1353,13 +1373,77 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } +static SNode* createSetOperProject(SNode* pNode) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + pCol->node.resType = ((SExprNode*)pNode)->resType; + strcpy(pCol->colName, ((SExprNode*)pNode)->aliasName); + strcpy(pCol->node.aliasName, pCol->colName); + return (SNode*)pCol; +} + +static bool dataTypeEqual(const SDataType* l, const SDataType* r) { + return (l->type == r->type && l->bytes == l->bytes && l->precision == r->precision && l->scale == l->scale); +} + +static int32_t createCastFunc(STranslateContext* pCxt, SNode* pExpr, SDataType dt, SNode** pCast) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pFunc->functionName, "cast"); + pFunc->node.resType = dt; + if (TSDB_CODE_SUCCESS != nodesListMakeAppend(&pFunc->pParameterList, pExpr)) { + nodesDestroyNode(pFunc); + return TSDB_CODE_OUT_OF_MEMORY; + } + if (DEAL_RES_ERROR == translateFunction(pCxt, pFunc)) { + nodesClearList(pFunc->pParameterList); + pFunc->pParameterList = NULL; + nodesDestroyNode(pFunc); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)pExpr)->aliasName); + } + *pCast = (SNode*)pFunc; + return TSDB_CODE_SUCCESS; +} + static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* pSetOperator) { - // todo + SNodeList* pLeftProjections = getProjectList(pSetOperator->pLeft); + SNodeList* pRightProjections = getProjectList(pSetOperator->pRight); + if (LIST_LENGTH(pLeftProjections) != LIST_LENGTH(pRightProjections)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INCORRECT_NUM_OF_COL); + } + + SNode* pLeft = NULL; + SNode* pRight = NULL; + FORBOTH(pLeft, pLeftProjections, pRight, pRightProjections) { + SExprNode* pLeftExpr = (SExprNode*)pLeft; + SExprNode* pRightExpr = (SExprNode*)pRight; + if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) { + SNode* pRightFunc = NULL; + int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + REPLACE_LIST2_NODE(pRightFunc); + pRightExpr = (SExprNode*)pRightFunc; + } + strcpy(pRightExpr->aliasName, pLeftExpr->aliasName); + pRightExpr->aliasName[strlen(pLeftExpr->aliasName)] = '\0'; + if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, createSetOperProject(pLeft))) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } return TSDB_CODE_SUCCESS; } static int32_t translateSetOperator(STranslateContext* pCxt, SSetOperator* pSetOperator) { int32_t code = translateQuery(pCxt, pSetOperator->pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = resetTranslateNamespace(pCxt); + } if (TSDB_CODE_SUCCESS == code) { code = translateQuery(pCxt, pSetOperator->pRight); } @@ -1837,7 +1921,8 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION); } SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0); - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { + SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); } return TSDB_CODE_SUCCESS; @@ -2794,8 +2879,8 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { return code; } -static int32_t extractSelectResultSchema(const SSelectStmt* pSelect, int32_t* numOfCols, SSchema** pSchema) { - *numOfCols = LIST_LENGTH(pSelect->pProjectionList); +static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t* numOfCols, SSchema** pSchema) { + *numOfCols = LIST_LENGTH(pProjections); *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); if (NULL == (*pSchema)) { return TSDB_CODE_OUT_OF_MEMORY; @@ -2803,7 +2888,7 @@ static int32_t extractSelectResultSchema(const SSelectStmt* pSelect, int32_t* nu SNode* pNode; int32_t index = 0; - FOREACH(pNode, pSelect->pProjectionList) { + FOREACH(pNode, pProjections) { SExprNode* pExpr = (SExprNode*)pNode; (*pSchema)[index].type = pExpr->resType.type; (*pSchema)[index].bytes = pExpr->resType.bytes; @@ -2862,7 +2947,8 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS switch (nodeType(pRoot)) { case QUERY_NODE_SELECT_STMT: - return extractSelectResultSchema((SSelectStmt*)pRoot, numOfCols, pSchema); + case QUERY_NODE_SET_OPERATOR: + return extractQueryResultSchema(getProjectList(pRoot), numOfCols, pSchema); case QUERY_NODE_EXPLAIN_STMT: return extractExplainResultSchema(numOfCols, pSchema); case QUERY_NODE_DESCRIBE_STMT: @@ -3485,6 +3571,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { switch (nodeType(pQuery->pRoot)) { case QUERY_NODE_SELECT_STMT: + case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_EXPLAIN_STMT: pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->haveResultSet = true; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 7f945eb70c..075caa868c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -118,6 +118,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "sliding value can not less than 1% of interval value"; case TSDB_CODE_PAR_ONLY_ONE_JSON_TAG: return "Only one tag if there is a json tag"; + case TSDB_CODE_PAR_INCORRECT_NUM_OF_COL: + return "Query block has incorrect number of result columns"; case TSDB_CODE_OUT_OF_MEMORY: return "Out of memory"; default: diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index e2e2207c70..f17322063a 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "os.h" #include "parser.h" #include "parInt.h" diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 48220dbef7..bf23058f7c 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -32,6 +32,7 @@ #include #include +#include "os.h" #include "functionMgt.h" #include "nodes.h" #include "parToken.h" @@ -133,16 +134,16 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYNSTATE 572 -#define YYNRULE 436 +#define YYNRULE 437 #define YYNTOKEN 222 #define YY_MAX_SHIFT 571 -#define YY_MIN_SHIFTREDUCE 849 -#define YY_MAX_SHIFTREDUCE 1284 -#define YY_ERROR_ACTION 1285 -#define YY_ACCEPT_ACTION 1286 -#define YY_NO_ACTION 1287 -#define YY_MIN_REDUCE 1288 -#define YY_MAX_REDUCE 1723 +#define YY_MIN_SHIFTREDUCE 850 +#define YY_MAX_SHIFTREDUCE 1286 +#define YY_ERROR_ACTION 1287 +#define YY_ACCEPT_ACTION 1288 +#define YY_NO_ACTION 1289 +#define YY_MIN_REDUCE 1290 +#define YY_MAX_REDUCE 1726 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -211,368 +212,368 @@ typedef union { *********** Begin parsing tables **********************************************/ #define YY_ACTTAB_COUNT (1960) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 269, 1591, 286, 472, 1575, 485, 281, 324, 469, 1503, - /* 10 */ 1286, 1399, 33, 31, 77, 1575, 123, 1561, 1300, 1561, - /* 20 */ 278, 382, 1105, 34, 32, 30, 29, 28, 1591, 1561, - /* 30 */ 1410, 1557, 1563, 1557, 1563, 469, 124, 249, 1103, 1591, - /* 40 */ 1367, 53, 437, 1557, 1564, 468, 469, 1387, 444, 1547, - /* 50 */ 12, 33, 31, 1227, 99, 53, 468, 1111, 484, 278, - /* 60 */ 1547, 1105, 1405, 294, 318, 448, 1575, 242, 1576, 471, - /* 70 */ 1578, 1579, 467, 104, 462, 1, 1406, 1103, 72, 1576, - /* 80 */ 471, 1578, 1579, 467, 361, 462, 484, 289, 1641, 12, - /* 90 */ 1591, 484, 250, 1637, 1494, 1496, 1111, 447, 568, 26, - /* 100 */ 204, 452, 1702, 1702, 1702, 440, 420, 468, 106, 102, - /* 110 */ 1104, 1547, 516, 485, 1, 136, 136, 136, 1327, 1700, - /* 120 */ 1700, 1700, 322, 446, 132, 1648, 1649, 444, 1653, 73, - /* 130 */ 1576, 471, 1578, 1579, 467, 519, 462, 568, 1410, 1641, - /* 140 */ 399, 36, 485, 271, 1637, 131, 36, 69, 1106, 1104, - /* 150 */ 421, 323, 104, 1126, 515, 514, 513, 200, 512, 1491, - /* 160 */ 127, 105, 1128, 427, 1668, 434, 145, 1410, 1402, 1109, - /* 170 */ 1110, 1451, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 464, - /* 180 */ 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1106, 102, 1702, - /* 190 */ 1459, 34, 32, 30, 29, 28, 268, 9, 8, 137, - /* 200 */ 68, 1457, 136, 133, 1648, 1649, 1700, 1653, 1109, 1110, - /* 210 */ 64, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 464, 1165, - /* 220 */ 1166, 1167, 1168, 1169, 1170, 1171, 33, 31, 438, 251, - /* 230 */ 226, 439, 435, 1440, 278, 248, 1105, 1126, 1575, 398, - /* 240 */ 886, 392, 885, 1459, 340, 397, 453, 352, 101, 283, - /* 250 */ 393, 391, 1103, 394, 1457, 1141, 353, 317, 390, 316, - /* 260 */ 887, 137, 1591, 1189, 12, 33, 31, 387, 386, 469, - /* 270 */ 137, 1111, 1203, 278, 22, 1105, 1655, 55, 267, 468, - /* 280 */ 1251, 174, 24, 1547, 34, 32, 30, 29, 28, 1, - /* 290 */ 1655, 1103, 34, 32, 30, 29, 28, 30, 29, 28, - /* 300 */ 1652, 73, 1576, 471, 1578, 1579, 467, 1401, 462, 137, - /* 310 */ 1111, 1641, 568, 1190, 1651, 271, 1637, 1714, 1129, 431, - /* 320 */ 1249, 1250, 1252, 1253, 1104, 412, 1675, 1388, 7, 1288, - /* 330 */ 351, 1195, 137, 346, 345, 344, 343, 342, 1130, 339, - /* 340 */ 338, 337, 336, 335, 331, 330, 329, 328, 327, 326, - /* 350 */ 325, 568, 1547, 97, 96, 95, 94, 93, 92, 91, - /* 360 */ 90, 89, 1106, 1104, 1702, 1702, 25, 276, 1184, 1185, - /* 370 */ 1186, 1187, 1188, 1192, 1193, 1194, 361, 136, 1701, 1311, - /* 380 */ 521, 1700, 1700, 1109, 1110, 939, 1154, 1155, 1156, 1157, - /* 390 */ 1158, 1159, 1160, 464, 1165, 1166, 1167, 1168, 1169, 1170, - /* 400 */ 1171, 1106, 564, 563, 941, 34, 32, 30, 29, 28, - /* 410 */ 88, 396, 395, 87, 86, 85, 84, 83, 82, 81, - /* 420 */ 80, 79, 1109, 1110, 1547, 1154, 1155, 1156, 1157, 1158, - /* 430 */ 1159, 1160, 464, 1165, 1166, 1167, 1168, 1169, 1170, 1171, - /* 440 */ 33, 31, 1172, 251, 1289, 309, 1310, 282, 278, 1127, - /* 450 */ 1105, 288, 137, 485, 472, 121, 261, 100, 199, 121, - /* 460 */ 1504, 385, 332, 1412, 311, 88, 1103, 1412, 87, 86, - /* 470 */ 85, 84, 83, 82, 81, 80, 79, 1189, 1410, 33, - /* 480 */ 31, 143, 100, 1385, 388, 1111, 385, 278, 1309, 1105, - /* 490 */ 1575, 1547, 34, 32, 30, 29, 28, 485, 485, 262, - /* 500 */ 1386, 260, 259, 7, 384, 1103, 77, 333, 51, 388, - /* 510 */ 1322, 50, 291, 389, 1591, 34, 32, 30, 29, 28, - /* 520 */ 121, 469, 1410, 1410, 1111, 485, 568, 1190, 1412, 1308, - /* 530 */ 1459, 468, 401, 1547, 360, 1547, 290, 485, 1104, 121, - /* 540 */ 1459, 1457, 7, 1191, 106, 1195, 1407, 1413, 516, 521, - /* 550 */ 1410, 1458, 62, 73, 1576, 471, 1578, 1579, 467, 511, - /* 560 */ 462, 1196, 1410, 1641, 1340, 568, 524, 271, 1637, 1714, - /* 570 */ 1131, 519, 347, 1403, 1547, 1655, 1106, 1104, 1698, 1241, - /* 580 */ 25, 276, 1184, 1185, 1186, 1187, 1188, 1192, 1193, 1194, - /* 590 */ 515, 514, 513, 1575, 512, 451, 23, 1109, 1110, 1650, - /* 600 */ 1154, 1155, 1156, 1157, 1158, 1159, 1160, 464, 1165, 1166, - /* 610 */ 1167, 1168, 1169, 1170, 1171, 1106, 398, 1591, 392, 147, - /* 620 */ 146, 1281, 397, 1307, 447, 101, 1395, 393, 391, 525, - /* 630 */ 394, 1382, 518, 517, 468, 390, 1109, 1110, 1547, 1154, - /* 640 */ 1155, 1156, 1157, 1158, 1159, 1160, 464, 1165, 1166, 1167, - /* 650 */ 1168, 1169, 1170, 1171, 33, 31, 73, 1576, 471, 1578, - /* 660 */ 1579, 467, 278, 462, 1105, 1575, 1641, 1397, 1547, 485, - /* 670 */ 271, 1637, 131, 34, 32, 30, 29, 28, 1528, 485, - /* 680 */ 1103, 1306, 1226, 485, 444, 885, 537, 535, 482, 1591, - /* 690 */ 185, 1669, 292, 1568, 1410, 1393, 469, 1495, 1496, 1111, - /* 700 */ 1280, 380, 485, 6, 1410, 1566, 468, 177, 1410, 104, - /* 710 */ 1547, 483, 485, 1305, 1304, 1660, 1222, 1, 1337, 1234, - /* 720 */ 1303, 218, 1302, 1299, 1298, 1128, 1547, 1410, 73, 1576, - /* 730 */ 471, 1578, 1579, 467, 463, 462, 510, 1410, 1641, 1536, - /* 740 */ 568, 455, 271, 1637, 1714, 102, 34, 32, 30, 29, - /* 750 */ 28, 1297, 1104, 1659, 1296, 1295, 419, 1294, 1547, 1547, - /* 760 */ 134, 1648, 1649, 1293, 1653, 1547, 1301, 1547, 1547, 1547, - /* 770 */ 544, 543, 542, 541, 293, 301, 540, 539, 538, 107, - /* 780 */ 533, 532, 531, 530, 529, 528, 527, 526, 114, 522, - /* 790 */ 1106, 1177, 165, 459, 536, 163, 1547, 1128, 312, 1547, - /* 800 */ 1547, 120, 1547, 1292, 1091, 1092, 1291, 450, 1547, 1368, - /* 810 */ 1141, 1109, 1110, 201, 1154, 1155, 1156, 1157, 1158, 1159, - /* 820 */ 1160, 464, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 432, - /* 830 */ 977, 508, 507, 506, 981, 505, 983, 984, 504, 986, - /* 840 */ 501, 1452, 992, 498, 994, 995, 495, 492, 1547, 122, - /* 850 */ 1575, 1547, 167, 169, 232, 166, 168, 171, 1222, 410, - /* 860 */ 170, 1320, 1114, 1575, 112, 47, 230, 188, 423, 1248, - /* 870 */ 1113, 190, 408, 35, 1591, 1283, 1284, 1197, 1225, 148, - /* 880 */ 35, 469, 413, 404, 1161, 9, 8, 1591, 456, 35, - /* 890 */ 379, 468, 207, 1066, 469, 1547, 209, 109, 194, 110, - /* 900 */ 448, 477, 1671, 215, 468, 112, 47, 490, 1547, 970, - /* 910 */ 965, 998, 1181, 239, 1576, 471, 1578, 1579, 467, 110, - /* 920 */ 462, 111, 445, 1002, 1592, 1009, 74, 1576, 471, 1578, - /* 930 */ 1579, 467, 203, 462, 1117, 911, 1641, 2, 1126, 1702, - /* 940 */ 1640, 1637, 1116, 71, 1575, 112, 296, 300, 256, 1008, - /* 950 */ 160, 110, 136, 130, 912, 113, 1700, 939, 258, 378, - /* 960 */ 1075, 374, 370, 366, 159, 223, 334, 1493, 1591, 144, - /* 970 */ 341, 349, 49, 48, 321, 469, 142, 354, 348, 350, - /* 980 */ 1135, 315, 355, 149, 1134, 468, 356, 152, 357, 1547, - /* 990 */ 1133, 54, 359, 257, 157, 307, 1575, 303, 299, 139, - /* 1000 */ 358, 155, 52, 362, 1132, 383, 381, 74, 1576, 471, - /* 1010 */ 1578, 1579, 467, 158, 462, 1400, 162, 1641, 78, 571, - /* 1020 */ 1591, 458, 1637, 266, 1396, 164, 1111, 466, 1532, 224, - /* 1030 */ 137, 115, 116, 222, 1398, 1394, 98, 468, 117, 118, - /* 1040 */ 175, 1547, 560, 414, 556, 552, 548, 221, 178, 415, - /* 1050 */ 418, 1575, 156, 180, 151, 422, 153, 1131, 424, 246, - /* 1060 */ 1576, 471, 1578, 1579, 467, 465, 462, 460, 1613, 183, - /* 1070 */ 425, 433, 1682, 150, 70, 1591, 475, 216, 1672, 1681, - /* 1080 */ 186, 430, 469, 189, 1662, 270, 436, 429, 5, 441, - /* 1090 */ 4, 1222, 468, 1130, 37, 1717, 1547, 103, 272, 1656, - /* 1100 */ 1575, 457, 196, 454, 16, 1622, 473, 481, 193, 478, - /* 1110 */ 479, 1502, 129, 474, 125, 1576, 471, 1578, 1579, 467, - /* 1120 */ 280, 462, 195, 1501, 1591, 211, 480, 225, 1699, 202, - /* 1130 */ 213, 469, 61, 63, 1411, 1383, 426, 227, 488, 181, - /* 1140 */ 220, 468, 567, 43, 128, 1547, 233, 234, 229, 231, - /* 1150 */ 1541, 1540, 1575, 295, 1083, 1537, 176, 297, 449, 1715, - /* 1160 */ 444, 298, 1099, 74, 1576, 471, 1578, 1579, 467, 1100, - /* 1170 */ 462, 140, 302, 1641, 1535, 304, 1591, 305, 1638, 306, - /* 1180 */ 1534, 308, 1575, 469, 1533, 104, 310, 1518, 141, 1078, - /* 1190 */ 1077, 313, 314, 468, 1512, 1511, 320, 1547, 319, 1510, - /* 1200 */ 428, 1509, 1486, 1049, 448, 1485, 1591, 1484, 1483, 1482, - /* 1210 */ 1481, 1480, 1479, 469, 108, 247, 1576, 471, 1578, 1579, - /* 1220 */ 467, 102, 462, 468, 1478, 1477, 1476, 1547, 1475, 1474, - /* 1230 */ 1473, 1472, 1471, 1575, 1470, 1469, 197, 1648, 443, 1468, - /* 1240 */ 442, 1467, 1466, 1702, 1465, 125, 1576, 471, 1578, 1579, - /* 1250 */ 467, 1464, 462, 1463, 1462, 1051, 136, 1591, 1461, 1460, - /* 1260 */ 1700, 1339, 1526, 1575, 469, 1520, 1508, 1499, 1389, 904, - /* 1270 */ 154, 1338, 1336, 1334, 468, 363, 364, 1332, 1547, 365, - /* 1280 */ 1330, 275, 368, 367, 372, 371, 369, 1591, 375, 1319, - /* 1290 */ 1716, 1575, 373, 376, 466, 377, 247, 1576, 471, 1578, - /* 1300 */ 1579, 467, 1318, 462, 468, 1315, 1391, 76, 1547, 1014, - /* 1310 */ 1390, 1328, 1017, 263, 1323, 1591, 938, 937, 1321, 936, - /* 1320 */ 161, 935, 469, 934, 931, 264, 246, 1576, 471, 1578, - /* 1330 */ 1579, 467, 468, 462, 930, 1614, 1547, 402, 265, 277, - /* 1340 */ 1575, 534, 285, 284, 1314, 405, 407, 1313, 536, 1105, - /* 1350 */ 409, 75, 1119, 1525, 247, 1576, 471, 1578, 1579, 467, - /* 1360 */ 46, 462, 1519, 1085, 1591, 1103, 119, 416, 1112, 1507, - /* 1370 */ 1506, 469, 1498, 3, 56, 182, 35, 1566, 1575, 13, - /* 1380 */ 41, 468, 14, 38, 1111, 1547, 11, 1111, 279, 1575, - /* 1390 */ 58, 187, 126, 179, 191, 417, 1247, 20, 184, 198, - /* 1400 */ 40, 192, 1591, 247, 1576, 471, 1578, 1579, 467, 469, - /* 1410 */ 462, 1240, 57, 1591, 21, 1219, 1218, 1274, 39, 468, - /* 1420 */ 469, 135, 15, 1547, 8, 568, 1269, 1268, 486, 273, - /* 1430 */ 468, 1273, 1272, 274, 1547, 17, 1164, 1104, 138, 1575, - /* 1440 */ 1115, 235, 1576, 471, 1578, 1579, 467, 1163, 462, 461, - /* 1450 */ 27, 1182, 241, 1576, 471, 1578, 1579, 467, 1162, 462, - /* 1460 */ 10, 1149, 18, 1591, 205, 470, 206, 19, 1497, 212, - /* 1470 */ 469, 476, 1245, 208, 210, 1106, 59, 60, 1120, 1121, - /* 1480 */ 468, 42, 64, 214, 1547, 999, 489, 1565, 287, 1575, - /* 1490 */ 217, 491, 493, 487, 996, 494, 1109, 1110, 496, 1123, - /* 1500 */ 1575, 993, 243, 1576, 471, 1578, 1579, 467, 497, 462, - /* 1510 */ 987, 499, 500, 1591, 985, 502, 503, 991, 976, 990, - /* 1520 */ 469, 989, 65, 988, 1591, 1011, 66, 67, 509, 1007, - /* 1530 */ 468, 469, 1004, 902, 1547, 520, 927, 1010, 945, 523, - /* 1540 */ 925, 468, 219, 924, 923, 1547, 922, 921, 1575, 920, - /* 1550 */ 919, 918, 236, 1576, 471, 1578, 1579, 467, 942, 462, - /* 1560 */ 940, 1575, 403, 244, 1576, 471, 1578, 1579, 467, 915, - /* 1570 */ 462, 914, 1591, 913, 910, 909, 908, 411, 907, 469, - /* 1580 */ 1335, 545, 546, 1333, 547, 1591, 549, 550, 551, 468, - /* 1590 */ 1331, 173, 469, 1547, 406, 553, 554, 555, 1329, 400, - /* 1600 */ 557, 558, 468, 1317, 559, 172, 1547, 561, 562, 1316, - /* 1610 */ 1312, 237, 1576, 471, 1578, 1579, 467, 565, 462, 1107, - /* 1620 */ 566, 570, 228, 569, 245, 1576, 471, 1578, 1579, 467, - /* 1630 */ 1575, 462, 45, 1287, 1287, 44, 1287, 1287, 1287, 1575, - /* 1640 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1650 */ 1287, 1287, 1287, 1287, 1591, 1287, 1287, 1287, 1287, 1287, - /* 1660 */ 1287, 469, 1287, 1591, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1670 */ 469, 468, 1287, 1287, 1287, 1547, 1287, 1287, 1287, 1287, - /* 1680 */ 468, 1287, 1287, 1287, 1547, 1287, 1287, 1287, 1287, 1575, - /* 1690 */ 1287, 1287, 1287, 238, 1576, 471, 1578, 1579, 467, 1287, - /* 1700 */ 462, 1287, 1587, 1576, 471, 1578, 1579, 467, 1287, 462, - /* 1710 */ 1287, 1287, 1287, 1591, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1720 */ 469, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1730 */ 468, 1287, 1287, 1287, 1547, 1287, 1287, 1287, 1287, 1575, - /* 1740 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1750 */ 1575, 1287, 1586, 1576, 471, 1578, 1579, 467, 1287, 462, - /* 1760 */ 1287, 1287, 1287, 1591, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1770 */ 469, 1287, 1287, 1287, 1591, 1287, 1287, 1287, 1287, 1287, - /* 1780 */ 468, 469, 1287, 1287, 1547, 1287, 1287, 1287, 1287, 1287, - /* 1790 */ 1287, 468, 1287, 1287, 1287, 1547, 1287, 1287, 1575, 1287, - /* 1800 */ 1287, 1287, 1585, 1576, 471, 1578, 1579, 467, 1287, 462, - /* 1810 */ 1287, 1575, 1287, 254, 1576, 471, 1578, 1579, 467, 1287, - /* 1820 */ 462, 1287, 1591, 1287, 1287, 1287, 1287, 1287, 1287, 469, - /* 1830 */ 1287, 1287, 1287, 1287, 1287, 1591, 1287, 1287, 1287, 468, - /* 1840 */ 1287, 1287, 469, 1547, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1850 */ 1287, 1287, 468, 1287, 1287, 1287, 1547, 1287, 1287, 1287, - /* 1860 */ 1287, 253, 1576, 471, 1578, 1579, 467, 1287, 462, 1287, - /* 1870 */ 1287, 1287, 1287, 1287, 255, 1576, 471, 1578, 1579, 467, - /* 1880 */ 1575, 462, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1575, - /* 1890 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1900 */ 1287, 1287, 1287, 1287, 1591, 1287, 1287, 1287, 1287, 1287, - /* 1910 */ 1287, 469, 1287, 1591, 1287, 1287, 1287, 1287, 1287, 1287, - /* 1920 */ 469, 468, 1287, 1287, 1287, 1547, 1287, 1287, 1287, 1287, - /* 1930 */ 468, 1287, 1287, 1287, 1547, 1287, 1287, 1287, 1287, 1287, - /* 1940 */ 1287, 1287, 1287, 252, 1576, 471, 1578, 1579, 467, 1287, - /* 1950 */ 462, 1287, 240, 1576, 471, 1578, 1579, 467, 1287, 462, + /* 0 */ 270, 1593, 287, 472, 1577, 485, 282, 325, 469, 1505, + /* 10 */ 1288, 1401, 33, 31, 78, 1577, 124, 1563, 1302, 1563, + /* 20 */ 279, 383, 1106, 34, 32, 30, 29, 28, 1593, 1563, + /* 30 */ 1412, 1559, 1565, 1559, 1565, 469, 125, 250, 1104, 1593, + /* 40 */ 1369, 1705, 438, 1559, 1566, 468, 469, 1389, 445, 1549, + /* 50 */ 12, 33, 31, 1228, 1704, 54, 468, 1112, 1702, 279, + /* 60 */ 1549, 1106, 484, 295, 319, 449, 1577, 243, 1578, 471, + /* 70 */ 1580, 1581, 467, 105, 462, 1, 1408, 1104, 73, 1578, + /* 80 */ 471, 1580, 1581, 467, 525, 462, 1384, 290, 1643, 12, + /* 90 */ 1593, 484, 251, 1639, 1496, 1498, 1112, 448, 568, 26, + /* 100 */ 205, 362, 1705, 1705, 1705, 441, 421, 468, 107, 103, + /* 110 */ 1105, 1549, 516, 485, 1, 137, 137, 137, 1129, 1702, + /* 120 */ 1702, 1702, 323, 447, 133, 1650, 1651, 445, 1655, 74, + /* 130 */ 1578, 471, 1580, 1581, 467, 519, 462, 568, 1412, 1643, + /* 140 */ 54, 144, 485, 272, 1639, 132, 36, 940, 1107, 1105, + /* 150 */ 422, 324, 105, 100, 515, 514, 513, 201, 512, 1127, + /* 160 */ 439, 1407, 1130, 428, 1670, 435, 942, 1412, 52, 1110, + /* 170 */ 1111, 51, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, + /* 180 */ 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1107, 103, 1705, + /* 190 */ 1461, 34, 32, 30, 29, 28, 269, 9, 8, 138, + /* 200 */ 69, 1459, 137, 134, 1650, 1651, 1702, 1655, 1110, 1111, + /* 210 */ 65, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, + /* 220 */ 1167, 1168, 1169, 1170, 1171, 1172, 33, 31, 138, 252, + /* 230 */ 227, 440, 436, 1442, 279, 249, 1106, 1127, 1577, 399, + /* 240 */ 63, 393, 521, 1461, 341, 398, 70, 353, 102, 284, + /* 250 */ 394, 392, 1104, 395, 1459, 1142, 354, 318, 391, 317, + /* 260 */ 106, 1405, 1593, 1190, 12, 33, 31, 1404, 1227, 469, + /* 270 */ 138, 1112, 1204, 279, 22, 1106, 1657, 56, 268, 468, + /* 280 */ 1252, 175, 24, 1549, 34, 32, 30, 29, 28, 1, + /* 290 */ 1657, 1104, 34, 32, 30, 29, 28, 30, 29, 28, + /* 300 */ 1654, 74, 1578, 471, 1580, 1581, 467, 128, 462, 138, + /* 310 */ 1112, 1643, 568, 1191, 1653, 272, 1639, 1717, 1453, 432, + /* 320 */ 1250, 1251, 1253, 1254, 1105, 413, 1677, 1390, 7, 1290, + /* 330 */ 352, 1196, 1128, 347, 346, 345, 344, 343, 138, 340, + /* 340 */ 339, 338, 337, 336, 332, 331, 330, 329, 328, 327, + /* 350 */ 326, 568, 59, 98, 97, 96, 95, 94, 93, 92, + /* 360 */ 91, 90, 1107, 1105, 1705, 1705, 25, 277, 1185, 1186, + /* 370 */ 1187, 1188, 1189, 1193, 1194, 1195, 362, 137, 1703, 1403, + /* 380 */ 1131, 1702, 1702, 1110, 1111, 6, 1155, 1156, 1157, 1158, + /* 390 */ 1159, 1160, 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, + /* 400 */ 1172, 1107, 524, 484, 452, 34, 32, 30, 29, 28, + /* 410 */ 89, 564, 563, 88, 87, 86, 85, 84, 83, 82, + /* 420 */ 81, 80, 1110, 1111, 1549, 1155, 1156, 1157, 1158, 1159, + /* 430 */ 1160, 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, 1172, + /* 440 */ 33, 31, 1173, 252, 1291, 536, 1313, 283, 279, 313, + /* 450 */ 1106, 289, 138, 485, 472, 122, 262, 101, 36, 122, + /* 460 */ 1506, 386, 333, 1414, 1226, 89, 1104, 1414, 88, 87, + /* 470 */ 86, 85, 84, 83, 82, 81, 80, 1190, 1412, 33, + /* 480 */ 31, 48, 101, 1387, 389, 1112, 386, 279, 1312, 1106, + /* 490 */ 1577, 1549, 34, 32, 30, 29, 28, 485, 485, 263, + /* 500 */ 200, 261, 260, 7, 385, 1104, 78, 334, 511, 389, + /* 510 */ 388, 387, 292, 390, 1593, 34, 32, 30, 29, 28, + /* 520 */ 122, 469, 1412, 1412, 1112, 485, 568, 1191, 1414, 1311, + /* 530 */ 1461, 468, 1493, 1549, 361, 1549, 291, 485, 1105, 146, + /* 540 */ 1223, 1459, 7, 1192, 107, 1196, 1409, 138, 516, 453, + /* 550 */ 1412, 397, 396, 74, 1578, 471, 1580, 1581, 467, 1461, + /* 560 */ 462, 1197, 1412, 1643, 1342, 568, 1132, 272, 1639, 1717, + /* 570 */ 1460, 519, 348, 1397, 1549, 1657, 1107, 1105, 1700, 1242, + /* 580 */ 25, 277, 1185, 1186, 1187, 1188, 1189, 1193, 1194, 1195, + /* 590 */ 515, 514, 513, 1577, 512, 1538, 23, 1110, 1111, 1652, + /* 600 */ 1155, 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, 1167, + /* 610 */ 1168, 1169, 1170, 1171, 1172, 1107, 399, 1593, 393, 148, + /* 620 */ 147, 1283, 398, 1310, 448, 102, 166, 394, 392, 164, + /* 630 */ 395, 302, 518, 517, 468, 391, 1110, 1111, 1549, 1155, + /* 640 */ 1156, 1157, 1158, 1159, 1160, 1161, 464, 1166, 1167, 1168, + /* 650 */ 1169, 1170, 1171, 1172, 33, 31, 74, 1578, 471, 1580, + /* 660 */ 1581, 467, 279, 462, 1106, 1577, 1643, 1388, 1549, 485, + /* 670 */ 272, 1639, 132, 34, 32, 30, 29, 28, 1530, 485, + /* 680 */ 1104, 1309, 122, 485, 445, 887, 186, 886, 482, 1593, + /* 690 */ 1415, 1671, 293, 451, 1412, 1399, 469, 537, 535, 1112, + /* 700 */ 1282, 1235, 485, 886, 1412, 888, 468, 1129, 1412, 105, + /* 710 */ 1549, 483, 485, 1308, 1307, 310, 521, 1, 1339, 381, + /* 720 */ 1306, 219, 1305, 1304, 1301, 1395, 1549, 1412, 74, 1578, + /* 730 */ 471, 1580, 1581, 467, 312, 462, 168, 1412, 1643, 167, + /* 740 */ 568, 455, 272, 1639, 1717, 103, 34, 32, 30, 29, + /* 750 */ 28, 1300, 1105, 1661, 1299, 1298, 178, 1297, 1549, 1549, + /* 760 */ 135, 1650, 1651, 1296, 1655, 1549, 459, 1549, 1549, 1549, + /* 770 */ 544, 543, 542, 541, 294, 463, 540, 539, 538, 108, + /* 780 */ 533, 532, 531, 530, 529, 528, 527, 526, 115, 522, + /* 790 */ 1107, 1178, 1497, 1498, 1662, 1223, 1549, 1129, 510, 1549, + /* 800 */ 1549, 121, 1549, 1295, 1092, 1093, 1294, 1293, 1549, 113, + /* 810 */ 1142, 1110, 1111, 424, 1155, 1156, 1157, 1158, 1159, 1160, + /* 820 */ 1161, 464, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1570, + /* 830 */ 978, 508, 507, 506, 982, 505, 984, 985, 504, 987, + /* 840 */ 501, 1568, 993, 498, 995, 996, 495, 492, 1549, 123, + /* 850 */ 1577, 1549, 1549, 170, 233, 1329, 169, 172, 1324, 411, + /* 860 */ 171, 1322, 1115, 1577, 47, 189, 231, 420, 1249, 191, + /* 870 */ 9, 8, 409, 1114, 1593, 1285, 1286, 400, 35, 149, + /* 880 */ 402, 469, 1198, 405, 1303, 1182, 35, 1593, 456, 35, + /* 890 */ 1162, 468, 208, 1067, 469, 1549, 210, 110, 1370, 111, + /* 900 */ 449, 477, 113, 216, 468, 47, 971, 202, 1549, 966, + /* 910 */ 433, 1454, 414, 240, 1578, 471, 1580, 1581, 467, 490, + /* 920 */ 462, 111, 195, 999, 380, 1003, 75, 1578, 471, 1580, + /* 930 */ 1581, 467, 446, 462, 1118, 1673, 1643, 1594, 112, 1705, + /* 940 */ 1642, 1639, 1010, 72, 1577, 1117, 204, 912, 2, 301, + /* 950 */ 161, 113, 137, 131, 111, 1009, 1702, 1127, 114, 379, + /* 960 */ 297, 375, 371, 367, 160, 257, 913, 259, 1593, 940, + /* 970 */ 224, 335, 50, 49, 322, 469, 143, 1495, 1076, 342, + /* 980 */ 145, 316, 350, 349, 351, 468, 355, 1136, 356, 1549, + /* 990 */ 150, 55, 357, 258, 158, 308, 1577, 304, 300, 140, + /* 1000 */ 1135, 358, 153, 359, 1134, 360, 156, 75, 1578, 471, + /* 1010 */ 1580, 1581, 467, 53, 462, 363, 159, 1643, 1133, 571, + /* 1020 */ 1593, 458, 1639, 382, 384, 1402, 79, 466, 163, 267, + /* 1030 */ 138, 1398, 1112, 223, 165, 116, 99, 468, 117, 1400, + /* 1040 */ 1396, 1549, 560, 118, 556, 552, 548, 222, 119, 176, + /* 1050 */ 225, 1577, 157, 415, 152, 423, 154, 1534, 179, 247, + /* 1060 */ 1578, 471, 1580, 1581, 467, 465, 462, 460, 1615, 419, + /* 1070 */ 181, 425, 426, 151, 71, 1593, 416, 217, 1132, 434, + /* 1080 */ 184, 187, 469, 1684, 475, 1674, 431, 1683, 5, 190, + /* 1090 */ 442, 271, 468, 437, 430, 4, 1549, 1223, 104, 1131, + /* 1100 */ 1577, 273, 37, 1720, 457, 454, 197, 481, 1658, 1664, + /* 1110 */ 16, 473, 1624, 474, 126, 1578, 471, 1580, 1581, 467, + /* 1120 */ 194, 462, 130, 1504, 1593, 196, 478, 1503, 226, 212, + /* 1130 */ 281, 469, 479, 214, 480, 62, 427, 1701, 203, 182, + /* 1140 */ 1413, 468, 64, 488, 221, 1549, 1385, 228, 567, 129, + /* 1150 */ 230, 234, 1577, 43, 1084, 235, 177, 232, 450, 1718, + /* 1160 */ 445, 1543, 1542, 75, 1578, 471, 1580, 1581, 467, 296, + /* 1170 */ 462, 1539, 298, 1643, 299, 1100, 1593, 1101, 1640, 141, + /* 1180 */ 303, 1537, 1577, 469, 305, 105, 306, 307, 1536, 309, + /* 1190 */ 1535, 1520, 311, 468, 142, 314, 315, 1549, 1079, 1078, + /* 1200 */ 429, 1514, 1513, 320, 449, 321, 1593, 1512, 1511, 1050, + /* 1210 */ 1488, 1487, 1486, 469, 109, 248, 1578, 471, 1580, 1581, + /* 1220 */ 467, 103, 462, 468, 1485, 1484, 1483, 1549, 1482, 1481, + /* 1230 */ 1480, 1479, 1478, 1577, 1477, 1476, 198, 1650, 444, 1475, + /* 1240 */ 443, 1474, 1473, 1705, 1472, 126, 1578, 471, 1580, 1581, + /* 1250 */ 467, 1471, 462, 1470, 1469, 1468, 137, 1593, 1467, 1466, + /* 1260 */ 1702, 1465, 1052, 1577, 469, 1464, 1463, 1462, 1341, 1528, + /* 1270 */ 1522, 1510, 1501, 1391, 468, 162, 155, 1340, 1549, 1338, + /* 1280 */ 1336, 276, 365, 364, 369, 368, 366, 1593, 905, 1334, + /* 1290 */ 1719, 1577, 370, 374, 466, 372, 248, 1578, 471, 1580, + /* 1300 */ 1581, 467, 1332, 462, 468, 378, 1321, 1320, 1549, 1317, + /* 1310 */ 373, 1393, 1015, 1392, 1018, 1593, 376, 377, 1330, 264, + /* 1320 */ 77, 939, 469, 938, 534, 937, 247, 1578, 471, 1580, + /* 1330 */ 1581, 467, 468, 462, 936, 1616, 1549, 935, 932, 278, + /* 1340 */ 1577, 931, 286, 285, 1325, 1323, 265, 403, 266, 1106, + /* 1350 */ 406, 1316, 1120, 408, 248, 1578, 471, 1580, 1581, 467, + /* 1360 */ 536, 462, 1315, 410, 1593, 1104, 76, 1527, 1113, 1521, + /* 1370 */ 1086, 469, 120, 1509, 417, 180, 1508, 1500, 1577, 57, + /* 1380 */ 183, 468, 3, 13, 1112, 1549, 35, 1112, 280, 1577, + /* 1390 */ 14, 188, 41, 1248, 127, 38, 46, 192, 185, 11, + /* 1400 */ 418, 1241, 1593, 248, 1578, 471, 1580, 1581, 467, 469, + /* 1410 */ 462, 1220, 193, 1593, 20, 1568, 199, 58, 1271, 468, + /* 1420 */ 469, 21, 40, 1549, 1219, 568, 39, 15, 486, 1276, + /* 1430 */ 468, 1270, 274, 1275, 1549, 1274, 275, 1105, 8, 1577, + /* 1440 */ 1116, 236, 1578, 471, 1580, 1581, 467, 136, 462, 461, + /* 1450 */ 17, 1165, 242, 1578, 471, 1580, 1581, 467, 1164, 462, + /* 1460 */ 1183, 27, 10, 1593, 1163, 18, 139, 1150, 206, 470, + /* 1470 */ 469, 19, 476, 1499, 209, 1107, 207, 1246, 1121, 211, + /* 1480 */ 468, 60, 213, 61, 1549, 215, 42, 65, 1122, 1577, + /* 1490 */ 1567, 1000, 489, 288, 218, 487, 1110, 1111, 491, 1124, + /* 1500 */ 1577, 997, 244, 1578, 471, 1580, 1581, 467, 493, 462, + /* 1510 */ 494, 977, 496, 1593, 994, 497, 499, 988, 500, 502, + /* 1520 */ 469, 992, 986, 503, 1593, 509, 991, 990, 989, 1012, + /* 1530 */ 468, 469, 66, 67, 1549, 68, 1008, 1005, 903, 1011, + /* 1540 */ 520, 468, 928, 946, 523, 1549, 926, 220, 1577, 925, + /* 1550 */ 924, 923, 237, 1578, 471, 1580, 1581, 467, 922, 462, + /* 1560 */ 921, 1577, 404, 245, 1578, 471, 1580, 1581, 467, 920, + /* 1570 */ 462, 919, 1593, 943, 941, 916, 915, 412, 914, 469, + /* 1580 */ 911, 910, 909, 908, 1337, 1593, 545, 546, 1335, 468, + /* 1590 */ 547, 174, 469, 1549, 407, 549, 550, 551, 1333, 401, + /* 1600 */ 553, 554, 468, 1331, 555, 173, 1549, 557, 558, 559, + /* 1610 */ 1319, 238, 1578, 471, 1580, 1581, 467, 561, 462, 562, + /* 1620 */ 1318, 1314, 565, 566, 246, 1578, 471, 1580, 1581, 467, + /* 1630 */ 1577, 462, 45, 1108, 229, 44, 569, 570, 1289, 1577, + /* 1640 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1650 */ 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, + /* 1660 */ 1289, 469, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1670 */ 469, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, + /* 1680 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1577, + /* 1690 */ 1289, 1289, 1289, 239, 1578, 471, 1580, 1581, 467, 1289, + /* 1700 */ 462, 1289, 1589, 1578, 471, 1580, 1581, 467, 1289, 462, + /* 1710 */ 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1720 */ 469, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1730 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1577, + /* 1740 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1750 */ 1577, 1289, 1588, 1578, 471, 1580, 1581, 467, 1289, 462, + /* 1760 */ 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1770 */ 469, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, + /* 1780 */ 468, 469, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1289, + /* 1790 */ 1289, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1577, 1289, + /* 1800 */ 1289, 1289, 1587, 1578, 471, 1580, 1581, 467, 1289, 462, + /* 1810 */ 1289, 1577, 1289, 255, 1578, 471, 1580, 1581, 467, 1289, + /* 1820 */ 462, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, 469, + /* 1830 */ 1289, 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 468, + /* 1840 */ 1289, 1289, 469, 1549, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1850 */ 1289, 1289, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, + /* 1860 */ 1289, 254, 1578, 471, 1580, 1581, 467, 1289, 462, 1289, + /* 1870 */ 1289, 1289, 1289, 1289, 256, 1578, 471, 1580, 1581, 467, + /* 1880 */ 1577, 462, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1577, + /* 1890 */ 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1900 */ 1289, 1289, 1289, 1289, 1593, 1289, 1289, 1289, 1289, 1289, + /* 1910 */ 1289, 469, 1289, 1593, 1289, 1289, 1289, 1289, 1289, 1289, + /* 1920 */ 469, 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, + /* 1930 */ 468, 1289, 1289, 1289, 1549, 1289, 1289, 1289, 1289, 1289, + /* 1940 */ 1289, 1289, 1289, 253, 1578, 471, 1580, 1581, 467, 1289, + /* 1950 */ 462, 1289, 241, 1578, 471, 1580, 1581, 467, 1289, 462, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 253, 249, 253, 266, 225, 231, 269, 231, 256, 272, /* 10 */ 222, 250, 12, 13, 240, 225, 224, 270, 226, 270, /* 20 */ 20, 247, 22, 12, 13, 14, 15, 16, 249, 270, /* 30 */ 256, 284, 285, 284, 285, 256, 234, 261, 38, 249, - /* 40 */ 238, 233, 290, 284, 285, 266, 256, 0, 231, 270, - /* 50 */ 50, 12, 13, 14, 246, 233, 266, 57, 20, 20, - /* 60 */ 270, 22, 254, 275, 275, 275, 225, 288, 289, 290, + /* 40 */ 238, 314, 290, 284, 285, 266, 256, 0, 231, 270, + /* 50 */ 50, 12, 13, 14, 327, 233, 266, 57, 331, 20, + /* 60 */ 270, 22, 20, 275, 275, 275, 225, 288, 289, 290, /* 70 */ 291, 292, 293, 256, 295, 75, 254, 38, 288, 289, - /* 80 */ 290, 291, 292, 293, 49, 295, 20, 258, 298, 50, + /* 80 */ 290, 291, 292, 293, 237, 295, 239, 258, 298, 50, /* 90 */ 249, 20, 302, 303, 265, 266, 57, 256, 98, 299, - /* 100 */ 300, 72, 314, 314, 314, 326, 231, 266, 61, 292, - /* 110 */ 110, 270, 65, 231, 75, 327, 327, 327, 0, 331, + /* 100 */ 300, 49, 314, 314, 314, 326, 231, 266, 61, 292, + /* 110 */ 110, 270, 65, 231, 75, 327, 327, 327, 20, 331, /* 120 */ 331, 331, 240, 306, 307, 308, 309, 231, 311, 288, /* 130 */ 289, 290, 291, 292, 293, 88, 295, 98, 256, 298, - /* 140 */ 22, 75, 231, 302, 303, 304, 75, 230, 148, 110, - /* 150 */ 275, 240, 256, 20, 107, 108, 109, 316, 111, 256, - /* 160 */ 248, 244, 20, 322, 323, 138, 263, 256, 251, 169, - /* 170 */ 170, 259, 172, 173, 174, 175, 176, 177, 178, 179, + /* 140 */ 233, 47, 231, 302, 303, 304, 75, 38, 148, 110, + /* 150 */ 275, 240, 256, 246, 107, 108, 109, 316, 111, 20, + /* 160 */ 20, 254, 20, 322, 323, 138, 57, 256, 74, 169, + /* 170 */ 170, 77, 172, 173, 174, 175, 176, 177, 178, 179, /* 180 */ 180, 181, 182, 183, 184, 185, 186, 148, 292, 314, /* 190 */ 249, 12, 13, 14, 15, 16, 255, 1, 2, 199, /* 200 */ 75, 260, 327, 307, 308, 309, 331, 311, 169, 170, /* 210 */ 85, 172, 173, 174, 175, 176, 177, 178, 179, 180, - /* 220 */ 181, 182, 183, 184, 185, 186, 12, 13, 20, 50, + /* 220 */ 181, 182, 183, 184, 185, 186, 12, 13, 199, 50, /* 230 */ 242, 204, 205, 245, 20, 18, 22, 20, 225, 52, - /* 240 */ 20, 54, 22, 249, 27, 58, 217, 30, 61, 255, + /* 240 */ 230, 54, 49, 249, 27, 58, 230, 30, 61, 255, /* 250 */ 63, 64, 38, 66, 260, 76, 39, 147, 71, 149, - /* 260 */ 40, 199, 249, 84, 50, 12, 13, 235, 236, 256, + /* 260 */ 244, 251, 249, 84, 50, 12, 13, 251, 4, 256, /* 270 */ 199, 57, 76, 20, 2, 22, 286, 157, 158, 266, /* 280 */ 169, 161, 2, 270, 12, 13, 14, 15, 16, 75, /* 290 */ 286, 38, 12, 13, 14, 15, 16, 14, 15, 16, - /* 300 */ 310, 288, 289, 290, 291, 292, 293, 225, 295, 199, - /* 310 */ 57, 298, 98, 134, 310, 302, 303, 304, 20, 208, + /* 300 */ 310, 288, 289, 290, 291, 292, 293, 248, 295, 199, + /* 310 */ 57, 298, 98, 134, 310, 302, 303, 304, 259, 208, /* 320 */ 209, 210, 211, 212, 110, 275, 313, 0, 75, 0, - /* 330 */ 113, 152, 199, 116, 117, 118, 119, 120, 20, 122, + /* 330 */ 113, 152, 20, 116, 117, 118, 119, 120, 199, 122, /* 340 */ 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, - /* 350 */ 133, 98, 270, 24, 25, 26, 27, 28, 29, 30, + /* 350 */ 133, 98, 4, 24, 25, 26, 27, 28, 29, 30, /* 360 */ 31, 32, 148, 110, 314, 314, 187, 188, 189, 190, /* 370 */ 191, 192, 193, 194, 195, 196, 49, 327, 327, 225, - /* 380 */ 49, 331, 331, 169, 170, 38, 172, 173, 174, 175, + /* 380 */ 20, 331, 331, 169, 170, 43, 172, 173, 174, 175, /* 390 */ 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, - /* 400 */ 186, 148, 228, 229, 57, 12, 13, 14, 15, 16, - /* 410 */ 21, 235, 236, 24, 25, 26, 27, 28, 29, 30, + /* 400 */ 186, 148, 57, 20, 72, 12, 13, 14, 15, 16, + /* 410 */ 21, 228, 229, 24, 25, 26, 27, 28, 29, 30, /* 420 */ 31, 32, 169, 170, 270, 172, 173, 174, 175, 176, /* 430 */ 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, - /* 440 */ 12, 13, 14, 50, 0, 144, 225, 241, 20, 20, - /* 450 */ 22, 241, 199, 231, 266, 249, 35, 61, 140, 249, - /* 460 */ 272, 65, 240, 257, 163, 21, 38, 257, 24, 25, + /* 440 */ 12, 13, 14, 50, 0, 72, 225, 241, 20, 76, + /* 450 */ 22, 241, 199, 231, 266, 249, 35, 61, 75, 249, + /* 460 */ 272, 65, 240, 257, 200, 21, 38, 257, 24, 25, /* 470 */ 26, 27, 28, 29, 30, 31, 32, 84, 256, 12, - /* 480 */ 13, 47, 61, 0, 88, 57, 65, 20, 225, 22, + /* 480 */ 13, 3, 61, 0, 88, 57, 65, 20, 225, 22, /* 490 */ 225, 270, 12, 13, 14, 15, 16, 231, 231, 78, - /* 500 */ 0, 80, 81, 75, 83, 38, 240, 240, 74, 88, - /* 510 */ 0, 77, 241, 247, 249, 12, 13, 14, 15, 16, + /* 500 */ 140, 80, 81, 75, 83, 38, 240, 240, 86, 88, + /* 510 */ 235, 236, 241, 247, 249, 12, 13, 14, 15, 16, /* 520 */ 249, 256, 256, 256, 57, 231, 98, 134, 257, 225, - /* 530 */ 249, 266, 22, 270, 240, 270, 255, 231, 110, 249, - /* 540 */ 249, 260, 75, 134, 61, 152, 240, 257, 65, 49, - /* 550 */ 256, 260, 230, 288, 289, 290, 291, 292, 293, 86, - /* 560 */ 295, 152, 256, 298, 0, 98, 57, 302, 303, 304, - /* 570 */ 20, 88, 67, 251, 270, 286, 148, 110, 313, 76, + /* 530 */ 249, 266, 256, 270, 240, 270, 255, 231, 110, 263, + /* 540 */ 198, 260, 75, 134, 61, 152, 240, 199, 65, 217, + /* 550 */ 256, 235, 236, 288, 289, 290, 291, 292, 293, 249, + /* 560 */ 295, 152, 256, 298, 0, 98, 20, 302, 303, 304, + /* 570 */ 260, 88, 67, 250, 270, 286, 148, 110, 313, 76, /* 580 */ 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, - /* 590 */ 107, 108, 109, 225, 111, 3, 187, 169, 170, 310, + /* 590 */ 107, 108, 109, 225, 111, 0, 187, 169, 170, 310, /* 600 */ 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, /* 610 */ 182, 183, 184, 185, 186, 148, 52, 249, 54, 114, - /* 620 */ 115, 141, 58, 225, 256, 61, 250, 63, 64, 237, - /* 630 */ 66, 239, 235, 236, 266, 71, 169, 170, 270, 172, + /* 620 */ 115, 141, 58, 225, 256, 61, 79, 63, 64, 82, + /* 630 */ 66, 36, 235, 236, 266, 71, 169, 170, 270, 172, /* 640 */ 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, /* 650 */ 183, 184, 185, 186, 12, 13, 288, 289, 290, 291, - /* 660 */ 292, 293, 20, 295, 22, 225, 298, 250, 270, 231, + /* 660 */ 292, 293, 20, 295, 22, 225, 298, 0, 270, 231, /* 670 */ 302, 303, 304, 12, 13, 14, 15, 16, 240, 231, - /* 680 */ 38, 225, 4, 231, 231, 22, 235, 236, 240, 249, - /* 690 */ 140, 323, 240, 75, 256, 250, 256, 265, 266, 57, - /* 700 */ 220, 38, 231, 43, 256, 87, 266, 250, 256, 256, - /* 710 */ 270, 240, 231, 225, 225, 197, 198, 75, 0, 14, - /* 720 */ 225, 240, 225, 225, 225, 20, 270, 256, 288, 289, - /* 730 */ 290, 291, 292, 293, 250, 295, 250, 256, 298, 0, + /* 680 */ 38, 225, 249, 231, 231, 20, 140, 22, 240, 249, + /* 690 */ 257, 323, 240, 215, 256, 250, 256, 235, 236, 57, + /* 700 */ 220, 14, 231, 22, 256, 40, 266, 20, 256, 256, + /* 710 */ 270, 240, 231, 225, 225, 144, 49, 75, 0, 38, + /* 720 */ 225, 240, 225, 225, 225, 250, 270, 256, 288, 289, + /* 730 */ 290, 291, 292, 293, 163, 295, 79, 256, 298, 82, /* 740 */ 98, 72, 302, 303, 304, 292, 12, 13, 14, 15, - /* 750 */ 16, 225, 110, 313, 225, 225, 278, 225, 270, 270, - /* 760 */ 307, 308, 309, 225, 311, 270, 226, 270, 270, 270, - /* 770 */ 52, 53, 54, 55, 56, 36, 58, 59, 60, 61, + /* 750 */ 16, 225, 110, 313, 225, 225, 250, 225, 270, 270, + /* 760 */ 307, 308, 309, 225, 311, 270, 50, 270, 270, 270, + /* 770 */ 52, 53, 54, 55, 56, 250, 58, 59, 60, 61, /* 780 */ 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - /* 790 */ 148, 14, 79, 50, 72, 82, 270, 20, 76, 270, - /* 800 */ 270, 140, 270, 225, 159, 160, 225, 215, 270, 238, - /* 810 */ 76, 169, 170, 334, 172, 173, 174, 175, 176, 177, - /* 820 */ 178, 179, 180, 181, 182, 183, 184, 185, 186, 325, + /* 790 */ 148, 14, 265, 266, 197, 198, 270, 20, 250, 270, + /* 800 */ 270, 140, 270, 225, 159, 160, 225, 225, 270, 72, + /* 810 */ 76, 169, 170, 76, 172, 173, 174, 175, 176, 177, + /* 820 */ 178, 179, 180, 181, 182, 183, 184, 185, 186, 75, /* 830 */ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, - /* 840 */ 99, 259, 101, 102, 103, 104, 105, 106, 270, 18, - /* 850 */ 225, 270, 79, 79, 23, 82, 82, 79, 198, 21, - /* 860 */ 82, 0, 38, 225, 72, 72, 35, 72, 76, 76, - /* 870 */ 38, 76, 34, 72, 249, 184, 185, 76, 200, 48, - /* 880 */ 72, 256, 282, 22, 76, 1, 2, 249, 219, 72, - /* 890 */ 228, 266, 72, 76, 256, 270, 76, 72, 319, 72, - /* 900 */ 275, 76, 287, 76, 266, 72, 72, 72, 270, 76, - /* 910 */ 76, 76, 169, 288, 289, 290, 291, 292, 293, 72, - /* 920 */ 295, 72, 312, 76, 249, 76, 288, 289, 290, 291, - /* 930 */ 292, 293, 328, 295, 110, 38, 298, 315, 20, 314, - /* 940 */ 302, 303, 110, 112, 225, 72, 231, 36, 283, 76, - /* 950 */ 33, 72, 327, 36, 57, 76, 331, 38, 235, 42, - /* 960 */ 146, 44, 45, 46, 47, 276, 231, 231, 249, 121, - /* 970 */ 264, 134, 141, 142, 143, 256, 145, 231, 262, 262, - /* 980 */ 20, 150, 280, 233, 20, 266, 266, 233, 274, 270, - /* 990 */ 20, 74, 267, 162, 77, 164, 225, 166, 167, 168, - /* 1000 */ 256, 233, 233, 231, 20, 249, 227, 288, 289, 290, - /* 1010 */ 291, 292, 293, 233, 295, 249, 249, 298, 231, 19, - /* 1020 */ 249, 302, 303, 227, 249, 249, 57, 256, 270, 280, - /* 1030 */ 199, 249, 249, 33, 249, 249, 36, 266, 249, 249, - /* 1040 */ 230, 270, 42, 155, 44, 45, 46, 47, 230, 279, - /* 1050 */ 266, 225, 135, 230, 137, 274, 139, 20, 256, 288, - /* 1060 */ 289, 290, 291, 292, 293, 294, 295, 296, 297, 230, - /* 1070 */ 267, 207, 324, 156, 74, 249, 206, 77, 287, 324, - /* 1080 */ 271, 270, 256, 271, 321, 270, 270, 202, 214, 213, - /* 1090 */ 201, 198, 266, 20, 121, 335, 270, 256, 221, 286, - /* 1100 */ 225, 218, 305, 216, 75, 301, 270, 107, 320, 137, - /* 1110 */ 268, 271, 318, 270, 288, 289, 290, 291, 292, 293, - /* 1120 */ 270, 295, 317, 271, 249, 256, 267, 245, 330, 329, - /* 1130 */ 230, 256, 230, 75, 256, 239, 136, 231, 252, 139, - /* 1140 */ 230, 266, 227, 277, 281, 270, 243, 243, 232, 223, - /* 1150 */ 0, 0, 225, 64, 154, 0, 156, 38, 332, 333, - /* 1160 */ 231, 165, 38, 288, 289, 290, 291, 292, 293, 38, - /* 1170 */ 295, 38, 165, 298, 0, 38, 249, 38, 303, 165, - /* 1180 */ 0, 38, 225, 256, 0, 256, 38, 0, 75, 110, - /* 1190 */ 148, 152, 151, 266, 0, 0, 144, 270, 53, 0, - /* 1200 */ 273, 0, 0, 87, 275, 0, 249, 0, 0, 0, + /* 840 */ 99, 87, 101, 102, 103, 104, 105, 106, 270, 18, + /* 850 */ 225, 270, 270, 79, 23, 0, 82, 79, 0, 21, + /* 860 */ 82, 0, 38, 225, 72, 72, 35, 278, 76, 76, + /* 870 */ 1, 2, 34, 38, 249, 184, 185, 22, 72, 48, + /* 880 */ 22, 256, 76, 22, 226, 169, 72, 249, 219, 72, + /* 890 */ 76, 266, 72, 76, 256, 270, 76, 72, 238, 72, + /* 900 */ 275, 76, 72, 76, 266, 72, 76, 334, 270, 76, + /* 910 */ 325, 259, 282, 288, 289, 290, 291, 292, 293, 72, + /* 920 */ 295, 72, 319, 76, 228, 76, 288, 289, 290, 291, + /* 930 */ 292, 293, 312, 295, 110, 287, 298, 249, 72, 314, + /* 940 */ 302, 303, 76, 112, 225, 110, 328, 38, 315, 36, + /* 950 */ 33, 72, 327, 36, 72, 76, 331, 20, 76, 42, + /* 960 */ 231, 44, 45, 46, 47, 283, 57, 235, 249, 38, + /* 970 */ 276, 231, 141, 142, 143, 256, 145, 231, 146, 264, + /* 980 */ 121, 150, 134, 262, 262, 266, 231, 20, 280, 270, + /* 990 */ 233, 74, 266, 162, 77, 164, 225, 166, 167, 168, + /* 1000 */ 20, 274, 233, 256, 20, 267, 233, 288, 289, 290, + /* 1010 */ 291, 292, 293, 233, 295, 231, 233, 298, 20, 19, + /* 1020 */ 249, 302, 303, 227, 249, 249, 231, 256, 249, 227, + /* 1030 */ 199, 249, 57, 33, 249, 249, 36, 266, 249, 249, + /* 1040 */ 249, 270, 42, 249, 44, 45, 46, 47, 249, 230, + /* 1050 */ 280, 225, 135, 155, 137, 274, 139, 270, 230, 288, + /* 1060 */ 289, 290, 291, 292, 293, 294, 295, 296, 297, 266, + /* 1070 */ 230, 256, 267, 156, 74, 249, 279, 77, 20, 207, + /* 1080 */ 230, 271, 256, 324, 206, 287, 270, 324, 214, 271, + /* 1090 */ 213, 270, 266, 270, 202, 201, 270, 198, 256, 20, + /* 1100 */ 225, 221, 121, 335, 218, 216, 305, 107, 286, 321, + /* 1110 */ 75, 270, 301, 270, 288, 289, 290, 291, 292, 293, + /* 1120 */ 320, 295, 318, 271, 249, 317, 137, 271, 245, 256, + /* 1130 */ 270, 256, 268, 230, 267, 230, 136, 330, 329, 139, + /* 1140 */ 256, 266, 75, 252, 230, 270, 239, 231, 227, 281, + /* 1150 */ 232, 243, 225, 277, 154, 243, 156, 223, 332, 333, + /* 1160 */ 231, 0, 0, 288, 289, 290, 291, 292, 293, 64, + /* 1170 */ 295, 0, 38, 298, 165, 38, 249, 38, 303, 38, + /* 1180 */ 165, 0, 225, 256, 38, 256, 38, 165, 0, 38, + /* 1190 */ 0, 0, 38, 266, 75, 152, 151, 270, 110, 148, + /* 1200 */ 273, 0, 0, 53, 275, 144, 249, 0, 0, 87, /* 1210 */ 0, 0, 0, 256, 121, 288, 289, 290, 291, 292, /* 1220 */ 293, 292, 295, 266, 0, 0, 0, 270, 0, 0, /* 1230 */ 0, 0, 0, 225, 0, 0, 307, 308, 309, 0, /* 1240 */ 311, 0, 0, 314, 0, 288, 289, 290, 291, 292, - /* 1250 */ 293, 0, 295, 0, 0, 22, 327, 249, 0, 0, - /* 1260 */ 331, 0, 0, 225, 256, 0, 0, 0, 0, 51, - /* 1270 */ 43, 0, 0, 0, 266, 38, 36, 0, 270, 43, - /* 1280 */ 0, 273, 36, 38, 36, 38, 43, 249, 38, 0, - /* 1290 */ 333, 225, 43, 36, 256, 43, 288, 289, 290, 291, - /* 1300 */ 292, 293, 0, 295, 266, 0, 0, 84, 270, 22, - /* 1310 */ 0, 0, 38, 22, 0, 249, 38, 38, 0, 38, - /* 1320 */ 82, 38, 256, 38, 38, 22, 288, 289, 290, 291, - /* 1330 */ 292, 293, 266, 295, 38, 297, 270, 39, 22, 273, - /* 1340 */ 225, 72, 12, 13, 0, 38, 22, 0, 72, 22, - /* 1350 */ 22, 20, 22, 0, 288, 289, 290, 291, 292, 293, - /* 1360 */ 140, 295, 0, 38, 249, 38, 153, 22, 38, 0, - /* 1370 */ 0, 256, 0, 72, 75, 43, 72, 87, 225, 203, - /* 1380 */ 72, 266, 203, 197, 57, 270, 203, 57, 273, 225, - /* 1390 */ 4, 76, 75, 137, 75, 140, 76, 75, 135, 87, - /* 1400 */ 140, 72, 249, 288, 289, 290, 291, 292, 293, 256, - /* 1410 */ 295, 76, 75, 249, 72, 76, 76, 76, 72, 266, - /* 1420 */ 256, 87, 72, 270, 2, 98, 38, 38, 98, 38, - /* 1430 */ 266, 38, 38, 38, 270, 72, 76, 110, 87, 225, - /* 1440 */ 110, 288, 289, 290, 291, 292, 293, 76, 295, 75, - /* 1450 */ 75, 169, 288, 289, 290, 291, 292, 293, 76, 295, - /* 1460 */ 75, 22, 75, 249, 87, 171, 76, 75, 0, 43, - /* 1470 */ 256, 138, 76, 75, 75, 148, 75, 75, 148, 22, - /* 1480 */ 266, 75, 85, 135, 270, 76, 38, 87, 38, 225, - /* 1490 */ 87, 75, 38, 86, 76, 75, 169, 170, 38, 169, - /* 1500 */ 225, 76, 288, 289, 290, 291, 292, 293, 75, 295, - /* 1510 */ 76, 38, 75, 249, 76, 38, 75, 100, 22, 100, - /* 1520 */ 256, 100, 75, 100, 249, 38, 75, 75, 88, 38, - /* 1530 */ 266, 256, 22, 51, 270, 50, 38, 110, 57, 73, - /* 1540 */ 38, 266, 72, 38, 38, 270, 38, 38, 225, 38, - /* 1550 */ 38, 22, 288, 289, 290, 291, 292, 293, 57, 295, + /* 1250 */ 293, 0, 295, 0, 0, 0, 327, 249, 0, 0, + /* 1260 */ 331, 0, 22, 225, 256, 0, 0, 0, 0, 0, + /* 1270 */ 0, 0, 0, 0, 266, 82, 43, 0, 270, 0, + /* 1280 */ 0, 273, 36, 38, 36, 38, 43, 249, 51, 0, + /* 1290 */ 333, 225, 43, 43, 256, 38, 288, 289, 290, 291, + /* 1300 */ 292, 293, 0, 295, 266, 43, 0, 0, 270, 0, + /* 1310 */ 36, 0, 22, 0, 38, 249, 38, 36, 0, 22, + /* 1320 */ 84, 38, 256, 38, 72, 38, 288, 289, 290, 291, + /* 1330 */ 292, 293, 266, 295, 38, 297, 270, 38, 38, 273, + /* 1340 */ 225, 38, 12, 13, 0, 0, 22, 39, 22, 22, + /* 1350 */ 38, 0, 22, 22, 288, 289, 290, 291, 292, 293, + /* 1360 */ 72, 295, 0, 22, 249, 38, 20, 0, 38, 0, + /* 1370 */ 38, 256, 153, 0, 22, 137, 0, 0, 225, 75, + /* 1380 */ 43, 266, 72, 203, 57, 270, 72, 57, 273, 225, + /* 1390 */ 203, 76, 72, 76, 75, 197, 140, 75, 135, 203, + /* 1400 */ 140, 76, 249, 288, 289, 290, 291, 292, 293, 256, + /* 1410 */ 295, 76, 72, 249, 75, 87, 87, 75, 38, 266, + /* 1420 */ 256, 72, 140, 270, 76, 98, 72, 72, 98, 76, + /* 1430 */ 266, 38, 38, 38, 270, 38, 38, 110, 2, 225, + /* 1440 */ 110, 288, 289, 290, 291, 292, 293, 87, 295, 75, + /* 1450 */ 72, 76, 288, 289, 290, 291, 292, 293, 76, 295, + /* 1460 */ 169, 75, 75, 249, 76, 75, 87, 22, 87, 171, + /* 1470 */ 256, 75, 138, 0, 75, 148, 76, 76, 148, 75, + /* 1480 */ 266, 75, 43, 75, 270, 135, 75, 85, 22, 225, + /* 1490 */ 87, 76, 38, 38, 87, 86, 169, 170, 75, 169, + /* 1500 */ 225, 76, 288, 289, 290, 291, 292, 293, 38, 295, + /* 1510 */ 75, 22, 38, 249, 76, 75, 38, 76, 75, 38, + /* 1520 */ 256, 100, 76, 75, 249, 88, 100, 100, 100, 38, + /* 1530 */ 266, 256, 75, 75, 270, 75, 38, 22, 51, 110, + /* 1540 */ 50, 266, 38, 57, 73, 270, 38, 72, 225, 38, + /* 1550 */ 38, 38, 288, 289, 290, 291, 292, 293, 38, 295, /* 1560 */ 38, 225, 4, 288, 289, 290, 291, 292, 293, 38, - /* 1570 */ 295, 38, 249, 38, 38, 38, 38, 19, 38, 256, - /* 1580 */ 0, 38, 36, 0, 43, 249, 38, 36, 43, 266, - /* 1590 */ 0, 33, 256, 270, 36, 38, 36, 43, 0, 41, - /* 1600 */ 38, 36, 266, 0, 43, 47, 270, 38, 37, 0, - /* 1610 */ 0, 288, 289, 290, 291, 292, 293, 22, 295, 22, - /* 1620 */ 21, 20, 22, 21, 288, 289, 290, 291, 292, 293, - /* 1630 */ 225, 295, 74, 336, 336, 77, 336, 336, 336, 225, + /* 1570 */ 295, 22, 249, 57, 38, 38, 38, 19, 38, 256, + /* 1580 */ 38, 38, 38, 38, 0, 249, 38, 36, 0, 266, + /* 1590 */ 43, 33, 256, 270, 36, 38, 36, 43, 0, 41, + /* 1600 */ 38, 36, 266, 0, 43, 47, 270, 38, 36, 43, + /* 1610 */ 0, 288, 289, 290, 291, 292, 293, 38, 295, 37, + /* 1620 */ 0, 0, 22, 21, 288, 289, 290, 291, 292, 293, + /* 1630 */ 225, 295, 74, 22, 22, 77, 21, 20, 336, 225, /* 1640 */ 336, 336, 336, 336, 336, 336, 336, 336, 336, 336, /* 1650 */ 336, 336, 336, 336, 249, 336, 336, 336, 336, 336, /* 1660 */ 336, 256, 336, 249, 336, 336, 336, 336, 336, 336, @@ -608,154 +609,154 @@ static const YYCODETYPE yy_lookahead[] = { }; #define YY_SHIFT_COUNT (571) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1610) +#define YY_SHIFT_MAX (1621) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 831, 0, 39, 214, 214, 214, 214, 253, 214, 214, /* 10 */ 428, 467, 642, 467, 467, 467, 467, 467, 467, 467, /* 20 */ 467, 467, 467, 467, 467, 467, 467, 467, 467, 467, - /* 30 */ 467, 467, 467, 467, 467, 467, 71, 66, 66, 66, - /* 40 */ 133, 1330, 1330, 110, 38, 38, 62, 1330, 38, 38, - /* 50 */ 38, 38, 38, 38, 35, 38, 142, 208, 62, 298, - /* 60 */ 142, 38, 38, 142, 38, 142, 298, 142, 142, 38, - /* 70 */ 331, 217, 179, 393, 393, 389, 1327, 421, 187, 1327, + /* 30 */ 467, 467, 467, 467, 467, 467, 71, 383, 383, 383, + /* 40 */ 139, 1330, 1330, 110, 42, 42, 29, 1330, 348, 42, + /* 50 */ 42, 42, 42, 42, 42, 52, 42, 98, 140, 29, + /* 60 */ 142, 98, 42, 42, 98, 42, 98, 142, 98, 98, + /* 70 */ 42, 193, 217, 179, 393, 393, 389, 1327, 421, 187, /* 80 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, - /* 90 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 220, 327, - /* 100 */ 347, 347, 318, 318, 318, 500, 347, 347, 429, 298, - /* 110 */ 142, 298, 142, 473, 509, 741, 741, 741, 741, 741, - /* 120 */ 741, 741, 1000, 444, 564, 480, 111, 396, 120, 27, - /* 130 */ 663, 550, 518, 660, 518, 705, 592, 678, 777, 918, - /* 140 */ 911, 919, 814, 918, 918, 848, 837, 837, 918, 960, - /* 150 */ 35, 298, 964, 35, 429, 970, 35, 35, 918, 35, - /* 160 */ 984, 142, 142, 142, 142, 142, 142, 142, 142, 142, - /* 170 */ 142, 142, 918, 984, 969, 960, 331, 888, 298, 331, - /* 180 */ 964, 331, 429, 970, 331, 1037, 864, 870, 969, 864, - /* 190 */ 870, 969, 969, 874, 876, 885, 889, 893, 429, 1073, - /* 200 */ 973, 877, 883, 887, 1029, 142, 870, 969, 969, 870, - /* 210 */ 969, 972, 429, 970, 331, 473, 331, 429, 1058, 509, - /* 220 */ 918, 331, 984, 1960, 1960, 1960, 1960, 1960, 1960, 718, - /* 230 */ 917, 329, 1558, 47, 483, 503, 272, 280, 661, 734, - /* 240 */ 11, 11, 11, 11, 11, 11, 11, 11, 434, 505, - /* 250 */ 196, 409, 283, 283, 283, 283, 739, 301, 722, 713, - /* 260 */ 773, 774, 778, 118, 510, 861, 838, 645, 792, 793, - /* 270 */ 795, 884, 691, 29, 669, 801, 743, 808, 618, 817, - /* 280 */ 820, 825, 827, 833, 824, 832, 834, 835, 847, 849, - /* 290 */ 873, 879, 125, 897, 1150, 1151, 1089, 1155, 1119, 996, - /* 300 */ 1124, 1131, 1133, 1007, 1174, 1137, 1139, 1014, 1180, 1143, - /* 310 */ 1184, 1148, 1187, 1113, 1039, 1041, 1079, 1042, 1194, 1195, - /* 320 */ 1145, 1052, 1199, 1201, 1116, 1202, 1205, 1207, 1208, 1209, - /* 330 */ 1210, 1211, 1212, 1224, 1225, 1226, 1228, 1229, 1230, 1231, - /* 340 */ 1232, 1234, 1093, 1235, 1239, 1241, 1242, 1244, 1251, 1233, - /* 350 */ 1253, 1254, 1258, 1259, 1261, 1262, 1265, 1266, 1267, 1227, - /* 360 */ 1268, 1218, 1271, 1272, 1237, 1240, 1236, 1273, 1245, 1246, - /* 370 */ 1243, 1277, 1247, 1248, 1249, 1280, 1250, 1257, 1252, 1289, - /* 380 */ 1302, 1305, 1306, 1223, 1238, 1274, 1269, 1276, 1287, 1310, - /* 390 */ 1278, 1279, 1281, 1283, 1285, 1269, 1276, 1286, 1296, 1311, - /* 400 */ 1291, 1314, 1303, 1298, 1318, 1316, 1307, 1344, 1324, 1347, - /* 410 */ 1328, 1331, 1353, 1220, 1325, 1362, 1213, 1345, 1255, 1256, - /* 420 */ 1369, 1370, 1260, 1372, 1299, 1332, 1263, 1301, 1304, 1176, - /* 430 */ 1315, 1308, 1320, 1317, 1319, 1322, 1335, 1329, 1290, 1337, - /* 440 */ 1342, 1179, 1339, 1340, 1312, 1186, 1346, 1334, 1341, 1350, - /* 450 */ 1183, 1386, 1388, 1389, 1391, 1393, 1394, 1395, 1422, 1282, - /* 460 */ 1363, 1360, 1374, 1371, 1375, 1382, 1351, 1385, 1387, 1377, - /* 470 */ 1439, 1294, 1392, 1390, 1396, 1398, 1399, 1333, 1401, 1468, - /* 480 */ 1426, 1348, 1402, 1397, 1400, 1403, 1457, 1406, 1407, 1409, - /* 490 */ 1448, 1450, 1416, 1418, 1454, 1420, 1425, 1460, 1433, 1434, - /* 500 */ 1473, 1437, 1438, 1477, 1441, 1417, 1419, 1421, 1423, 1496, - /* 510 */ 1440, 1447, 1487, 1427, 1451, 1452, 1491, 1269, 1276, 1510, - /* 520 */ 1482, 1485, 1498, 1481, 1466, 1470, 1502, 1505, 1506, 1508, - /* 530 */ 1509, 1511, 1512, 1529, 1501, 1269, 1522, 1276, 1531, 1533, - /* 540 */ 1535, 1536, 1537, 1538, 1540, 1580, 1543, 1546, 1541, 1583, - /* 550 */ 1548, 1551, 1545, 1590, 1557, 1560, 1554, 1598, 1562, 1565, - /* 560 */ 1561, 1603, 1569, 1571, 1609, 1610, 1595, 1599, 1597, 1600, - /* 570 */ 1602, 1601, + /* 90 */ 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 1327, 665, + /* 100 */ 327, 109, 109, 360, 360, 360, 667, 109, 109, 312, + /* 110 */ 142, 98, 142, 98, 422, 345, 741, 741, 741, 741, + /* 120 */ 741, 741, 741, 1000, 444, 564, 480, 111, 396, 120, + /* 130 */ 27, 681, 546, 597, 342, 597, 687, 478, 264, 777, + /* 140 */ 937, 913, 931, 832, 937, 937, 859, 848, 848, 937, + /* 150 */ 967, 52, 142, 980, 52, 312, 984, 52, 52, 937, + /* 160 */ 52, 998, 98, 98, 98, 98, 98, 98, 98, 98, + /* 170 */ 98, 98, 98, 937, 998, 975, 967, 193, 898, 142, + /* 180 */ 193, 980, 193, 312, 984, 193, 1058, 872, 878, 975, + /* 190 */ 872, 878, 975, 975, 874, 877, 892, 894, 899, 312, + /* 200 */ 1079, 981, 880, 886, 889, 1035, 98, 878, 975, 975, + /* 210 */ 878, 975, 989, 312, 984, 193, 422, 193, 312, 1067, + /* 220 */ 345, 937, 193, 998, 1960, 1960, 1960, 1960, 1960, 1960, + /* 230 */ 718, 917, 329, 1558, 47, 483, 503, 272, 280, 661, + /* 240 */ 734, 11, 11, 11, 11, 11, 11, 11, 11, 94, + /* 250 */ 505, 196, 409, 283, 283, 283, 283, 595, 571, 373, + /* 260 */ 547, 657, 774, 778, 855, 858, 861, 838, 645, 737, + /* 270 */ 792, 793, 869, 691, 332, 669, 806, 716, 814, 754, + /* 280 */ 817, 820, 825, 827, 830, 824, 835, 833, 847, 849, + /* 290 */ 866, 879, 882, 125, 909, 1161, 1162, 1105, 1171, 1134, + /* 300 */ 1009, 1137, 1139, 1141, 1015, 1181, 1146, 1148, 1022, 1188, + /* 310 */ 1151, 1190, 1154, 1191, 1119, 1043, 1045, 1088, 1051, 1201, + /* 320 */ 1202, 1150, 1061, 1207, 1208, 1122, 1210, 1211, 1212, 1224, + /* 330 */ 1225, 1226, 1228, 1229, 1230, 1231, 1232, 1234, 1235, 1239, + /* 340 */ 1241, 1242, 1244, 1093, 1251, 1253, 1254, 1255, 1258, 1259, + /* 350 */ 1240, 1261, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, + /* 360 */ 1233, 1273, 1237, 1277, 1279, 1245, 1246, 1243, 1280, 1247, + /* 370 */ 1248, 1249, 1289, 1257, 1274, 1250, 1302, 1278, 1281, 1262, + /* 380 */ 1306, 1307, 1309, 1311, 1236, 1193, 1276, 1252, 1288, 1290, + /* 390 */ 1313, 1283, 1285, 1287, 1296, 1299, 1252, 1288, 1300, 1303, + /* 400 */ 1318, 1297, 1344, 1324, 1308, 1345, 1326, 1312, 1351, 1331, + /* 410 */ 1362, 1341, 1346, 1367, 1256, 1332, 1369, 1219, 1352, 1260, + /* 420 */ 1238, 1373, 1376, 1282, 1377, 1304, 1337, 1263, 1310, 1314, + /* 430 */ 1180, 1315, 1320, 1317, 1319, 1322, 1339, 1325, 1340, 1328, + /* 440 */ 1342, 1349, 1187, 1335, 1348, 1329, 1198, 1354, 1360, 1353, + /* 450 */ 1355, 1196, 1380, 1393, 1394, 1395, 1397, 1398, 1436, 1291, + /* 460 */ 1378, 1375, 1374, 1382, 1386, 1388, 1379, 1387, 1390, 1381, + /* 470 */ 1445, 1298, 1396, 1400, 1401, 1399, 1404, 1334, 1406, 1473, + /* 480 */ 1439, 1350, 1408, 1402, 1403, 1407, 1466, 1411, 1409, 1415, + /* 490 */ 1454, 1455, 1423, 1425, 1470, 1435, 1438, 1474, 1440, 1441, + /* 500 */ 1478, 1443, 1446, 1481, 1448, 1421, 1426, 1427, 1428, 1489, + /* 510 */ 1437, 1457, 1491, 1429, 1458, 1460, 1498, 1252, 1288, 1515, + /* 520 */ 1487, 1490, 1504, 1486, 1471, 1475, 1508, 1511, 1512, 1513, + /* 530 */ 1520, 1522, 1531, 1549, 1516, 1252, 1536, 1288, 1537, 1538, + /* 540 */ 1540, 1542, 1543, 1544, 1545, 1584, 1548, 1551, 1547, 1588, + /* 550 */ 1557, 1560, 1554, 1598, 1562, 1565, 1561, 1603, 1569, 1572, + /* 560 */ 1566, 1610, 1579, 1582, 1620, 1621, 1600, 1602, 1611, 1612, + /* 570 */ 1615, 1617, }; -#define YY_REDUCE_COUNT (228) -#define YY_REDUCE_MIN (-263) +#define YY_REDUCE_COUNT (229) +#define YY_REDUCE_MIN (-273) #define YY_REDUCE_MAX (1664) static const short yy_reduce_ofst[] = { /* 0 */ -212, -210, -159, 368, 13, 265, 440, 625, 638, 719, /* 10 */ 771, 826, 875, 927, -221, 957, 1008, 1038, 1066, 1115, /* 20 */ 1153, 1164, 1214, 1264, 1275, 1323, 1336, 1405, 1414, 1464, /* 30 */ 1514, 1525, 1573, 1586, 1655, 1664, 929, -183, -104, 453, - /* 40 */ -125, -253, -251, -211, -226, 266, 50, -241, -118, -89, - /* 50 */ 222, 267, 294, 306, -192, 438, -59, -248, 51, -263, - /* 60 */ 206, 448, 471, -6, 481, 210, -171, 281, 271, 452, - /* 70 */ -83, -224, -200, -200, -200, -208, 82, -88, -198, 154, + /* 40 */ -125, -253, -251, -211, -226, 266, 50, -241, -273, -118, + /* 50 */ -89, 222, 267, 294, 306, -93, 438, -59, -248, 51, + /* 60 */ -263, 206, 448, 471, -6, 481, 210, -171, 281, 271, + /* 70 */ 452, 16, -224, -200, -200, -200, -208, 154, 59, -198, /* 80 */ 221, 263, 304, 398, 456, 488, 489, 495, 497, 498, - /* 90 */ 499, 526, 529, 530, 532, 538, 578, 581, 174, -178, - /* 100 */ 32, 176, -10, 4, 289, 322, 397, 451, -97, 188, - /* 110 */ 290, 432, 291, -12, 392, -239, 376, 417, 445, 457, - /* 120 */ 484, 486, 478, 540, 571, 479, 504, 582, 600, 579, - /* 130 */ 662, 615, 610, 610, 610, 675, 604, 622, 675, 715, - /* 140 */ 665, 723, 689, 735, 736, 706, 716, 717, 746, 702, - /* 150 */ 750, 720, 714, 754, 744, 725, 768, 769, 772, 780, - /* 160 */ 779, 756, 766, 767, 775, 776, 782, 783, 785, 786, - /* 170 */ 789, 790, 787, 796, 758, 749, 810, 770, 784, 818, - /* 180 */ 781, 823, 802, 803, 839, 791, 748, 809, 811, 755, - /* 190 */ 812, 815, 816, 763, 788, 794, 805, 610, 841, 813, - /* 200 */ 797, 760, 798, 800, 804, 675, 840, 836, 843, 852, - /* 210 */ 850, 842, 869, 859, 900, 882, 902, 878, 886, 896, - /* 220 */ 906, 910, 915, 866, 863, 903, 904, 916, 926, + /* 90 */ 499, 526, 529, 530, 532, 538, 578, 581, 582, 183, + /* 100 */ -178, 275, 316, -10, 4, 289, 10, 397, 462, 276, + /* 110 */ 188, 433, 527, 310, -12, -153, -239, 323, 445, 475, + /* 120 */ 506, 525, 548, 589, 658, 660, 573, 585, 652, 630, + /* 130 */ 603, 696, 648, 620, 620, 620, 688, 618, 633, 688, + /* 140 */ 729, 682, 732, 694, 740, 746, 715, 721, 722, 755, + /* 150 */ 708, 757, 726, 727, 769, 747, 738, 773, 780, 784, + /* 160 */ 783, 796, 775, 776, 779, 782, 785, 786, 789, 790, + /* 170 */ 791, 794, 799, 795, 802, 787, 770, 819, 797, 803, + /* 180 */ 828, 781, 840, 815, 805, 850, 798, 759, 810, 816, + /* 190 */ 763, 818, 821, 823, 788, 800, 804, 808, 620, 842, + /* 200 */ 822, 801, 768, 807, 809, 811, 688, 852, 841, 843, + /* 210 */ 856, 860, 864, 873, 867, 903, 883, 905, 884, 891, + /* 220 */ 907, 916, 914, 921, 876, 868, 908, 912, 918, 934, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 10 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 20 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 30 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 40 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 50 */ 1285, 1285, 1285, 1285, 1344, 1285, 1285, 1285, 1285, 1285, - /* 60 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 70 */ 1342, 1487, 1285, 1643, 1285, 1285, 1285, 1285, 1285, 1285, - /* 80 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 90 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1344, - /* 100 */ 1285, 1285, 1654, 1654, 1654, 1342, 1285, 1285, 1285, 1285, - /* 110 */ 1285, 1285, 1285, 1439, 1285, 1285, 1285, 1285, 1285, 1285, - /* 120 */ 1285, 1285, 1521, 1285, 1285, 1718, 1285, 1392, 1527, 1678, - /* 130 */ 1285, 1670, 1646, 1660, 1647, 1285, 1703, 1663, 1285, 1285, - /* 140 */ 1285, 1285, 1513, 1285, 1285, 1492, 1489, 1489, 1285, 1285, - /* 150 */ 1344, 1285, 1285, 1344, 1285, 1285, 1344, 1344, 1285, 1344, - /* 160 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 170 */ 1285, 1285, 1285, 1285, 1285, 1285, 1342, 1523, 1285, 1342, - /* 180 */ 1285, 1342, 1285, 1285, 1342, 1285, 1685, 1683, 1285, 1685, - /* 190 */ 1683, 1285, 1285, 1697, 1693, 1676, 1674, 1660, 1285, 1285, - /* 200 */ 1285, 1721, 1709, 1705, 1285, 1285, 1683, 1285, 1285, 1683, - /* 210 */ 1285, 1500, 1285, 1285, 1342, 1285, 1342, 1285, 1408, 1285, - /* 220 */ 1285, 1342, 1285, 1515, 1529, 1442, 1442, 1345, 1290, 1285, - /* 230 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 240 */ 1590, 1696, 1695, 1619, 1618, 1617, 1615, 1589, 1285, 1285, - /* 250 */ 1285, 1285, 1583, 1584, 1582, 1581, 1285, 1285, 1285, 1285, - /* 260 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 270 */ 1285, 1644, 1285, 1706, 1710, 1285, 1285, 1285, 1567, 1285, - /* 280 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 290 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 300 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 310 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 320 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 330 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 340 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 350 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 360 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 370 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 380 */ 1285, 1285, 1285, 1285, 1285, 1285, 1455, 1454, 1285, 1285, - /* 390 */ 1285, 1285, 1285, 1285, 1285, 1372, 1371, 1285, 1285, 1285, - /* 400 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 410 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 420 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1667, 1677, 1285, - /* 430 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1567, 1285, - /* 440 */ 1694, 1285, 1653, 1649, 1285, 1285, 1645, 1285, 1285, 1704, - /* 450 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1639, 1285, - /* 460 */ 1612, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 470 */ 1285, 1577, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 480 */ 1285, 1285, 1285, 1285, 1566, 1285, 1285, 1285, 1285, 1285, - /* 490 */ 1285, 1285, 1436, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 500 */ 1285, 1285, 1285, 1285, 1285, 1421, 1419, 1418, 1417, 1285, - /* 510 */ 1414, 1285, 1285, 1285, 1285, 1285, 1285, 1445, 1444, 1285, - /* 520 */ 1285, 1285, 1285, 1285, 1285, 1365, 1285, 1285, 1285, 1285, - /* 530 */ 1285, 1285, 1285, 1285, 1285, 1356, 1285, 1355, 1285, 1285, - /* 540 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 550 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 560 */ 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, 1285, - /* 570 */ 1285, 1285, + /* 0 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 10 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 20 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 30 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 40 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 50 */ 1287, 1287, 1287, 1287, 1287, 1346, 1287, 1287, 1287, 1287, + /* 60 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 70 */ 1287, 1344, 1489, 1287, 1645, 1287, 1287, 1287, 1287, 1287, + /* 80 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 90 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 100 */ 1346, 1287, 1287, 1656, 1656, 1656, 1344, 1287, 1287, 1287, + /* 110 */ 1287, 1287, 1287, 1287, 1441, 1287, 1287, 1287, 1287, 1287, + /* 120 */ 1287, 1287, 1287, 1523, 1287, 1287, 1721, 1287, 1394, 1529, + /* 130 */ 1680, 1287, 1672, 1648, 1662, 1649, 1287, 1706, 1665, 1287, + /* 140 */ 1287, 1287, 1287, 1515, 1287, 1287, 1494, 1491, 1491, 1287, + /* 150 */ 1287, 1346, 1287, 1287, 1346, 1287, 1287, 1346, 1346, 1287, + /* 160 */ 1346, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 170 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1344, 1525, 1287, + /* 180 */ 1344, 1287, 1344, 1287, 1287, 1344, 1287, 1687, 1685, 1287, + /* 190 */ 1687, 1685, 1287, 1287, 1699, 1695, 1678, 1676, 1662, 1287, + /* 200 */ 1287, 1287, 1724, 1712, 1708, 1287, 1287, 1685, 1287, 1287, + /* 210 */ 1685, 1287, 1502, 1287, 1287, 1344, 1287, 1344, 1287, 1410, + /* 220 */ 1287, 1287, 1344, 1287, 1517, 1531, 1444, 1444, 1347, 1292, + /* 230 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 240 */ 1287, 1592, 1698, 1697, 1621, 1620, 1619, 1617, 1591, 1287, + /* 250 */ 1287, 1287, 1287, 1585, 1586, 1584, 1583, 1287, 1287, 1287, + /* 260 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 270 */ 1287, 1287, 1646, 1287, 1709, 1713, 1287, 1287, 1287, 1569, + /* 280 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 290 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 300 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 310 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 320 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 330 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 340 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 350 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 360 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 370 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 380 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1457, 1456, 1287, + /* 390 */ 1287, 1287, 1287, 1287, 1287, 1287, 1374, 1373, 1287, 1287, + /* 400 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 410 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 420 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1669, 1679, + /* 430 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1569, + /* 440 */ 1287, 1696, 1287, 1655, 1651, 1287, 1287, 1647, 1287, 1287, + /* 450 */ 1707, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1641, 1287, + /* 460 */ 1614, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 470 */ 1287, 1579, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 480 */ 1287, 1287, 1287, 1287, 1568, 1287, 1287, 1287, 1287, 1287, + /* 490 */ 1287, 1287, 1438, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 500 */ 1287, 1287, 1287, 1287, 1287, 1423, 1421, 1420, 1419, 1287, + /* 510 */ 1416, 1287, 1287, 1287, 1287, 1287, 1287, 1447, 1446, 1287, + /* 520 */ 1287, 1287, 1287, 1287, 1287, 1367, 1287, 1287, 1287, 1287, + /* 530 */ 1287, 1287, 1287, 1287, 1287, 1358, 1287, 1357, 1287, 1287, + /* 540 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 550 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 560 */ 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, 1287, + /* 570 */ 1287, 1287, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1619,28 +1620,29 @@ static const char *const yyRuleName[] = { /* 411 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", /* 412 */ "query_expression_body ::= query_primary", /* 413 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 414 */ "query_primary ::= query_specification", - /* 415 */ "order_by_clause_opt ::=", - /* 416 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 417 */ "slimit_clause_opt ::=", - /* 418 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 419 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 420 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 421 */ "limit_clause_opt ::=", - /* 422 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 423 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 424 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 425 */ "subquery ::= NK_LP query_expression NK_RP", - /* 426 */ "search_condition ::= common_expression", - /* 427 */ "sort_specification_list ::= sort_specification", - /* 428 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 429 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 430 */ "ordering_specification_opt ::=", - /* 431 */ "ordering_specification_opt ::= ASC", - /* 432 */ "ordering_specification_opt ::= DESC", - /* 433 */ "null_ordering_opt ::=", - /* 434 */ "null_ordering_opt ::= NULLS FIRST", - /* 435 */ "null_ordering_opt ::= NULLS LAST", + /* 414 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 415 */ "query_primary ::= query_specification", + /* 416 */ "order_by_clause_opt ::=", + /* 417 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 418 */ "slimit_clause_opt ::=", + /* 419 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 420 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 421 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 422 */ "limit_clause_opt ::=", + /* 423 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 424 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 425 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 426 */ "subquery ::= NK_LP query_expression NK_RP", + /* 427 */ "search_condition ::= common_expression", + /* 428 */ "sort_specification_list ::= sort_specification", + /* 429 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 430 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 431 */ "ordering_specification_opt ::=", + /* 432 */ "ordering_specification_opt ::= ASC", + /* 433 */ "ordering_specification_opt ::= DESC", + /* 434 */ "null_ordering_opt ::=", + /* 435 */ "null_ordering_opt ::= NULLS FIRST", + /* 436 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2637,28 +2639,29 @@ static const struct { { 275, -4 }, /* (411) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { 327, -1 }, /* (412) query_expression_body ::= query_primary */ { 327, -4 }, /* (413) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 331, -1 }, /* (414) query_primary ::= query_specification */ - { 328, 0 }, /* (415) order_by_clause_opt ::= */ - { 328, -3 }, /* (416) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 329, 0 }, /* (417) slimit_clause_opt ::= */ - { 329, -2 }, /* (418) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 329, -4 }, /* (419) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 329, -4 }, /* (420) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 330, 0 }, /* (421) limit_clause_opt ::= */ - { 330, -2 }, /* (422) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 330, -4 }, /* (423) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 330, -4 }, /* (424) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 292, -3 }, /* (425) subquery ::= NK_LP query_expression NK_RP */ - { 313, -1 }, /* (426) search_condition ::= common_expression */ - { 332, -1 }, /* (427) sort_specification_list ::= sort_specification */ - { 332, -3 }, /* (428) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 333, -3 }, /* (429) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 334, 0 }, /* (430) ordering_specification_opt ::= */ - { 334, -1 }, /* (431) ordering_specification_opt ::= ASC */ - { 334, -1 }, /* (432) ordering_specification_opt ::= DESC */ - { 335, 0 }, /* (433) null_ordering_opt ::= */ - { 335, -2 }, /* (434) null_ordering_opt ::= NULLS FIRST */ - { 335, -2 }, /* (435) null_ordering_opt ::= NULLS LAST */ + { 327, -3 }, /* (414) query_expression_body ::= query_expression_body UNION query_expression_body */ + { 331, -1 }, /* (415) query_primary ::= query_specification */ + { 328, 0 }, /* (416) order_by_clause_opt ::= */ + { 328, -3 }, /* (417) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 329, 0 }, /* (418) slimit_clause_opt ::= */ + { 329, -2 }, /* (419) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 329, -4 }, /* (420) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 329, -4 }, /* (421) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 330, 0 }, /* (422) limit_clause_opt ::= */ + { 330, -2 }, /* (423) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 330, -4 }, /* (424) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 330, -4 }, /* (425) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 292, -3 }, /* (426) subquery ::= NK_LP query_expression NK_RP */ + { 313, -1 }, /* (427) search_condition ::= common_expression */ + { 332, -1 }, /* (428) sort_specification_list ::= sort_specification */ + { 332, -3 }, /* (429) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 333, -3 }, /* (430) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 334, 0 }, /* (431) ordering_specification_opt ::= */ + { 334, -1 }, /* (432) ordering_specification_opt ::= ASC */ + { 334, -1 }, /* (433) ordering_specification_opt ::= DESC */ + { 335, 0 }, /* (434) null_ordering_opt ::= */ + { 335, -2 }, /* (435) null_ordering_opt ::= NULLS FIRST */ + { 335, -2 }, /* (436) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3052,7 +3055,7 @@ static YYACTIONTYPE yy_reduce( case 275: /* literal_list ::= signed_literal */ yytestcase(yyruleno==275); case 325: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==325); case 380: /* select_sublist ::= select_item */ yytestcase(yyruleno==380); - case 427: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==427); + case 428: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==428); { yylhsminor.yy376 = createNodeList(pCxt, yymsp[0].minor.yy392); } yymsp[0].minor.yy376 = yylhsminor.yy376; break; @@ -3064,7 +3067,7 @@ static YYACTIONTYPE yy_reduce( case 276: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==276); case 326: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==326); case 381: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==381); - case 428: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==428); + case 429: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==429); { yylhsminor.yy376 = addNodeToList(pCxt, yymsp[-2].minor.yy376, yymsp[0].minor.yy392); } yymsp[-2].minor.yy376 = yylhsminor.yy376; break; @@ -3147,7 +3150,7 @@ static YYACTIONTYPE yy_reduce( case 151: /* tags_def_opt ::= */ yytestcase(yyruleno==151); case 388: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==388); case 405: /* group_by_clause_opt ::= */ yytestcase(yyruleno==405); - case 415: /* order_by_clause_opt ::= */ yytestcase(yyruleno==415); + case 416: /* order_by_clause_opt ::= */ yytestcase(yyruleno==416); { yymsp[1].minor.yy376 = NULL; } break; case 121: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ @@ -3394,8 +3397,8 @@ static YYACTIONTYPE yy_reduce( case 395: /* sliding_opt ::= */ yytestcase(yyruleno==395); case 397: /* fill_opt ::= */ yytestcase(yyruleno==397); case 409: /* having_clause_opt ::= */ yytestcase(yyruleno==409); - case 417: /* slimit_clause_opt ::= */ yytestcase(yyruleno==417); - case 421: /* limit_clause_opt ::= */ yytestcase(yyruleno==421); + case 418: /* slimit_clause_opt ::= */ yytestcase(yyruleno==418); + case 422: /* limit_clause_opt ::= */ yytestcase(yyruleno==422); { yymsp[1].minor.yy392 = NULL; } break; case 202: /* like_pattern_opt ::= LIKE NK_STRING */ @@ -3565,7 +3568,7 @@ static YYACTIONTYPE yy_reduce( case 361: /* table_reference ::= joined_table */ yytestcase(yyruleno==361); case 365: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==365); case 412: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==412); - case 414: /* query_primary ::= query_specification */ yytestcase(yyruleno==414); + case 415: /* query_primary ::= query_specification */ yytestcase(yyruleno==415); { yylhsminor.yy392 = yymsp[0].minor.yy392; } yymsp[0].minor.yy392 = yylhsminor.yy392; break; @@ -3625,7 +3628,7 @@ static YYACTIONTYPE yy_reduce( case 273: /* signed_literal ::= duration_literal */ case 327: /* star_func_para ::= expression */ yytestcase(yyruleno==327); case 382: /* select_item ::= common_expression */ yytestcase(yyruleno==382); - case 426: /* search_condition ::= common_expression */ yytestcase(yyruleno==426); + case 427: /* search_condition ::= common_expression */ yytestcase(yyruleno==427); { yylhsminor.yy392 = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); } yymsp[0].minor.yy392 = yylhsminor.yy392; break; @@ -3917,7 +3920,7 @@ static YYACTIONTYPE yy_reduce( break; case 389: /* partition_by_clause_opt ::= PARTITION BY expression_list */ case 406: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==406); - case 416: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==416); + case 417: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==417); { yymsp[-2].minor.yy376 = yymsp[0].minor.yy376; } break; case 391: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ @@ -3976,42 +3979,46 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy392, yymsp[0].minor.yy392); } yymsp[-3].minor.yy392 = yylhsminor.yy392; break; - case 418: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 422: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==422); + case 414: /* query_expression_body ::= query_expression_body UNION query_expression_body */ +{ yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } + yymsp[-2].minor.yy392 = yylhsminor.yy392; + break; + case 419: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 423: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==423); { yymsp[-1].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 419: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 423: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==423); + case 420: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 424: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==424); { yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 420: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 424: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==424); + case 421: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 425: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==425); { yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 425: /* subquery ::= NK_LP query_expression NK_RP */ + case 426: /* subquery ::= NK_LP query_expression NK_RP */ { yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy392); } yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 429: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + case 430: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ { yylhsminor.yy392 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), yymsp[-1].minor.yy386, yymsp[0].minor.yy361); } yymsp[-2].minor.yy392 = yylhsminor.yy392; break; - case 430: /* ordering_specification_opt ::= */ + case 431: /* ordering_specification_opt ::= */ { yymsp[1].minor.yy386 = ORDER_ASC; } break; - case 431: /* ordering_specification_opt ::= ASC */ + case 432: /* ordering_specification_opt ::= ASC */ { yymsp[0].minor.yy386 = ORDER_ASC; } break; - case 432: /* ordering_specification_opt ::= DESC */ + case 433: /* ordering_specification_opt ::= DESC */ { yymsp[0].minor.yy386 = ORDER_DESC; } break; - case 433: /* null_ordering_opt ::= */ + case 434: /* null_ordering_opt ::= */ { yymsp[1].minor.yy361 = NULL_ORDER_DEFAULT; } break; - case 434: /* null_ordering_opt ::= NULLS FIRST */ + case 435: /* null_ordering_opt ::= NULLS FIRST */ { yymsp[-1].minor.yy361 = NULL_ORDER_FIRST; } break; - case 435: /* null_ordering_opt ::= NULLS LAST */ + case 436: /* null_ordering_opt ::= NULLS LAST */ { yymsp[-1].minor.yy361 = NULL_ORDER_LAST; } break; default: diff --git a/source/libs/parser/test/CMakeLists.txt b/source/libs/parser/test/CMakeLists.txt index 869232002a..96de3fb735 100644 --- a/source/libs/parser/test/CMakeLists.txt +++ b/source/libs/parser/test/CMakeLists.txt @@ -17,3 +17,11 @@ TARGET_LINK_LIBRARIES( parserTest PUBLIC os util common nodes parser catalog transport gtest function planner qcom ) + +if(${BUILD_WINGETOPT}) + target_include_directories( + parserTest + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) + target_link_libraries(parserTest PUBLIC wingetopt) +endif() \ No newline at end of file diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 5723b93f8b..0f3e2fe414 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -12,7 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - #include #include "stub.h" @@ -23,6 +22,9 @@ #pragma GCC diagnostic pop +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "mockCatalog.h" namespace { diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 3ef0eaed42..d36448b686 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -40,13 +40,11 @@ public: virtual TableBuilder& setVgid(int16_t vgid) { schema()->vgId = vgid; - SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, }; - - vgroup.epSet.eps[0] = (SEp){"dnode_1", 6030}; - vgroup.epSet.eps[1] = (SEp){"dnode_2", 6030}; - vgroup.epSet.eps[2] = (SEp){"dnode_3", 6030}; + SVgroupInfo vgroup = { vgid, 0, 0, {0}, 0}; + addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); + addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); vgroup.epSet.inUse = 0; - vgroup.epSet.numOfEps = 3; meta_->vgs.emplace_back(vgroup); return *this; @@ -148,7 +146,7 @@ public: meta_[db][tbname]->schema->uid = id_++; meta_[db][tbname]->schema->tableType = TSDB_CHILD_TABLE; - SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,}; + SVgroupInfo vgroup = { vgid, 0, 0, {0}, 0}; addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); diff --git a/source/libs/parser/test/parserInsertTest.cpp b/source/libs/parser/test/parserInsertTest.cpp index d292fcf8b0..4aec141b55 100644 --- a/source/libs/parser/test/parserInsertTest.cpp +++ b/source/libs/parser/test/parserInsertTest.cpp @@ -15,6 +15,7 @@ #include +#include "os.h" #include "parInt.h" using namespace std; diff --git a/source/libs/parser/test/parserTestMain.cpp b/source/libs/parser/test/parserTestMain.cpp index 8cd69418b9..95faadfc34 100644 --- a/source/libs/parser/test/parserTestMain.cpp +++ b/source/libs/parser/test/parserTestMain.cpp @@ -14,14 +14,20 @@ */ #include +#include +#include #include #include -#include "mockCatalog.h" +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif +#include "os.h" #include "parserTestUtil.h" #include "parToken.h" #include "functionMgt.h" +#include "mockCatalog.h" bool g_isDump = false; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 78adea15ce..b13c0a8888 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -22,6 +22,7 @@ typedef struct SLogicPlanContext { } SLogicPlanContext; typedef int32_t (*FCreateLogicNode)(SLogicPlanContext*, SSelectStmt*, SLogicNode**); +typedef int32_t (*FCreateSetOpLogicNode)(SLogicPlanContext*, SSetOperator*, SLogicNode**); static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNode* pTable, SLogicNode** pLogicNode); static int32_t createQueryLogicNode(SLogicPlanContext* pCxt, SNode* pStmt, SLogicNode** pLogicNode); @@ -84,13 +85,19 @@ static EDealRes doNameExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t rewriteExpr(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { +static int32_t rewriteExprForSelect(SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { nodesWalkExprs(pExprs, doNameExpr, NULL); SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); return cxt.errCode; } +static int32_t rewriteExprs(SNodeList* pExprs, SNodeList* pTarget) { + SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; + nodesRewriteExprs(pTarget, doRewriteExpr, &cxt); + return cxt.errCode; +} + static int32_t pushLogicNode(SLogicPlanContext* pCxt, SLogicNode** pOldRoot, SLogicNode* pNewRoot) { if (NULL == pNewRoot->pChildren) { pNewRoot->pChildren = nodesMakeList(); @@ -343,7 +350,9 @@ static SColumnNode* createColumnByExpr(const char* pStmtName, SExprNode* pExpr) } pCol->node.resType = pExpr->resType; strcpy(pCol->colName, pExpr->aliasName); - strcpy(pCol->tableAlias, pStmtName); + if (NULL != pStmtName) { + strcpy(pCol->tableAlias, pStmtName); + } return pCol; } @@ -430,10 +439,10 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + code = rewriteExprForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); } if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { @@ -469,7 +478,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm } if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); + code = rewriteExprForSelect(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW); } if (TSDB_CODE_SUCCESS == code) { @@ -720,7 +729,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { - code = rewriteExpr(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT); + code = rewriteExprForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_DISTINCT); } // set the output @@ -768,11 +777,160 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return code; } +static int32_t createSetOpChildLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, FCreateSetOpLogicNode func, SLogicNode** pRoot) { + SLogicNode* pNode = NULL; + int32_t code = func(pCxt, pSetOperator, &pNode); + if (TSDB_CODE_SUCCESS == code && NULL != pNode) { + code = pushLogicNode(pCxt, pRoot, pNode); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNode); + } + return code; +} + +static int32_t createSetOpSortLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + if (NULL == pSetOperator->pOrderByList) { + return TSDB_CODE_SUCCESS; + } + + SSortLogicNode* pSort = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (NULL == pSort) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + + pSort->node.pTargets = nodesCloneList(pSetOperator->pProjectionList); + if (NULL == pSort->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + pSort->pSortKeys = nodesCloneList(pSetOperator->pOrderByList); + if (NULL == pSort->pSortKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pSort; + } else { + nodesDestroyNode(pSort); + } + + return code; +} + +static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + SProjectLogicNode* pProject = (SProjectLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_PROJECT); + if (NULL == pProject) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (NULL != pSetOperator->pLimit) { + pProject->limit = ((SLimitNode*)pSetOperator->pLimit)->limit; + pProject->offset = ((SLimitNode*)pSetOperator->pLimit)->offset; + } else { + pProject->limit = -1; + pProject->offset = -1; + } + + int32_t code = TSDB_CODE_SUCCESS; + + pProject->pProjections = nodesCloneList(pSetOperator->pProjectionList); + if (NULL == pProject->pProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByProjections(pCxt, NULL, pSetOperator->pProjectionList, &pProject->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pProject; + } else { + nodesDestroyNode(pProject); + } + + return code; +} + +static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); + if (NULL == pAgg) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t code = TSDB_CODE_SUCCESS; + pAgg->pGroupKeys = nodesCloneList(pSetOperator->pProjectionList); + if (NULL == pAgg->pGroupKeys) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprs(pAgg->pGroupKeys, pSetOperator->pOrderByList); + } + + // set the output + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pAgg; + } else { + nodesDestroyNode(pAgg); + } + + return code; +} + +static int32_t createSetOpLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { + SLogicNode* pSetOp = NULL; + int32_t code = TSDB_CODE_SUCCESS; + switch (pSetOperator->opType) { + case SET_OP_TYPE_UNION_ALL: + code = createSetOpProjectLogicNode(pCxt, pSetOperator, &pSetOp); + break; + case SET_OP_TYPE_UNION: + code = createSetOpAggLogicNode(pCxt, pSetOperator, &pSetOp); + break; + default: + code = -1; + break; + } + + SLogicNode* pLeft = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = createQueryLogicNode(pCxt, pSetOperator->pLeft, &pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pSetOp->pChildren, (SNode*)pLeft); + } + SLogicNode* pRight = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = createQueryLogicNode(pCxt, pSetOperator->pRight, &pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pSetOp->pChildren, (SNode*)pRight); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pSetOp; + } else { + nodesDestroyNode(pSetOp); + } + + return code; +} + static int32_t createSetOperatorLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, SLogicNode** pLogicNode) { SLogicNode* pRoot = NULL; - int32_t code = createQueryLogicNode(pCxt, pSetOperator->pLeft, &pRoot); + int32_t code = createSetOpLogicNode(pCxt, pSetOperator, &pRoot); if (TSDB_CODE_SUCCESS == code) { - code = createQueryLogicNode(pCxt, pSetOperator->pRight, &pRoot); + code = createSetOpChildLogicNode(pCxt, pSetOperator, createSetOpSortLogicNode, &pRoot); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 34bee3fd0b..832135e90e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -228,6 +228,8 @@ static int32_t cpdMergeConds(SNode** pDst, SNodeList** pSrc) { if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogicCond->condType = LOGIC_COND_TYPE_AND; pLogicCond->pParameterList = *pSrc; *pDst = (SNode*)pLogicCond; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index caeab20970..745029730f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -38,7 +38,7 @@ typedef struct SPhysiPlanContext { static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; - if (NULL != pStmtName) { + if (NULL != pStmtName && '\0' != pStmtName[0]) { return sprintf(pKey, "%s.%s", pStmtName, pCol->node.aliasName); } if ('\0' == pCol->tableAlias[0]) { @@ -47,7 +47,7 @@ static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) { return sprintf(pKey, "%s.%s", pCol->tableAlias, pCol->colName); } - if (NULL != pStmtName) { + if (NULL != pStmtName && '\0' != pStmtName[0]) { return sprintf(pKey, "%s.%s", pStmtName, ((SExprNode*)pNode)->aliasName); } return sprintf(pKey, "%s", ((SExprNode*)pNode)->aliasName); diff --git a/source/libs/planner/src/planScaleOut.c b/source/libs/planner/src/planScaleOut.c index 2b5fd12e22..7bb97b59d7 100644 --- a/source/libs/planner/src/planScaleOut.c +++ b/source/libs/planner/src/planScaleOut.c @@ -129,7 +129,7 @@ static int32_t pushHierarchicalPlan(SNodeList* pParentsGroup, SNodeList* pCurren return code; } -static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t* pLevel, SNodeList* pParentsGroup) { +static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32_t level, SNodeList* pParentsGroup) { SNodeList* pCurrentGroup = nodesMakeList(); if (NULL == pCurrentGroup) { return TSDB_CODE_OUT_OF_MEMORY; @@ -138,13 +138,13 @@ static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32 int32_t code = TSDB_CODE_SUCCESS; switch (pSubplan->subplanType) { case SUBPLAN_TYPE_MERGE: - code = scaleOutForMerge(pCxt, pSubplan, *pLevel, pCurrentGroup); + code = scaleOutForMerge(pCxt, pSubplan, level, pCurrentGroup); break; case SUBPLAN_TYPE_SCAN: - code = scaleOutForScan(pCxt, pSubplan, *pLevel, pCurrentGroup); + code = scaleOutForScan(pCxt, pSubplan, level, pCurrentGroup); break; case SUBPLAN_TYPE_MODIFY: - code = scaleOutForModify(pCxt, pSubplan, *pLevel, pCurrentGroup); + code = scaleOutForModify(pCxt, pSubplan, level, pCurrentGroup); break; default: break; @@ -152,13 +152,12 @@ static int32_t doScaleOut(SScaleOutContext* pCxt, SLogicSubplan* pSubplan, int32 if (TSDB_CODE_SUCCESS == code) { code = pushHierarchicalPlan(pParentsGroup, pCurrentGroup); - ++(*pLevel); } if (TSDB_CODE_SUCCESS == code) { SNode* pChild; FOREACH(pChild, pSubplan->pChildren) { - code = doScaleOut(pCxt, (SLogicSubplan*)pChild, pLevel, pCurrentGroup); + code = doScaleOut(pCxt, (SLogicSubplan*)pChild, level + 1, pCurrentGroup); if (TSDB_CODE_SUCCESS != code) { break; } @@ -194,7 +193,7 @@ int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQue } SScaleOutContext cxt = { .pPlanCxt = pCxt, .subplanId = 1 }; - int32_t code = doScaleOut(&cxt, pLogicSubplan, &(pPlan->totalLevel), pPlan->pTopSubplans); + int32_t code = doScaleOut(&cxt, pLogicSubplan, 0, pPlan->pTopSubplans); if (TSDB_CODE_SUCCESS == code) { *pLogicPlan = pPlan; } else { diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index e54cf33934..b419414ca6 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -45,7 +45,17 @@ typedef struct SCtjInfo { SLogicSubplan* pSubplan; } SCtjInfo; -typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, SStsInfo* pInfo); +typedef struct SUaInfo { + SProjectLogicNode* pProject; + SLogicSubplan* pSubplan; +} SUaInfo; + +typedef struct SUnInfo { + SAggLogicNode* pAgg; + SLogicSubplan* pSubplan; +} SUnInfo; + +typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, void* pInfo); static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SScanLogicNode* pScan, int32_t flag) { SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); @@ -132,16 +142,10 @@ static bool stsFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SStsInfo info = {0}; - if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STS, stsFindSplitNode, &info)) { + if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STS, (FSplFindSplitNode)stsFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } - if (NULL == info.pSubplan->pChildren) { - info.pSubplan->pChildren = nodesMakeList(); - if (NULL == info.pSubplan->pChildren) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_STS)); + int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_STS)); if (TSDB_CODE_SUCCESS == code) { code = splCreateExchangeNode(pCxt, info.pSubplan, info.pScan, SUBPLAN_TYPE_MERGE); } @@ -173,7 +177,7 @@ static SLogicNode* ctjMatchByNode(SLogicNode* pNode) { return NULL; } -static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { +static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SCtjInfo* pInfo) { SLogicNode* pSplitNode = ctjMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pScan = (SScanLogicNode*)pSplitNode; @@ -184,16 +188,10 @@ static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SCtjInfo info = {0}; - if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_CTJ, ctjFindSplitNode, &info)) { + if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_CTJ, (FSplFindSplitNode)ctjFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } - if (NULL == info.pSubplan->pChildren) { - info.pSubplan->pChildren = nodesMakeList(); - if (NULL == info.pSubplan->pChildren) { - return TSDB_CODE_OUT_OF_MEMORY; - } - } - int32_t code = nodesListStrictAppend(info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_CTJ)); + int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pScan, SPLIT_FLAG_CTJ)); if (TSDB_CODE_SUCCESS == code) { code = splCreateExchangeNode(pCxt, info.pSubplan, info.pScan, info.pSubplan->subplanType); } @@ -202,9 +200,171 @@ static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { return code; } +static SLogicNode* uaMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static bool uaFindSplitNode(SLogicSubplan* pSubplan, SUaInfo* pInfo) { + SLogicNode* pSplitNode = uaMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + pInfo->pProject = (SProjectLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + } + return NULL != pSplitNode; +} + +static SLogicSubplan* uaCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode) { + SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); + if (NULL == pSubplan) { + return NULL; + } + pSubplan->id.groupId = pCxt->groupId; + pSubplan->subplanType = SUBPLAN_TYPE_SCAN; + pSubplan->pNode = pNode; + return pSubplan; +} + +static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { + SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pCxt->groupId; + // pExchange->precision = pScan->pMeta->tableInfo.precision; + pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); + if (NULL == pExchange->node.pTargets) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + + if (NULL == pProject->node.pParent) { + pSubplan->pNode = (SLogicNode*)pExchange; + nodesDestroyNode(pProject); + return TSDB_CODE_SUCCESS; + } + + SNode* pNode; + FOREACH(pNode, pProject->node.pParent->pChildren) { + if (nodesEqualNode(pNode, pProject)) { + REPLACE_NODE(pExchange); + nodesDestroyNode(pNode); + return TSDB_CODE_SUCCESS; + } + } + nodesDestroyNode(pExchange); + return TSDB_CODE_FAILED; +} + +static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SUaInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)uaFindSplitNode, &info)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + + SNode* pChild = NULL; + FOREACH(pChild, info.pProject->node.pChildren) { + code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild)); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(NULL); + } else { + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + nodesClearList(info.pProject->node.pChildren); + info.pProject->node.pChildren = NULL; + code = uaCreateExchangeNode(pCxt, info.pSubplan, info.pProject); + } + ++(pCxt->groupId); + pCxt->split = true; + return code; +} + +static SLogicNode* unMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { + return pNode; + } + SNode* pChild; + FOREACH(pChild, pNode->pChildren) { + SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild); + if (NULL != pSplitNode) { + return pSplitNode; + } + } + return NULL; +} + +static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SAggLogicNode* pAgg) { + SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pCxt->groupId; + // pExchange->precision = pScan->pMeta->tableInfo.precision; + pExchange->node.pTargets = nodesCloneList(pAgg->node.pTargets); + if (NULL == pExchange->node.pTargets) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + + return nodesListMakeAppend(&pAgg->node.pChildren, pExchange); +} + +static bool unFindSplitNode(SLogicSubplan* pSubplan, SUnInfo* pInfo) { + SLogicNode* pSplitNode = unMatchByNode(pSubplan->pNode); + if (NULL != pSplitNode) { + pInfo->pAgg = (SAggLogicNode*)pSplitNode; + pInfo->pSubplan = pSubplan; + } + return NULL != pSplitNode; +} + +static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SUnInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unFindSplitNode, &info)) { + return TSDB_CODE_SUCCESS; + } + + int32_t code = TSDB_CODE_SUCCESS; + + SNode* pChild = NULL; + FOREACH(pChild, info.pAgg->node.pChildren) { + code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, uaCreateSubplan(pCxt, (SLogicNode*)pChild)); + if (TSDB_CODE_SUCCESS == code) { + REPLACE_NODE(NULL); + } else { + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + nodesClearList(info.pAgg->node.pChildren); + info.pAgg->node.pChildren = NULL; + code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg); + } + ++(pCxt->groupId); + pCxt->split = true; + return code; +} + static const SSplitRule splitRuleSet[] = { { .pName = "SuperTableScan", .splitFunc = stsSplit }, { .pName = "ChildTableJoin", .splitFunc = ctjSplit }, + { .pName = "UnionAll", .splitFunc = uaSplit }, + { .pName = "Union", .splitFunc = unSplit } }; static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); diff --git a/source/libs/planner/test/CMakeLists.txt b/source/libs/planner/test/CMakeLists.txt index e068cd8698..f0c59b969f 100644 --- a/source/libs/planner/test/CMakeLists.txt +++ b/source/libs/planner/test/CMakeLists.txt @@ -22,3 +22,12 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/libs/planner/inc" PRIVATE "${TD_SOURCE_DIR}/source/libs/parser/test" ) + + +if(${BUILD_WINGETOPT}) + target_include_directories( + plannerTest + PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" + ) + target_link_libraries(plannerTest PUBLIC wingetopt) +endif() \ No newline at end of file diff --git a/source/libs/planner/test/planSetOpTest.cpp b/source/libs/planner/test/planSetOpTest.cpp new file mode 100644 index 0000000000..5ace503959 --- /dev/null +++ b/source/libs/planner/test/planSetOpTest.cpp @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanSetOpTest : public PlannerTestBase { + +}; + +TEST_F(PlanSetOpTest, unionAll) { + useDb("root", "test"); + + run("select c1, c2 from t1 where c1 > 10 union all select c1, c2 from t1 where c1 > 20"); +} + +TEST_F(PlanSetOpTest, union) { + useDb("root", "test"); + + run("select c1, c2 from t1 where c1 > 10 union select c1, c2 from t1 where c1 > 20"); +} diff --git a/source/libs/planner/test/plannerTestMain.cpp b/source/libs/planner/test/planTestMain.cpp similarity index 98% rename from source/libs/planner/test/plannerTestMain.cpp rename to source/libs/planner/test/planTestMain.cpp index 2878dc7954..84cf75c5be 100644 --- a/source/libs/planner/test/plannerTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -16,7 +16,7 @@ #include #include - +#include "getopt.h" #include "mockCatalog.h" #include "planTestUtil.h" diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 25457d3e41..f124970875 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "planTestUtil.h" #include @@ -62,7 +63,11 @@ public: doScaleOutLogicPlan(&cxt, pLogicSubplan, &pLogicPlan); SQueryPlan* pPlan = nullptr; - doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan, NULL); + doCreatePhysiPlan(&cxt, pLogicPlan, &pPlan); + + if (g_isDump) { + dump(); + } } catch (...) { dump(); throw; @@ -87,6 +92,7 @@ private: string splitLogicPlan_; string scaledLogicPlan_; string physiPlan_; + vector physiSubplans_; }; void reset() { @@ -115,6 +121,10 @@ private: cout << res_.scaledLogicPlan_ << endl; cout << "physical plan : " << endl; cout << res_.physiPlan_ << endl; + cout << "physical subplan : " << endl; + for (const auto& subplan : res_.physiSubplans_) { + cout << subplan << endl; + } } void doParseSql(const string& sql, SQuery** pQuery) { @@ -153,9 +163,17 @@ private: res_.scaledLogicPlan_ = toString((SNode*)(*pLogicPlan)); } - void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList) { + void doCreatePhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan) { + SArray* pExecNodeList = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SQueryNodeAddr)); DO_WITH_THROW(createPhysiPlan, pCxt, pLogicPlan, pPlan, pExecNodeList); res_.physiPlan_ = toString((SNode*)(*pPlan)); + SNode* pNode; + FOREACH(pNode, (*pPlan)->pSubplans) { + SNode* pSubplan; + FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { + res_.physiSubplans_.push_back(toString(pSubplan)); + } + } } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 932107dfc1..9e37784cab 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -157,6 +157,28 @@ int32_t queryBuildGetIndexMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } +int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SRetrieveFuncReq funcReq = {0}; + funcReq.numOfFuncs = 1; + funcReq.ignoreCodeComment = true; + funcReq.pFuncNames = taosArrayInit(1, strlen(input) + 1); + taosArrayPush(funcReq.pFuncNames, input); + + int32_t bufLen = tSerializeSRetrieveFuncReq(NULL, 0, &funcReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSRetrieveFuncReq(pBuf, bufLen, &funcReq); + + taosArrayDestroy(funcReq.pFuncNames); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; @@ -379,6 +401,31 @@ int32_t queryProcessGetIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { + SRetrieveFuncRsp out = {0}; + + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSRetrieveFuncRsp(msg, msgSize, &out) != 0) { + qError("tDeserializeSRetrieveFuncRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + if (1 != out.numOfFuncs) { + qError("invalid func num returned, numOfFuncs:%d", out.numOfFuncs); + return TSDB_CODE_INVALID_MSG; + } + + SFuncInfo * funcInfo = taosArrayGet(out.pFuncInfos, 0); + + memcpy(output, funcInfo, sizeof(*funcInfo)); + taosArrayDestroy(out.pFuncInfos); + + return TSDB_CODE_SUCCESS; +} + void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -386,6 +433,7 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryBuildQnodeListMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -393,6 +441,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryProcessQnodeListRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index d62f9f04b8..cc82349fee 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -95,6 +95,9 @@ typedef struct SQWPhaseInput { } SQWPhaseInput; typedef struct SQWPhaseOutput { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif } SQWPhaseOutput; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 7179fa6984..5a57a47df8 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -277,8 +280,8 @@ int32_t qwtCreateExecTask(void* tsdb, int32_t vgId, uint64_t taskId, struct SSub qwtTestSinkMaxBlockNum = taosRand() % 100 + 1; qwtTestSinkQueryEnd = false; - *pTaskInfo = (qTaskInfo_t)qwtTestCaseIdx+1; - *handle = (DataSinkHandle)qwtTestCaseIdx+2; + *pTaskInfo = (qTaskInfo_t)((char*)qwtTestCaseIdx+1); + *handle = (DataSinkHandle)((char*)qwtTestCaseIdx+2); ++qwtTestCaseIdx; @@ -435,9 +438,16 @@ void stubSetStringToPlan() { static Stub stub; stub.set(qStringToSubplan, qwtStringToPlan); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qStringToSubplan", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qStringToSubplan$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtStringToPlan); } @@ -448,9 +458,16 @@ void stubSetExecTask() { static Stub stub; stub.set(qExecTask, qwtExecTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qExecTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qExecTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtExecTask); } @@ -463,9 +480,16 @@ void stubSetCreateExecTask() { static Stub stub; stub.set(qCreateExecTask, qwtCreateExecTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qCreateExecTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qCreateExecTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtCreateExecTask); } @@ -476,9 +500,16 @@ void stubSetAsyncKillTask() { static Stub stub; stub.set(qAsyncKillTask, qwtKillTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qAsyncKillTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qAsyncKillTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtKillTask); } @@ -489,9 +520,16 @@ void stubSetDestroyTask() { static Stub stub; stub.set(qDestroyTask, qwtDestroyTask); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qDestroyTask", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^qDestroyTask$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtDestroyTask); } @@ -503,9 +541,16 @@ void stubSetDestroyDataSinker() { static Stub stub; stub.set(dsDestroyDataSinker, qwtDestroyDataSinker); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsDestroyDataSinker", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsDestroyDataSinker$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtDestroyDataSinker); } @@ -516,9 +561,16 @@ void stubSetGetDataLength() { static Stub stub; stub.set(dsGetDataLength, qwtGetDataLength); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsGetDataLength", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsGetDataLength$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtGetDataLength); } @@ -529,9 +581,16 @@ void stubSetEndPut() { static Stub stub; stub.set(dsEndPut, qwtEndPut); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsEndPut", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsEndPut$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtEndPut); } @@ -542,9 +601,16 @@ void stubSetPutDataBlock() { static Stub stub; stub.set(dsPutDataBlock, qwtPutDataBlock); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsPutDataBlock", result); +#endif +#ifdef LINUX AddrAny any("libexecutor.so"); std::map result; any.get_global_func_addr_dynsym("^dsPutDataBlock$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtPutDataBlock); } @@ -555,9 +621,16 @@ void stubSetRpcSendResponse() { static Stub stub; stub.set(rpcSendResponse, qwtRpcSendResponse); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendResponse", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendResponse$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtRpcSendResponse); } @@ -568,9 +641,16 @@ void stubSetGetDataBlock() { static Stub stub; stub.set(dsGetDataBlock, qwtGetDataBlock); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("dsGetDataBlock", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^dsGetDataBlock$", result); +#endif for (const auto& f : result) { stub.set(f.second, qwtGetDataBlock); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 0559792740..53e108bb17 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -621,13 +621,13 @@ int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu if (subPos > 0) { startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? subPos - 1 : (subPos - 1) * TSDB_NCHAR_SIZE; - startPosBytes = MIN(startPosBytes, len); + startPosBytes = TMIN(startPosBytes, len); } else { startPosBytes = (GET_PARAM_TYPE(pInput) == TSDB_DATA_TYPE_VARCHAR) ? len + subPos : len + subPos * TSDB_NCHAR_SIZE; - startPosBytes = MAX(startPosBytes, 0); + startPosBytes = TMAX(startPosBytes, 0); } - int32_t resLen = MIN(subLen, len - startPosBytes); + int32_t resLen = TMIN(subLen, len - startPosBytes); if (resLen > 0) { memcpy(varDataVal(output), varDataVal(input) + startPosBytes, resLen); } @@ -716,7 +716,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t len = sprintf(varDataVal(output), "%.*s", (int32_t)(outputLen - VARSTR_HEADER_SIZE), *(int8_t *)input ? "true" : "false"); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { - int32_t len = MIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); + int32_t len = TMIN(varDataLen(input), outputLen - VARSTR_HEADER_SIZE); len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { @@ -750,7 +750,7 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = MIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); + int32_t len = TMIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); memcpy(output, input, len); varDataSetLen(output, len - VARSTR_HEADER_SIZE); } else { diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 42998aba00..c5191947e0 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -364,7 +367,7 @@ TEST(columnTest, smallint_column_greater_double_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, true); - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -419,7 +422,7 @@ TEST(columnTest, int_column_greater_smallint_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, false); - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -463,7 +466,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -527,7 +530,7 @@ TEST(columnTest, binary_column_in_binary_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -575,7 +578,7 @@ TEST(columnTest, binary_column_like_binary) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -622,7 +625,7 @@ TEST(columnTest, binary_column_is_null) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -669,7 +672,7 @@ TEST(columnTest, binary_column_is_not_null) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -706,7 +709,7 @@ TEST(opTest, smallint_column_greater_int_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -744,7 +747,7 @@ TEST(opTest, smallint_value_add_int_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -787,7 +790,7 @@ TEST(opTest, bigint_column_multi_binary_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -830,7 +833,7 @@ TEST(opTest, smallint_column_and_binary_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -868,7 +871,7 @@ TEST(opTest, smallint_column_or_float_column) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -906,7 +909,7 @@ TEST(opTest, smallint_column_or_double_value) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -950,7 +953,7 @@ TEST(opTest, binary_column_is_true) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1021,7 +1024,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1090,7 +1093,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1159,7 +1162,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1228,7 +1231,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1299,7 +1302,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = {.numOfCols= src->info.numOfCols, .pDataBlock = src->pDataBlock}; + SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index de76e7e274..f6140a75d3 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -27,6 +27,9 @@ #pragma GCC diagnostic ignored "-Wpointer-arith" #include +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index d1def1bef1..0f6961018c 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -52,14 +52,26 @@ typedef struct SSchHbTrans { typedef struct SSchApiStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchApiStat; typedef struct SSchRuntimeStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchRuntimeStat; typedef struct SSchJobStat { +#ifdef WINDOWS + size_t avoidCompilationErrors; +#endif + } SSchJobStat; typedef struct SSchedulerStat { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 5d9322a575..fc0e05aaf1 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -25,7 +25,9 @@ #pragma GCC diagnostic ignored "-Wformat" #include - +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif #include "os.h" #include "tglobal.h" @@ -275,9 +277,16 @@ void schtSetPlanToString() { static Stub stub; stub.set(qSubPlanToString, schtPlanToString); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qSubPlanToString", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qSubPlanToString$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtPlanToString); } @@ -288,9 +297,16 @@ void schtSetExecNode() { static Stub stub; stub.set(qSetSubplanExecutionNode, schtExecNode); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("qSetSubplanExecutionNode", result); +#endif +#ifdef LINUX AddrAny any("libplanner.so"); std::map result; any.get_global_func_addr_dynsym("^qSetSubplanExecutionNode$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtExecNode); } @@ -301,9 +317,16 @@ void schtSetRpcSendRequest() { static Stub stub; stub.set(rpcSendRequest, schtRpcSendRequest); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("rpcSendRequest", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^rpcSendRequest$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtRpcSendRequest); } @@ -324,9 +347,16 @@ void schtSetAsyncSendMsgToServer() { static Stub stub; stub.set(asyncSendMsgToServer, schtAsyncSendMsgToServer); { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("asyncSendMsgToServer", result); +#endif +#ifdef LINUX AddrAny any("libtransport.so"); std::map result; any.get_global_func_addr_dynsym("^asyncSendMsgToServer$", result); +#endif for (const auto& f : result) { stub.set(f.second, schtAsyncSendMsgToServer); } @@ -382,6 +412,7 @@ void *schtCreateFetchRspThread(void *param) { schReleaseJob(job); assert(code == 0); + return NULL; } @@ -413,6 +444,7 @@ void *schtFetchRspThread(void *aa) { assert(code == 0 || code); } + return NULL; } void schtFreeQueryJob(int32_t freeThread) { @@ -595,6 +627,7 @@ void* schtRunJobThread(void *aa) { schedulerDestroy(); + return NULL; } void* schtFreeJobThread(void *aa) { @@ -602,6 +635,7 @@ void* schtFreeJobThread(void *aa) { taosUsleep(taosRand() % 100); schtFreeQueryJob(1); } + return NULL; } diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index 726ebdae07..86fec5f484 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -14,7 +14,7 @@ */ #include "syncEnv.h" -#include +// #include SSyncEnv *gSyncEnv = NULL; diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 3a62b19c10..01490030f2 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ # tdb -add_library(tdb "") +add_library(tdb STATIC "") target_sources(tdb PRIVATE "src/db/tdbPCache.c" diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 8a95635ddc..0dfebe9820 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -42,14 +42,17 @@ struct SBTree { ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) -typedef struct __attribute__((__packed__)) { + +#pragma pack(push,1) +typedef struct { TDB_BTREE_PAGE_COMMON_HDR } SLeafHdr; -typedef struct __attribute__((__packed__)) { - TDB_BTREE_PAGE_COMMON_HDR; +typedef struct { + TDB_BTREE_PAGE_COMMON_HDR SPgno pgno; // right-most child } SIntHdr; +#pragma pack(pop) typedef struct { u8 flags; diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 86272b3cad..ce753ad930 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -28,11 +28,10 @@ struct SPCache { SPage lru; }; -#define PCACHE_PAGE_HASH(pPgid) \ - ({ \ - u32 *t = (u32 *)((pPgid)->fileid); \ - t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; \ - }) +static inline int tdbPCachePageHash(const SPgid *pPgid) { + u32 *t = (u32 *)((pPgid)->fileid); + return t[0] + t[1] + t[2] + t[3] + t[4] + t[5] + (pPgid)->pgno; +} #define PAGE_IS_PINNED(pPage) ((pPage)->pLruNext == NULL) static int tdbPCacheOpenImpl(SPCache *pCache); @@ -130,7 +129,7 @@ static SPage *tdbPCacheFetchImpl(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) SPage *pPage; // 1. Search the hash table - pPage = pCache->pgHash[PCACHE_PAGE_HASH(pPgid) % pCache->nHash]; + pPage = pCache->pgHash[tdbPCachePageHash(pPgid) % pCache->nHash]; while (pPage) { if (TDB_IS_SAME_PAGE(&(pPage->pgid), pPgid)) break; pPage = pPage->pHashNext; @@ -218,7 +217,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { SPage **ppPage; int h; - h = PCACHE_PAGE_HASH(&(pPage->pgid)); + h = tdbPCachePageHash(&(pPage->pgid)); for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext)) ; ASSERT(*ppPage == pPage); @@ -230,7 +229,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { int h; - h = PCACHE_PAGE_HASH(&(pPage->pgid)) % pCache->nHash; + h = tdbPCachePageHash(&(pPage->pgid)) % pCache->nHash; pPage->pHashNext = pCache->pgHash[h]; pCache->pgHash[h] = pPage; diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index ea75f7fbeb..05939579d8 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -435,17 +435,20 @@ static int tdbPageDefragment(SPage *pPage) { } /* ---------------------------------------------------------------------------------------------------------- */ -typedef struct __attribute__((__packed__)) { + +#pragma pack(push,1) +typedef struct { u16 cellNum; u16 cellBody; u16 cellFree; u16 nFree; } SPageHdr; -typedef struct __attribute__((__packed__)) { +typedef struct { u16 szCell; u16 nxOffset; } SFreeCell; +#pragma pack(pop) // cellNum static inline int getPageCellNum(SPage *pPage) { return ((SPageHdr *)(pPage->pPageHdr))[0].cellNum; } @@ -517,17 +520,19 @@ SPageMethods pageMethods = { setPageFreeCellInfo // setFreeCellInfo }; -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { u8 cellNum[3]; u8 cellBody[3]; u8 cellFree[3]; u8 nFree[3]; } SPageHdrL; -typedef struct __attribute__((__packed__)) { +typedef struct { u8 szCell[3]; u8 nxOffset[3]; } SFreeCellL; +#pragma pack(pop) // cellNum static inline int getLPageCellNum(SPage *pPage) { return TDB_GET_U24(((SPageHdrL *)(pPage->pPageHdr))[0].cellNum); } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 7672ccf34e..8835b7f759 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -15,13 +15,15 @@ #include "tdbInt.h" -typedef struct __attribute__((__packed__)) { +#pragma pack(push,1) +typedef struct { u8 hdrString[16]; u16 pageSize; SPgno freePage; u32 nFreePages; u8 reserved[102]; } SFileHdr; +#pragma pack(pop) TDB_STATIC_ASSERT(sizeof(SFileHdr) == 128, "Size of file header is not correct"); diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h index 77a1ee5f6e..3707a922ee 100644 --- a/source/libs/tdb/src/inc/tdbPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -47,10 +47,13 @@ typedef struct { void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset); } SPageMethods; +#pragma pack(push,1) + // Page footer -typedef struct __attribute__((__packed__)) { +typedef struct { u8 cksm[4]; } SPageFtr; +#pragma pack(pop) struct SPage { tdb_spinlock_t lock; @@ -80,22 +83,23 @@ struct SPage { #define P_LOCK_BUSY 1 #define P_LOCK_FAIL -1 +static inline int tdbTryLockPage(tdb_spinlock_t *pLock) { + int ret; + if (tdbSpinlockTrylock(pLock) == 0) { + ret = P_LOCK_SUCC; + } else if (errno == EBUSY) { + ret = P_LOCK_BUSY; + } else { + ret = P_LOCK_FAIL; + } + return ret; +} + #define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0) #define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock)) #define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock)) #define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock)) -#define TDB_TRY_LOCK_PAGE(pPage) \ - ({ \ - int ret; \ - if (tdbSpinlockTrylock(&((pPage)->lock)) == 0) { \ - ret = P_LOCK_SUCC; \ - } else if (errno == EBUSY) { \ - ret = P_LOCK_BUSY; \ - } else { \ - ret = P_LOCK_FAIL; \ - } \ - ret; \ - }) +#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock)) // APIs #define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage)) diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index 7085e71567..cffd3f2746 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -31,20 +31,20 @@ extern "C" { int tdbGnrtFileID(const char *fname, uint8_t *fileid, bool unique); int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size); -#define TDB_REALLOC(PTR, SIZE) \ - ({ \ - void *nPtr; \ - if ((PTR) == NULL || ((int *)(PTR))[-1] < (SIZE)) { \ - nPtr = tdbOsRealloc((PTR) ? (char *)(PTR) - sizeof(int) : NULL, (SIZE) + sizeof(int)); \ - if (nPtr) { \ - ((int *)nPtr)[0] = (SIZE); \ - nPtr = (char *)nPtr + sizeof(int); \ - } \ - } else { \ - nPtr = (PTR); \ - } \ - nPtr; \ - }) +static inline void *tdbRealloc(void *ptr, size_t size) { + void *nPtr; + if ((ptr) == NULL || ((int *)(ptr))[-1] < (size)) { + nPtr = tdbOsRealloc((ptr) ? (char *)(ptr) - sizeof(int) : NULL, (size) + sizeof(int)); + if (nPtr) { + ((int *)nPtr)[0] = (size); + nPtr = (char *)nPtr + sizeof(int); + } + } else { + nPtr = (ptr); + } + return nPtr; +} +#define TDB_REALLOC(PTR, SIZE) tdbRealloc(PTR, SIZE) #define TDB_FREE(PTR) \ do { \ diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index e50045d30b..18a3a28bab 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -366,7 +366,7 @@ const STfsFile *tfsReaddir(STfsDir *pTfsDir) { void tfsClosedir(STfsDir *pTfsDir) { if (pTfsDir) { if (pTfsDir->pDir != NULL) { - taosCloseDir(pTfsDir->pDir); + taosCloseDir(&pTfsDir->pDir); pTfsDir->pDir = NULL; } taosMemoryFree(pTfsDir); @@ -455,7 +455,7 @@ static int32_t tfsFormatDir(char *idir, char *odir) { } char tmp[PATH_MAX] = {0}; - if (realpath(wep.we_wordv[0], tmp) == NULL) { + if (taosRealPath(wep.we_wordv[0], tmp, PATH_MAX) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); wordfree(&wep); return -1; @@ -499,7 +499,7 @@ static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) { char adir[TMPNAME_LEN * 2] = "\0"; if (pTfsDir->pDir != NULL) { - taosCloseDir(pTfsDir->pDir); + taosCloseDir(&pTfsDir->pDir); pTfsDir->pDir = NULL; } diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index ad50948a02..eaca9b0fc7 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -63,13 +63,14 @@ typedef struct { void (*cfp)(void* parent, SRpcMsg*, SEpSet*); int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + int (*retry)(void* parent, SRpcMsg*, SEpSet*); - int32_t refCount; - void* parent; - void* idPool; // handle to ID pool - void* tmrCtrl; // handle to timer - SHashObj* hash; // handle returned by hash utility - void* tcphandle; // returned handle from TCP initialization + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization TdThreadMutex mutex; } SRpcInfo; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index ebb90338cd..fa517d6d61 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -39,6 +39,7 @@ void* rpcOpen(const SRpcInit* pInit) { // register callback handle pRpc->cfp = pInit->cfp; pRpc->afp = pInit->afp; + pRpc->retry = pInit->rfp; if (pInit->connType == TAOS_CONN_SERVER) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; @@ -100,11 +101,10 @@ void rpcSendRedirectRsp(void* thandle, const SEpSet* pEpSet) { SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - rpcMsg.contLen = sizeof(SEpSet); - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - if (rpcMsg.pCont == NULL) return; - - memcpy(rpcMsg.pCont, pEpSet, sizeof(SEpSet)); + SMEpSet msg = {.epSet = *pEpSet}; + int32_t len = tSerializeSMEpSet(NULL, 0, &msg); + rpcMsg.pCont = rpcMallocCont(len); + tSerializeSMEpSet(rpcMsg.pCont, len, &msg); rpcMsg.code = TSDB_CODE_RPC_REDIRECT; rpcMsg.handle = thandle; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b81310b90b..b43b8a1e0c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -31,12 +31,8 @@ typedef struct SCliConn { int hThrdIdx; STransCtx ctx; - bool broken; // link broken or not - ConnStatus status; // - int release; // 1: release - // spi configure - char spi; - char secured; + bool broken; // link broken or not + ConnStatus status; // char* ip; uint32_t port; @@ -44,7 +40,6 @@ typedef struct SCliConn { // debug and log info struct sockaddr_in addr; struct sockaddr_in locaddr; - } SCliConn; typedef struct SCliMsg { @@ -102,6 +97,8 @@ static void cliSendCb(uv_write_t* req, int status); static void cliConnCb(uv_connect_t* req, int status); static void cliAsyncCb(uv_async_t* handle); +static void cliAppCb(SCliConn* pConn, STransMsg* pMsg); + static SCliConn* cliCreateConn(SCliThrdObj* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); @@ -303,8 +300,6 @@ void cliHandleResp(SCliConn* conn) { TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), transMsg.contLen); - conn->secured = pHead->secured; - if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tTrace("except, server continue send while cli ignore it"); // transUnrefCliHandle(conn); @@ -318,7 +313,8 @@ void cliHandleResp(SCliConn* conn) { if (pCtx == NULL || pCtx->pSem == NULL) { tTrace("%s cli conn %p handle resp", pTransInst->label, conn); - (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + cliAppCb(conn, &transMsg); + //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle resp", pTransInst->label, conn); memcpy((char*)pCtx->pRsp, (char*)&transMsg, sizeof(transMsg)); @@ -384,7 +380,8 @@ void cliHandleExcept(SCliConn* pConn) { once = true; continue; } - (pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); + cliAppCb(pConn, &transMsg); + //(pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); } else { tTrace("%s cli conn(sync) %p handle except", pTransInst->label, pConn); memcpy((char*)(pCtx->pRsp), (char*)(&transMsg), sizeof(transMsg)); @@ -884,6 +881,18 @@ int cliRBChoseIdx(STrans* pTransInst) { } return index % pTransInst->numOfThreads; } +void cliAppCb(SCliConn* pConn, STransMsg* transMsg) { + SCliThrdObj* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + if (transMsg->code == TSDB_CODE_RPC_REDIRECT && pTransInst->retry != NULL) { + SMEpSet emsg = {0}; + tDeserializeSMEpSet(transMsg->pCont, transMsg->contLen, &emsg); + pTransInst->retry(pTransInst, transMsg, &(emsg.epSet)); + } else { + pTransInst->cfp(pTransInst->parent, transMsg, NULL); + } +} void transCloseClient(void* arg) { SCliObj* cli = arg; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index ec66f3e8df..59a30051ef 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -30,7 +30,6 @@ typedef struct SSrvConn { uv_timer_t pTimer; queue queue; - int ref; int persist; // persist connection or not SConnBuffer readBuf; // read buf, int inType; @@ -692,8 +691,6 @@ static void uvDestroyConn(uv_handle_t* handle) { if (thrd->quit && QUEUE_IS_EMPTY(&thrd->conn)) { tTrace("work thread quit"); uv_walk(thrd->loop, uvWalkCb, NULL); - // uv_loop_close(thrd->loop); - // uv_stop(thrd->loop); } } @@ -756,8 +753,6 @@ void uvHandleQuit(SSrvMsg* msg, SWorkThrdObj* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { uv_walk(thrd->loop, uvWalkCb, NULL); - // uv_loop_close(thrd->loop); - // uv_stop(thrd->loop); } else { destroyAllConn(thrd); } @@ -851,10 +846,8 @@ void transRefSrvHandle(void* handle) { if (handle == NULL) { return; } - SSrvConn* conn = handle; - int ref = T_REF_INC((SSrvConn*)handle); - UNUSED(ref); + tDebug("server conn %p ref count: %d", handle, ref); } void transUnrefSrvHandle(void* handle) { diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index cdf0908167..2b1c4fa623 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -12,8 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include - #include #include "os.h" #include "rpcLog.h" diff --git a/source/libs/transport/test/syncClient.c b/source/libs/transport/test/syncClient.c index 27a468a86f..3f1a7805b4 100644 --- a/source/libs/transport/test/syncClient.c +++ b/source/libs/transport/test/syncClient.c @@ -12,8 +12,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include - #include #include "os.h" #include "rpcLog.h" diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp index 2488dda3f6..35009c7686 100644 --- a/source/libs/transport/test/transportTests.cpp +++ b/source/libs/transport/test/transportTests.cpp @@ -156,14 +156,14 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; @@ -176,14 +176,14 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryMalloc(12); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); key++; @@ -198,7 +198,7 @@ TEST_F(TransCtxEnv, mergeTest) { STransCtx *src = (STransCtx *)taosMemoryCalloc(1, sizeof(STransCtx)); transCtxInit(src); { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); @@ -206,7 +206,7 @@ TEST_F(TransCtxEnv, mergeTest) { key++; } { - STransCtxVal val1 = {.val = NULL, .clone = NULL, .freeFunc = (void (*)(const void*))taosMemoryFree}; + STransCtxVal val1 = { NULL, NULL, (void (*)(const void*))taosMemoryFree}; val1.val = taosMemoryCalloc(1, 11); memcpy(val1.val, val.c_str(), val.size()); taosHashPut(src->args, &key, sizeof(key), &val1, sizeof(val1)); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 9f8fe8d84d..3e6663f464 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -145,7 +145,7 @@ int walCheckAndRepairMeta(SWal* pWal) { } } - taosCloseDir(pDir); + taosCloseDir(&pDir); regfree(&logRegPattern); regfree(&idxRegPattern); @@ -351,7 +351,7 @@ static int walFindCurMetaVer(SWal* pWal) { break; } } - taosCloseDir(pDir); + taosCloseDir(&pDir); regfree(&walMetaRegexPattern); return metaVer; } diff --git a/source/os/CMakeLists.txt b/source/os/CMakeLists.txt index 261ec312ad..d709bbbcc2 100644 --- a/source/os/CMakeLists.txt +++ b/source/os/CMakeLists.txt @@ -6,7 +6,8 @@ target_include_directories( PUBLIC "${TD_SOURCE_DIR}/include" PUBLIC "${TD_SOURCE_DIR}/include/util" PUBLIC "${TD_SOURCE_DIR}/contrib/pthread" - PUBLIC "${TD_SOURCE_DIR}/contrib/gnuregex" + PUBLIC "${TD_SOURCE_DIR}/contrib/iconv" + PUBLIC "${TD_SOURCE_DIR}/contrib/msvcregex" ) # iconv find_path(IconvApiIncludes iconv.h PATHS) @@ -17,5 +18,14 @@ if(USE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY) endif () target_link_libraries( - os pthread dl rt m + os pthread ) +if(NOT TD_WINDOWS) + target_link_libraries( + os dl m rt + ) +else() + target_link_libraries( + os ws2_32 iconv msvcregex wcwidth winmm + ) +endif(NOT TD_WINDOWS) diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 8b452947d3..0fe946bf68 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -16,78 +16,101 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS // add -char interlocked_add_fetch_8(char volatile* ptr, char val) { +int8_t interlocked_add_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedExchangeAdd8(ptr, val) + val; } -short interlocked_add_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_add_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedExchangeAdd16(ptr, val) + val; } -long interlocked_add_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_add_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedExchangeAdd(ptr, val) + val; } -__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { +int64_t interlocked_add_fetch_64(int64_t volatile* ptr, int64_t val) { return InterlockedExchangeAdd64(ptr, val) + val; } -char interlocked_and_fetch_8(char volatile* ptr, char val) { +void* interlocked_add_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)(_InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)val) + (int32_t)val); +#else + return (void*)(InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)val) + (int64_t)val); +#endif +} + +int8_t interlocked_and_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedAnd8(ptr, val) & val; } -short interlocked_and_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_and_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedAnd16(ptr, val) & val; } -long interlocked_and_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_and_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedAnd(ptr, val) & val; } - -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifndef _M_IX86 - return _InterlockedAnd64(ptr, val) & val; -#else - __int64 old, res; +int64_t interlocked_and_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old & val; } while (_InterlockedCompareExchange64(ptr, res, old) != old); return res; +#else + return _InterlockedAnd64(ptr, val) & val; #endif } -__int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_and_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_and_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_and_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} + +int64_t interlocked_fetch_and_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while (_InterlockedCompareExchange64(ptr, old & val, old) != old); return old; #else - return _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedAnd64((int64_t volatile*)(ptr), (int64_t)(val)); #endif } -char interlocked_or_fetch_8(char volatile* ptr, char val) { +void* interlocked_fetch_and_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)_InterlockedAnd64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int8_t interlocked_or_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedOr8(ptr, val) | val; } -short interlocked_or_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_or_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedOr16(ptr, val) | val; } -long interlocked_or_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_or_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedOr(ptr, val) | val; } -__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old, res; +int64_t interlocked_or_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old | val; @@ -98,33 +121,49 @@ __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { #endif } -__int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_or_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_or_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_or_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} + +int64_t interlocked_fetch_or_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while(_InterlockedCompareExchange64(ptr, old | val, old) != old); return old; #else - return _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedOr64((int64_t volatile*)(ptr), (int64_t)(val)); #endif } -char interlocked_xor_fetch_8(char volatile* ptr, char val) { +void* interlocked_fetch_or_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int8_t interlocked_xor_fetch_8(int8_t volatile* ptr, int8_t val) { return _InterlockedXor8(ptr, val) ^ val; } -short interlocked_xor_fetch_16(short volatile* ptr, short val) { +int16_t interlocked_xor_fetch_16(int16_t volatile* ptr, int16_t val) { return _InterlockedXor16(ptr, val) ^ val; } -long interlocked_xor_fetch_32(long volatile* ptr, long val) { +int32_t interlocked_xor_fetch_32(int32_t volatile* ptr, int32_t val) { return _InterlockedXor(ptr, val) ^ val; } -__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old, res; +int64_t interlocked_xor_fetch_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old, res; do { old = *ptr; res = old ^ val; @@ -135,15 +174,62 @@ __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { #endif } -__int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { -#ifdef _M_IX86 - __int64 old; +void* interlocked_xor_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int64_t interlocked_fetch_xor_64(int64_t volatile* ptr, int64_t val) { +#ifdef _TD_WINDOWS_32 + int64_t old; do { old = *ptr; } while (_InterlockedCompareExchange64(ptr, old ^ val, old) != old); return old; #else - return _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)); + return _InterlockedXor64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +void* interlocked_fetch_xor_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)_InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); +#else + return (void*)interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); +#endif +} + +int32_t interlocked_sub_fetch_32(int32_t volatile* ptr, int32_t val) { + return interlocked_add_fetch_32(ptr, -val); +} + +int64_t interlocked_sub_fetch_64(int64_t volatile* ptr, int64_t val) { + return interlocked_add_fetch_64(ptr, -val); +} + +void* interlocked_sub_fetch_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_sub_fetch_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_add_fetch_64((int64_t volatile*)ptr, (int64_t)val); +#endif +} +int32_t interlocked_fetch_sub_32(int32_t volatile* ptr, int32_t val) { + return _InterlockedExchangeAdd(ptr, -val); +} + +int64_t interlocked_fetch_sub_64(int64_t volatile* ptr, int64_t val) { + return _InterlockedExchangeAdd64(ptr, -val); +} + +void* interlocked_fetch_sub_ptr(void* volatile* ptr, void* val) { +#ifdef _TD_WINDOWS_32 + return (void*)interlocked_fetch_sub_32((int32_t volatile*)ptr, (int32_t)val); +#else + return (void*)interlocked_fetch_sub_64((int64_t volatile*)ptr, (int64_t)val); #endif } @@ -189,7 +275,7 @@ int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { int8_t atomic_load_8(int8_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int8_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -199,7 +285,7 @@ int8_t atomic_load_8(int8_t volatile *ptr) { } int16_t atomic_load_16(int16_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int16_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -209,7 +295,7 @@ int16_t atomic_load_16(int16_t volatile *ptr) { } int32_t atomic_load_32(int32_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int32_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -219,7 +305,7 @@ int32_t atomic_load_32(int32_t volatile *ptr) { } int64_t atomic_load_64(int64_t volatile *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(int64_t volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -229,7 +315,7 @@ int64_t atomic_load_64(int64_t volatile *ptr) { } void* atomic_load_ptr(void *ptr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return (*(void* volatile*)(ptr)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), 0); @@ -239,7 +325,7 @@ void* atomic_load_ptr(void *ptr) { } void atomic_store_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int8_t volatile*)(ptr)) = (int8_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -249,7 +335,7 @@ void atomic_store_8(int8_t volatile *ptr, int8_t val) { } void atomic_store_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int16_t volatile*)(ptr)) = (int16_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -259,7 +345,7 @@ void atomic_store_16(int16_t volatile *ptr, int16_t val) { } void atomic_store_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int32_t volatile*)(ptr)) = (int32_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -269,7 +355,7 @@ void atomic_store_32(int32_t volatile *ptr, int32_t val) { } void atomic_store_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(int64_t volatile*)(ptr)) = (int64_t)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -279,7 +365,7 @@ void atomic_store_64(int64_t volatile *ptr, int64_t val) { } void atomic_store_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS ((*(void* volatile*)(ptr)) = (void*)(val)); #elif defined(_TD_NINGSI_60) (*(ptr)=(val)); @@ -289,7 +375,7 @@ void atomic_store_ptr(void *ptr, void *val) { } int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val); @@ -299,7 +385,7 @@ int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val); @@ -309,7 +395,7 @@ int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val); @@ -319,7 +405,7 @@ int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val); @@ -329,7 +415,7 @@ int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { } void* atomic_exchange_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #ifdef _WIN64 return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #else @@ -343,7 +429,7 @@ void* atomic_exchange_ptr(void *ptr, void *val) { } int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange8((int8_t volatile*)(ptr), (int8_t)(newval), (int8_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -353,7 +439,7 @@ int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t } int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange16((int16_t volatile*)(ptr), (int16_t)(newval), (int16_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -363,7 +449,7 @@ int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, in } int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange((int32_t volatile*)(ptr), (int32_t)(newval), (int32_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -373,7 +459,7 @@ int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, in } int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchange64((int64_t volatile*)(ptr), (int64_t)(newval), (int64_t)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -383,7 +469,7 @@ int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, in } void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)); #elif defined(_TD_NINGSI_60) return __sync_val_compare_and_swap(ptr, oldval, newval); @@ -393,7 +479,7 @@ void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { } int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -403,7 +489,7 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -413,7 +499,7 @@ int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -423,7 +509,7 @@ int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); @@ -432,18 +518,18 @@ int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_add_fetch_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +void* atomic_add_fetch_ptr(void *ptr, void *val) { +#ifdef WINDOWS return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_add_and_fetch((ptr), (val)); #else - return __atomic_add_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); + return __atomic_add_fetch((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST); #endif } int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -453,7 +539,7 @@ int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -463,7 +549,7 @@ int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -473,7 +559,7 @@ int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); @@ -482,18 +568,18 @@ int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_fetch_add_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAddptr((void* volatile*)(ptr), (void*)(val)); +void* atomic_fetch_add_ptr(void *ptr, void *val) { +#ifdef WINDOWS + return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_add((ptr), (val)); #else - return __atomic_fetch_add((void **)(ptr), (val), __ATOMIC_SEQ_CST); + return __atomic_fetch_add((void **)(ptr), (size_t)(val), __ATOMIC_SEQ_CST); #endif } int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_8((int8_t volatile*)(ptr), -(int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); @@ -503,7 +589,7 @@ int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_add_fetch_16((int16_t volatile*)(ptr), -(int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); @@ -513,8 +599,8 @@ int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_32((int32_t volatile*)(ptr), -(int32_t)(val)); +#ifdef WINDOWS + return interlocked_sub_fetch_32(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -523,8 +609,8 @@ int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_64((int64_t volatile*)(ptr), -(int64_t)(val)); +#ifdef WINDOWS + return interlocked_sub_fetch_64(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -532,9 +618,9 @@ int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return interlocked_add_fetch_ptr((void* volatile*)(ptr), -(void*)(val)); +void* atomic_sub_fetch_ptr(void *ptr, void* val) { +#ifdef WINDOWS + return interlocked_sub_fetch_ptr(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_sub_and_fetch((ptr), (val)); #else @@ -543,7 +629,7 @@ void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { } int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), -(int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -553,7 +639,7 @@ int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), -(int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -563,17 +649,17 @@ int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAdd((int32_t volatile*)(ptr), -(int32_t)(val)); +#ifdef WINDOWS + return interlocked_fetch_sub_32(ptr, val); #elif defined(_TD_NINGSI_60) - return __sync_fetch_and_sub((ptr), (val)); + return __sync_fetch_and_sub(ptr, val); #else return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); #endif } int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), -(int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); @@ -582,9 +668,9 @@ int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { #endif } -void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - return _InterlockedExchangeAddptr((void* volatile*)(ptr), -(void*)(val)); +void* atomic_fetch_sub_ptr(void *ptr, void* val) { +#ifdef WINDOWS + return interlocked_fetch_sub_ptr(ptr, val); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_sub((ptr), (val)); #else @@ -593,7 +679,7 @@ void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { } int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -603,7 +689,7 @@ int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -613,7 +699,7 @@ int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -623,7 +709,7 @@ int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -633,7 +719,7 @@ int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_and_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_and_and_fetch((ptr), (val)); @@ -643,7 +729,7 @@ void* atomic_and_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -653,7 +739,7 @@ int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -663,7 +749,7 @@ int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -673,7 +759,7 @@ int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_and_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -683,7 +769,7 @@ int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_and_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_and((ptr), (val)); @@ -693,7 +779,7 @@ void* atomic_fetch_and_ptr(void *ptr, void *val) { } int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -703,7 +789,7 @@ int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -713,7 +799,7 @@ int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -723,7 +809,7 @@ int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -733,7 +819,7 @@ int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_or_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_or_and_fetch((ptr), (val)); @@ -743,7 +829,7 @@ void* atomic_or_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -753,7 +839,7 @@ int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -763,7 +849,7 @@ int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -773,7 +859,7 @@ int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -783,7 +869,7 @@ int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_or_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_or((ptr), (val)); @@ -793,7 +879,7 @@ void* atomic_fetch_or_ptr(void *ptr, void *val) { } int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -803,7 +889,7 @@ int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -813,7 +899,7 @@ int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -823,7 +909,7 @@ int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -833,7 +919,7 @@ int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { } void* atomic_xor_fetch_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_xor_and_fetch((ptr), (val)); @@ -843,7 +929,7 @@ void* atomic_xor_fetch_ptr(void *ptr, void *val) { } int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor8((int8_t volatile*)(ptr), (int8_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -853,7 +939,7 @@ int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { } int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor16((int16_t volatile*)(ptr), (int16_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -863,7 +949,7 @@ int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { } int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return _InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -873,7 +959,7 @@ int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { } int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); @@ -883,7 +969,7 @@ int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { } void* atomic_fetch_xor_ptr(void *ptr, void *val) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val)); #elif defined(_TD_NINGSI_60) return __sync_fetch_and_xor((ptr), (val)); diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index a955fb3b0a..46fa2ecd3a 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -17,19 +17,40 @@ #define ALLOW_FORBID_FUNC #include "os.h" -#include "osString.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) -/* - * windows implementation - */ +#ifdef WINDOWS -// todo +#include + +typedef struct TdDirEntry { + WIN32_FIND_DATA findFileData; +} TdDirEntry; + + +typedef struct TdDir { + TdDirEntry dirEntry; + HANDLE hFind; +} TdDir; + +int wordexp(char *words, wordexp_t *pwordexp, int flags) { + pwordexp->we_offs = 0; + pwordexp->we_wordc = 1; + pwordexp->we_wordv[0] = pwordexp->wordPos; + + memset(pwordexp->wordPos, 0, 1025); + if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) { + pwordexp->we_wordv[0] = words; + printf("failed to parse relative path:%s to abs path", words); + return -1; + } + + printf("parse relative path:%s to abs path:%s", words, pwordexp->wordPos); + return 0; +} + +void wordfree(wordexp_t *pwordexp) {} #else -/* - * linux implementation - */ #include #include @@ -39,19 +60,21 @@ typedef struct dirent dirent; typedef struct DIR TdDir; -typedef struct dirent TdDirent; +typedef struct dirent TdDirEntry; + +#endif void taosRemoveDir(const char *dirname) { - DIR *dir = opendir(dirname); - if (dir == NULL) return; + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir == NULL) return; - struct dirent *de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); - if (de->d_type & DT_DIR) { + snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + if (taosDirEntryIsDir(de)) { taosRemoveDir(filename); } else { (void)taosRemoveFile(filename); @@ -59,13 +82,14 @@ void taosRemoveDir(const char *dirname) { } } - closedir(dir); + taosCloseDir(&pDir); rmdir(dirname); //printf("dir:%s is removed\n", dirname); + return; } -bool taosDirExist(char *dirname) { return taosCheckExistFile(dirname); } +bool taosDirExist(const char *dirname) { return taosCheckExistFile(dirname); } int32_t taosMkDir(const char *dirname) { int32_t code = mkdir(dirname, 0755); @@ -118,18 +142,18 @@ int32_t taosMulMkDir(const char *dirname) { } void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { - DIR *dir = opendir(dirname); - if (dir == NULL) return; + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir == NULL) return; - int64_t sec = taosGetTimestampSec(); - struct dirent *de = NULL; + int64_t sec = taosGetTimestampSec(); + TdDirEntryPtr de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + while ((de = taosReadDir(pDir)) != NULL) { + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; char filename[1024]; - snprintf(filename, sizeof(filename), "%s/%s", dirname, de->d_name); - if (de->d_type & DT_DIR) { + snprintf(filename, sizeof(filename), "%s/%s", dirname, taosGetDirEntryName(de)); + if (taosDirEntryIsDir(de)) { continue; } else { int32_t len = (int32_t)strlen(filename); @@ -156,7 +180,7 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { } } - closedir(dir); + taosCloseDir(&pDir); rmdir(dirname); } @@ -177,65 +201,131 @@ int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) { return 0; } -int32_t taosRealPath(char *dirname, int32_t maxlen) { +int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen) { char tmp[PATH_MAX] = {0}; +#ifdef WINDOWS + if (_fullpath(dirname, tmp, maxlen) != NULL) { +#else if (realpath(dirname, tmp) != NULL) { - strncpy(dirname, tmp, maxlen); +#endif + if (realPath == NULL) { + strncpy(dirname, tmp, maxlen); + } else { + strncpy(realPath, tmp, maxlen); + } + return 0; } - return 0; + return -1; } bool taosIsDir(const char *dirname) { - DIR *dir = opendir(dirname); - if (dir != NULL) { - closedir(dir); + TdDirPtr pDir = taosOpenDir(dirname); + if (pDir != NULL) { + taosCloseDir(&pDir); return true; } return false; } char* taosDirName(char *name) { +#ifdef WINDOWS + char Drive1[MAX_PATH], Dir1[MAX_PATH]; + _splitpath(name, Drive1, Dir1, NULL, NULL); + size_t dirNameLen = strlen(Drive1) + strlen(Dir1); + if (dirNameLen > 0) { + name[dirNameLen] = 0; + } + return name; +#else return dirname(name); +#endif } char* taosDirEntryBaseName(char *name) { - return basename(name); +#ifdef WINDOWS + char Filename1[MAX_PATH], Ext1[MAX_PATH]; + _splitpath(name, NULL, NULL, Filename1, Ext1); + return name + (strlen(name) - strlen(Filename1) - strlen(Ext1)); +#else + return (char*)basename(name); +#endif } TdDirPtr taosOpenDir(const char *dirname) { if (dirname == NULL) { return NULL; } + +#ifdef WINDOWS + char szFind[MAX_PATH]; //这是要找的 + HANDLE hFind; + + TdDirPtr pDir = taosMemoryMalloc(sizeof(TdDir)); + + strcpy(szFind, dirname); + strcat(szFind, "\\*.*"); //利用通配符找这个目录下的所以文件，包括目录 + + pDir->hFind = FindFirstFile(szFind, &(pDir->dirEntry.findFileData)); + if (INVALID_HANDLE_VALUE == pDir->hFind) { + taosMemoryFree(pDir); + return NULL; + } + return pDir; +#else return (TdDirPtr)opendir(dirname); +#endif + } TdDirEntryPtr taosReadDir(TdDirPtr pDir) { if (pDir == NULL) { return NULL; } +#ifdef WINDOWS + if (!FindNextFile(pDir->hFind, &(pDir->dirEntry.findFileData))) { + return NULL; + } + return (TdDirEntryPtr)&(pDir->dirEntry.findFileData); +#else return (TdDirEntryPtr)readdir((DIR*)pDir); +#endif } bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) { if (pDirEntry == NULL) { return false; } +#ifdef WINDOWS + return (pDirEntry->findFileData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; +#else return (((dirent*)pDirEntry)->d_type & DT_DIR) != 0; +#endif } char* taosGetDirEntryName(TdDirEntryPtr pDirEntry) { if (pDirEntry == NULL) { return NULL; } +#ifdef WINDOWS + return pDirEntry->findFileData.cFileName; +#else return ((dirent*)pDirEntry)->d_name; +#endif } -int32_t taosCloseDir(TdDirPtr pDir) { - if (pDir == NULL) { +int32_t taosCloseDir(TdDirPtr *ppDir) { + if (ppDir == NULL || *ppDir == NULL) { return -1; } - return closedir((DIR*)pDir); -} - +#ifdef WINDOWS + FindClose((*ppDir)->hFind); + taosMemoryFree(*ppDir); + *ppDir = NULL; + return 0; +#else + closedir((DIR*)*ppDir); + *ppDir = NULL; + return 0; #endif +} diff --git a/source/os/src/osEnv.c b/source/os/src/osEnv.c index 22884298ef..6746025f78 100644 --- a/source/os/src/osEnv.c +++ b/source/os/src/osEnv.c @@ -16,8 +16,6 @@ #define _DEFAULT_SOURCE #include "osEnv.h" -extern void taosWinSocketInit(); - char configDir[PATH_MAX] = {0}; char tsDataDir[PATH_MAX] = {0}; char tsLogDir[PATH_MAX] = {0}; @@ -37,6 +35,7 @@ int64_t tsOpenMax = 0; int64_t tsStreamMax = 0; float tsNumOfCores = 0; int64_t tsTotalMemoryKB = 0; +char* tsProcPath = NULL; void osDefaultInit() { taosSeedRand(taosSafeRand()); @@ -50,7 +49,7 @@ void osDefaultInit() { tsNumOfCores = 2; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS taosWinSocketInit(); const char *tmpDir = getenv("tmp"); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 3d3d13fcb6..786e58a6d2 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -16,8 +16,11 @@ #include "os.h" #include "osSemaphore.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include +#define F_OK 0 +#define W_OK 2 +#define R_OK 4 #if defined(_MSDOS) #define open _open @@ -33,6 +36,8 @@ extern int openU(const char *, int, ...); /* MsvcLibX UTF-8 version of open */ #endif /* defined(_UTF8_SOURCE) */ #endif /* defined(_WIN32) */ +#define _SEND_FILE_STEP_ 1000 + #else #include #include @@ -66,7 +71,7 @@ typedef struct TdFile { #define FILE_WITH_LOCK 1 void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS const char *tdengineTmpFileNamePrefix = "tdengine-"; char tmpPath[PATH_MAX]; @@ -117,7 +122,7 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, cha } int64_t taosCopyFile(const char *from, const char *to) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else char buffer[4096]; @@ -160,7 +165,7 @@ _err: int32_t taosRemoveFile(const char *path) { return remove(path); } int32_t taosRenameFile(const char *oldName, const char *newName) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS int32_t code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (code < 0) { // printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); @@ -178,7 +183,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { } int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else struct stat fileStat; @@ -199,7 +204,7 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { #endif } int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else struct stat fileStat; @@ -223,7 +228,7 @@ int32_t taosDevInoFile(const char *path, int64_t *stDev, int64_t *stIno) { void autoDelFileListAdd(const char *path) { return; } TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return NULL; #else int fd = -1; @@ -285,7 +290,7 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) { } int64_t taosCloseFile(TdFilePtr *ppFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (ppFile == NULL || *ppFile == NULL) { @@ -365,8 +370,15 @@ int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset) #if FILE_WITH_LOCK taosThreadRwlockRdlock(&(pFile->rwlock)); #endif - assert(pFile->fd >= 0); // Please check if you have closed the file. + assert(pFile->fd >= 0); // Please check if you have closed the file. +#ifdef WINDOWS + size_t pos = lseek(pFile->fd, 0, SEEK_CUR); + lseek(pFile->fd, (long)offset, SEEK_SET); + int64_t ret = read(pFile->fd, buf, count); + lseek(pFile->fd, pos, SEEK_SET); +#else int64_t ret = pread(pFile->fd, buf, count, offset); +#endif #if FILE_WITH_LOCK taosThreadRwlockUnlock(&(pFile->rwlock)); #endif @@ -380,7 +392,7 @@ int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count) { #if FILE_WITH_LOCK taosThreadRwlockWrlock(&(pFile->rwlock)); #endif - /*assert(pFile->fd >= 0); // Please check if you have closed the file.*/ + assert(pFile->fd >= 0); // Please check if you have closed the file. int64_t nleft = count; int64_t nwritten = 0; @@ -423,7 +435,7 @@ int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence) { } int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { @@ -450,7 +462,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { } int32_t taosLockFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { @@ -463,7 +475,7 @@ int32_t taosLockFile(TdFilePtr pFile) { } int32_t taosUnLockFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else if (pFile == NULL) { @@ -476,10 +488,10 @@ int32_t taosUnLockFile(TdFilePtr pFile) { } int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (pFile->fd < 0) { errno = EBADF; - uError("%s\n", "fd arg was negative"); + printf("%s\n", "fd arg was negative"); return -1; } @@ -489,7 +501,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { li_0.QuadPart = (int64_t)0; BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT); if (!cur) { - uError("SetFilePointerEx Error getting current position in file.\n"); + printf("SetFilePointerEx Error getting current position in file.\n"); return -1; } @@ -498,7 +510,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN); if (cur2 == 0) { int error = GetLastError(); - uError("SetFilePointerEx GetLastError is: %d\n", error); + printf("SetFilePointerEx GetLastError is: %d\n", error); switch (error) { case ERROR_INVALID_HANDLE: errno = EBADF; @@ -512,7 +524,7 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { if (!SetEndOfFile(h)) { int error = GetLastError(); - uError("SetEndOfFile GetLastError is:%d", error); + printf("SetEndOfFile GetLastError is:%d", error); switch (error) { case ERROR_INVALID_HANDLE: errno = EBADF; @@ -536,10 +548,10 @@ int32_t taosFtruncateFile(TdFilePtr pFile, int64_t l_size) { } int32_t taosFsyncFile(TdFilePtr pFile) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (pFile->fd < 0) { errno = EBADF; - uError("%s\n", "fd arg was negative"); + printf("%s\n", "fd arg was negative"); return -1; } @@ -558,84 +570,51 @@ int32_t taosFsyncFile(TdFilePtr pFile) { #endif } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { + if (pFileOut == NULL || pFileIn == NULL) { + return 0; + } + assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); -#define _SEND_FILE_STEP_ 1000 +#ifdef WINDOWS -int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { - fseek(in_file, (int32_t)(*offset), 0); + lseek(pFileIn->fd, (int32_t)(*offset), 0); int64_t writeLen = 0; uint8_t buffer[_SEND_FILE_STEP_] = {0}; - for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); + for (int64_t len = 0; len < (size - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + size_t rlen = read(pFileIn->fd, (void *)buffer, _SEND_FILE_STEP_); if (rlen <= 0) { return writeLen; } else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); + write(pFileOut->fd, (void *)buffer, (uint32_t)rlen); return (int64_t)(writeLen + rlen); } else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); + write(pFileOut->fd, (void *)buffer, _SEND_FILE_STEP_); writeLen += _SEND_FILE_STEP_; } } - int64_t remain = count - writeLen; + int64_t remain = size - writeLen; if (remain > 0) { - size_t rlen = fread(buffer, 1, (size_t)remain, in_file); + size_t rlen = read(pFileIn->fd, (void *)buffer, (size_t)remain); if (rlen <= 0) { return writeLen; } else { - fwrite(buffer, 1, (size_t)remain, out_file); + write(pFileOut->fd, (void *)buffer, (uint32_t)remain); writeLen += remain; } } - return writeLen; -} - -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { - if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0); - - int64_t writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = {0}; - - for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_); - if (rlen <= 0) { - return writeLen; - } else if (rlen < _SEND_FILE_STEP_) { - taosWriteSocket(dfd, buffer, rlen); - return (int64_t)(writeLen + rlen); - } else { - taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_); - writeLen += _SEND_FILE_STEP_; - } - } - - int64_t remain = count - writeLen; - if (remain > 0) { - int32_t rlen = read(sfd, buffer, (int32_t)remain); - if (rlen <= 0) { - return writeLen; - } else { - taosWriteSocket(sfd, buffer, (int32_t)remain); - writeLen += remain; - } - } - - return writeLen; -} #elif defined(_TD_DARWIN_64) -int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) { int r = 0; if (offset) { r = fseek(in_file, *offset, SEEK_SET); if (r == -1) return -1; } - off_t len = count; + off_t len = size; while (len > 0) { char buf[1024 * 16]; off_t n = sizeof(buf); @@ -651,64 +630,10 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co } len -= m; } - return count - len; -} - -int64_t taosSendFile(SocketFd dfd, FileFd sfd, int64_t *offset, int64_t count) { - int r = 0; - if (offset) { - r = lseek(sfd, *offset, SEEK_SET); - if (r == -1) return -1; - } - off_t len = count; - while (len > 0) { - char buf[1024 * 16]; - off_t n = sizeof(buf); - if (len < n) n = len; - size_t m = read(sfd, buf, n); - if (m == -1) return -1; - if (m == 0) break; - size_t l = write(dfd, buf, m); - if (l == -1) return -1; - len -= l; - } - return count - len; -} + return size - len; #else -// int64_t taosSendFile(int fdDst, TdFilePtr pFileSrc, int64_t *offset, int64_t size) { -// if (pFileSrc == NULL) { -// return 0; -// } -// assert(pFileSrc->fd >= 0); - -// int64_t leftbytes = size; -// int64_t sentbytes; - -// while (leftbytes > 0) { -// sentbytes = sendfile(fdDst, pFileSrc->fd, offset, leftbytes); -// if (sentbytes == -1) { -// if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { -// continue; -// } else { -// return -1; -// } -// } else if (sentbytes == 0) { -// return (int64_t)(size - leftbytes); -// } - -// leftbytes -= sentbytes; -// } - -// return size; -// } - -int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size) { - if (pFileOut == NULL || pFileIn == NULL) { - return 0; - } - assert(pFileIn->fd >= 0 && pFileOut->fd >= 0); int64_t leftbytes = size; int64_t sentbytes; @@ -728,9 +653,8 @@ int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, in } return size; -} - #endif +} void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { if (pFile == NULL) { @@ -745,22 +669,10 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { fflush(pFile->fp); } -#if !defined(WINDOWS) -void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length) { - if (pFile == NULL) { - return NULL; - } - assert(pFile->fd >= 0); // Please check if you have closed the file. - - void *ptr = mmap(NULL, length, PROT_READ, MAP_SHARED, pFile->fd, 0); - return ptr; -} -#endif - bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; } int32_t taosUmaskFile(int32_t maskVal) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else return umask(maskVal); @@ -769,16 +681,26 @@ int32_t taosUmaskFile(int32_t maskVal) { int32_t taosGetErrorFile(TdFilePtr pFile) { return errno; } int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf) { - if (pFile == NULL) { + if (pFile == NULL || ptrBuf == NULL ) { return -1; } if (*ptrBuf != NULL) { taosMemoryFreeClear(*ptrBuf); } assert(pFile->fp != NULL); - +#ifdef WINDOWS + *ptrBuf = taosMemoryMalloc(1024); + if (*ptrBuf == NULL) return -1; + if (fgets(*ptrBuf, 1023, pFile->fp) == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + (*ptrBuf)[1023] = 0; + return strlen(*ptrBuf); +#else size_t len = 0; return getline(ptrBuf, &len, pFile->fp); +#endif } int32_t taosEOFFile(TdFilePtr pFile) { if (pFile == NULL) { @@ -789,8 +711,6 @@ int32_t taosEOFFile(TdFilePtr pFile) { return feof(pFile->fp); } -#if !defined(WINDOWS) - bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { int flags = 0; @@ -805,10 +725,11 @@ bool taosCheckAccessFile(const char *pathname, int32_t tdFileAccessOptions) { if (tdFileAccessOptions & TD_FILE_ACCESS_WRITE_OK) { flags |= W_OK; } - +#ifdef WINDOWS + return _access(pathname, flags) == 0; +#else return access(pathname, flags) == 0; +#endif } bool taosCheckExistFile(const char *pathname) { return taosCheckAccessFile(pathname, TD_FILE_ACCESS_EXIST_OK); }; - -#endif // WINDOWS diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 2df61c08d4..d2369ea2a2 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "osLocale.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #if (_WIN64) #include #include @@ -87,7 +87,7 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { } void taosGetSystemLocale(char *outLocale, char *outCharset) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char *locale = setlocale(LC_CTYPE, "chs"); if (locale != NULL) { tstrncpy(outLocale, locale, TD_LOCALE_LEN); diff --git a/source/os/src/osLz4.c b/source/os/src/osLz4.c index 29d219f37b..f4e4951857 100644 --- a/source/os/src/osLz4.c +++ b/source/os/src/osLz4.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index cd756b45e2..5b733daec2 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -35,7 +35,7 @@ typedef struct TdMemoryInfo { // static TdMemoryInfoPtr GlobalMemoryPtr = NULL; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #define tstrdup(str) _strdup(str) #else #define tstrdup(str) strdup(str) @@ -181,7 +181,11 @@ int32_t taosMemorySize(void *ptr) { assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); return pTdMemoryInfo->memorySize; +#else +#ifdef WINDOWS + return _msize(ptr); #else return malloc_usable_size(ptr); #endif +#endif } diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c index b6de638ac2..f92a3b3783 100644 --- a/source/os/src/osProc.c +++ b/source/os/src/osProc.c @@ -17,9 +17,10 @@ #define _DEFAULT_SOURCE #include "os.h" -static char *tsProcPath = NULL; - int32_t taosNewProc(char **args) { +#ifdef WINDOWS + return 0; +#else int32_t pid = fork(); if (pid == 0) { args[0] = tsProcPath; @@ -30,23 +31,36 @@ int32_t taosNewProc(char **args) { } else { return pid; } +#endif } void taosWaitProc(int32_t pid) { +#ifdef WINDOWS +#else int32_t status = -1; waitpid(pid, &status, 0); +#endif } -void taosKillProc(int32_t pid) { kill(pid, SIGINT); } +void taosKillProc(int32_t pid) { +#ifdef WINDOWS +#else + kill(pid, SIGINT); +#endif +} bool taosProcExist(int32_t pid) { +#ifdef WINDOWS + return false; +#else int32_t p = getpgid(pid); return p >= 0; +#endif } // the length of the new name must be less than the original name to take effect void taosSetProcName(int32_t argc, char **argv, const char *name) { - prctl(PR_SET_NAME, name); + setThreadName(name); for (int32_t i = 0; i < argc; ++i) { int32_t len = strlen(argv[i]); diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 847f3a52a6..461a72e962 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -15,7 +15,9 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS +#include "windows.h" +#include "wincrypt.h" #else #include #include @@ -25,9 +27,25 @@ void taosSeedRand(uint32_t seed) { return srand(seed); } uint32_t taosRand(void) { return rand(); } -uint32_t taosRandR(uint32_t *pSeed) { return rand_r(pSeed); } +uint32_t taosRandR(uint32_t *pSeed) { +#ifdef WINDOWS + return rand_s(pSeed); +#else + return rand_r(pSeed); +#endif +} uint32_t taosSafeRand(void) { +#ifdef WINDOWS + uint32_t seed; + HCRYPTPROV hCryptProv; + if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, 0)) return seed; + if (hCryptProv != NULL) { + if (!CryptGenRandom(hCryptProv, 4, &seed)) return seed; + } + if (hCryptProv != NULL) CryptReleaseContext(hCryptProv, 0); + return seed; +#else TdFilePtr pFile; int seed; @@ -43,6 +61,7 @@ uint32_t taosSafeRand(void) { } return (uint32_t)seed; +#endif } void taosRandStr(char* str, int32_t size) { diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 1297fdbc27..b2c7d185e6 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation diff --git a/source/os/src/osShm.c b/source/os/src/osShm.c index b276b48d0e..1cd51f94a0 100644 --- a/source/os/src/osShm.c +++ b/source/os/src/osShm.c @@ -22,15 +22,20 @@ static int32_t shmids[MAX_SHMIDS] = {0}; static void taosDeleteCreatedShms() { +#if defined(WINDOWS) +#else for (int32_t i = 0; i < MAX_SHMIDS; ++i) { int32_t shmid = shmids[i] - 1; if (shmid >= 0) { shmctl(shmid, IPC_RMID, NULL); } } +#endif } int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { +#if defined(WINDOWS) +#else pShm->id = -1; #if 1 @@ -64,10 +69,13 @@ int32_t taosCreateShm(SShm* pShm, int32_t key, int32_t shmsize) { shmctl(pShm->id, IPC_RMID, NULL); #endif +#endif return 0; } void taosDropShm(SShm* pShm) { +#if defined(WINDOWS) +#else if (pShm->id >= 0) { if (pShm->ptr != NULL) { shmdt(pShm->ptr); @@ -77,14 +85,18 @@ void taosDropShm(SShm* pShm) { pShm->id = -1; pShm->size = 0; pShm->ptr = NULL; +#endif } int32_t taosAttachShm(SShm* pShm) { +#if defined(WINDOWS) +#else errno = 0; void* ptr = shmat(pShm->id, NULL, 0); if (errno == 0) { pShm->ptr = ptr; } +#endif return errno; } diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index 1d7fa517e5..d9b225868a 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS /* * windows implementation @@ -47,6 +47,8 @@ void taosDflSignal(int32_t signum) { signal(signum, SIG_DFL); } +void taosKillChildOnParentStopped() { } + #else /* diff --git a/source/os/src/osSleep.c b/source/os/src/osSleep.c index 724347b0bc..870467ceef 100644 --- a/source/os/src/osSleep.c +++ b/source/os/src/osSleep.c @@ -23,7 +23,7 @@ #endif void taosSsleep(int32_t s) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS Sleep(1000 * s); #else sleep(s); @@ -31,7 +31,7 @@ void taosSsleep(int32_t s) { } void taosMsleep(int32_t ms) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS Sleep(ms); #else usleep(ms * 1000); @@ -39,8 +39,15 @@ void taosMsleep(int32_t ms) { } void taosUsleep(int32_t us) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - nanosleep(1000 * us); +#ifdef WINDOWS + HANDLE timer; + LARGE_INTEGER interval; + interval.QuadPart = (10 * us); + + timer = CreateWaitableTimer(NULL, TRUE, NULL); + SetWaitableTimer(timer, &interval, 0, NULL, NULL, 0); + WaitForSingleObject(timer, INFINITE); + CloseHandle(timer); #else usleep(us); #endif diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index d79d7c3d58..0430f68a26 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -25,9 +25,6 @@ #include #include #include -#include -#include -#include "winsock2.h" #else #include #include @@ -68,7 +65,7 @@ int32_t taosSendto(TdSocketPtr pSocket, void *buf, int len, unsigned int flags, if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); #else return sendto(pSocket->fd, buf, len, flags, dest_addr, addrlen); @@ -78,7 +75,7 @@ int32_t taosWriteSocket(TdSocketPtr pSocket, void *buf, int len) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return send(pSocket->fd, buf, len, 0); ; #else @@ -89,7 +86,7 @@ int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return recv(pSocket->fd, buf, len, 0); ; #else @@ -97,14 +94,14 @@ int32_t taosReadSocket(TdSocketPtr pSocket, void *buf, int len) { #endif } -int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen) { +int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } return recvfrom(pSocket->fd, buf, len, flags, destAddr, addrLen); } int32_t taosCloseSocketNoCheck1(SocketFd fd) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return closesocket(fd); #else return close(fd); @@ -205,8 +202,8 @@ int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { #endif } -void taosWinSocketInit1() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +void taosWinSocketInit() { +#ifdef WINDOWS static char flag = 0; if (flag == 0) { WORD wVersionRequested; @@ -223,7 +220,7 @@ int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS u_long mode; if (on) { mode = 1; @@ -255,7 +252,7 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { return 0; } @@ -274,21 +271,21 @@ int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void return setsockopt(pSocket->fd, level, optname, optval, optlen); #else - return setsockopt(pSocket->fd, level, optname, optval, (socklen_t)optlen); + return setsockopt(pSocket->fd, level, optname, optval, (int)optlen); #endif } int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return 0; #else - return getsockopt(pSocket->fd, level, optname, optval, (socklen_t *)optlen); + return getsockopt(pSocket->fd, level, optname, optval, (int *)optlen); #endif } uint32_t taosInetAddr(const char *ipAddr) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS uint32_t value; int32_t ret = inet_pton(AF_INET, ipAddr, &value); if (ret <= 0) { @@ -301,7 +298,7 @@ uint32_t taosInetAddr(const char *ipAddr) { #endif } const char *taosInetNtoa(struct in_addr ipInt) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS // not thread safe, only for debug usage while print log static char tmpDstStr[16]; return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); @@ -693,7 +690,7 @@ TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { } TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, - socklen_t *addrLen) { + int *addrLen) { if (pServerSocket == NULL || pServerSocket->fd < 0) { return NULL; } @@ -753,12 +750,12 @@ int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len } void taosBlockSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigMask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { // printf("failed to block SIGPIPE"); } @@ -864,26 +861,26 @@ void tinet_ntoa(char *ipstr, uint32_t ip) { } void taosIgnSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else signal(SIGPIPE, SIG_IGN); #endif } void taosSetMaskSIGPIPE() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigMask(SIG_SETMASK, &signal_mask, NULL); + int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { // printf("failed to setmask SIGPIPE"); } #endif } -int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, socklen_t *addrLen) { +int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen) { if (pSocket == NULL || pSocket->fd < 0) { return -1; } @@ -893,7 +890,7 @@ int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, sockle TdEpollPtr taosCreateEpoll(int32_t size) { EpollFd fd = -1; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else fd = epoll_create(size); #endif @@ -915,7 +912,7 @@ int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocke if (pEpoll == NULL || pEpoll->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else code = epoll_ctl(pEpoll->fd, epollOperate, pSocket->fd, event); #endif @@ -926,7 +923,7 @@ int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxE if (pEpoll == NULL || pEpoll->fd < 0) { return -1; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #else code = epoll_wait(pEpoll->fd, event, maxEvents, timeout); #endif diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 5adb564e52..ed596a051d 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -24,6 +24,55 @@ extern int wcwidth(wchar_t c); extern int wcswidth(const wchar_t *s, size_t n); +#ifdef WINDOWS +char *strsep(char **stringp, const char *delim) { + char * s; + const char *spanp; + int32_t c, sc; + char *tok; + if ((s = *stringp) == NULL) + return (NULL); + for (tok = s;;) { + c = *s++; + spanp = delim; + do { + if ((sc = *spanp++) == c) { + if (c == 0) + s = NULL; + else + s[-1] = 0; + *stringp = s; + return (tok); + } + } while (sc != 0); + } + /* NOTREACHED */ +} +/* Duplicate a string, up to at most size characters */ +char *strndup(const char *s, size_t size) { + size_t l; + char *s2; + l = strlen(s); + if (l > size) l=size; + s2 = malloc(l+1); + if (s2) { + strncpy(s2, s, l); + s2[l] = '\0'; + } + return s2; +} +/* Copy no more than N characters of SRC to DEST, returning the address of + the terminating '\0' in DEST, if any, or else DEST + N. */ +char *stpncpy (char *dest, const char *src, size_t n) { + size_t size = strnlen (src, n); + memcpy (dest, src, size); + dest += size; + if (size == n) + return dest; + return memset (dest, '\0', n - size); +} +#endif + int64_t taosStr2int64(const char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); diff --git a/source/os/src/osStrptime.c b/source/os/src/osStrptime.c index 99549f02cb..8d878577ea 100644 --- a/source/os/src/osStrptime.c +++ b/source/os/src/osStrptime.c @@ -38,7 +38,7 @@ // //#include "lukemftp.h" -// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +// #ifdef WINDOWS // #include // #include @@ -100,7 +100,7 @@ // #endif // char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { -// #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +// #ifdef WINDOWS // char c; // const char *bp; // size_t len = 0; diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 4ffbc13fb3..ccc302e646 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -17,7 +17,23 @@ #include "os.h" #include "taoserror.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#define PROCESS_ITEM 12 + +typedef struct { + uint64_t user; + uint64_t nice; + uint64_t system; + uint64_t idle; +} SysCpuInfo; + +typedef struct { + uint64_t utime; // user time + uint64_t stime; // kernel time + uint64_t cutime; // all user time + uint64_t cstime; // all dead time +} ProcCpuInfo; + +#ifdef WINDOWS /* * windows implementation @@ -92,22 +108,6 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) { #include #include -#define PROCESS_ITEM 12 - -typedef struct { - uint64_t user; - uint64_t nice; - uint64_t system; - uint64_t idle; -} SysCpuInfo; - -typedef struct { - uint64_t utime; // user time - uint64_t stime; // kernel time - uint64_t cutime; // all user time - uint64_t cstime; // all dead time -} ProcCpuInfo; - static pid_t tsProcId; static char tsSysNetFile[] = "/proc/net/dev"; static char tsSysCpuFile[] = "/proc/stat"; @@ -125,8 +125,12 @@ static void taosGetProcIOnfos() { snprintf(tsProcCpuFile, sizeof(tsProcCpuFile), "/proc/%d/stat", tsProcId); snprintf(tsProcIOFile, sizeof(tsProcIOFile), "/proc/%d/io", tsProcId); } +#endif static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else TdFilePtr pFile = taosOpenFile(tsSysCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return -1; @@ -145,10 +149,14 @@ static int32_t taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); +#endif return 0; } static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { +#ifdef WINDOWS +#elif defined(_TD_DARWIN_64) +#else TdFilePtr pFile = taosOpenFile(tsProcCpuFile, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { return -1; @@ -172,10 +180,10 @@ static int32_t taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { if (line != NULL) taosMemoryFreeClear(line); taosCloseFile(&pFile); +#endif return 0; } -#endif bool taosCheckSystemIsSmallEnd() { union check { @@ -187,7 +195,7 @@ bool taosCheckSystemIsSmallEnd() { } void taosGetSystemInfo() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS taosGetCpuCores(&tsNumOfCores); taosGetTotalMemory(&tsTotalMemoryKB); @@ -210,7 +218,7 @@ void taosGetSystemInfo() { } int32_t taosGetEmail(char *email, int32_t maxLen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) const char *filepath = "/usr/local/taos/email"; @@ -241,7 +249,7 @@ int32_t taosGetEmail(char *email, int32_t maxLen) { } int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -296,7 +304,7 @@ int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen) { } int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) char *line = NULL; size_t size = 0; @@ -355,7 +363,7 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { } int32_t taosGetCpuCores(float *numOfCores) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS SYSTEM_INFO info; GetSystemInfo(&info); *numOfCores = info.dwNumberOfProcessors; @@ -399,7 +407,7 @@ void taosGetCpuUsage(double *cpu_system, double *cpu_engine) { } int32_t taosGetTotalMemory(int64_t *totalKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { @@ -417,7 +425,7 @@ int32_t taosGetTotalMemory(int64_t *totalKB) { } int32_t taosGetProcMemory(int64_t *usedKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS unsigned bytes_used = 0; #if defined(_WIN64) && defined(_MSC_VER) @@ -469,7 +477,7 @@ int32_t taosGetProcMemory(int64_t *usedKB) { } int32_t taosGetSysMemory(int64_t *usedKB) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS MEMORYSTATUSEX memsStat; memsStat.dwLength = sizeof(memsStat); if (!GlobalMemoryStatusEx(&memsStat)) { @@ -534,7 +542,7 @@ int32_t taosGetDiskSize(char *dataDir, SDiskSize *diskSize) { } int32_t taosGetProcIO(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, int64_t *write_bytes) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS IO_COUNTERS io_counter; if (GetProcessIoCounters(GetCurrentProcess(), &io_counter)) { if (rchars) *rchars = io_counter.ReadTransferCount; @@ -620,7 +628,7 @@ void taosGetProcIODelta(int64_t *rchars, int64_t *wchars, int64_t *read_bytes, i } int32_t taosGetCardInfo(int64_t *receive_bytes, int64_t *transmit_bytes) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS *receive_bytes = 0; *transmit_bytes = 0; return 0; @@ -691,7 +699,7 @@ void taosGetCardInfoDelta(int64_t *receive_bytes, int64_t *transmit_bytes) { } void taosKillSystem() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS printf("function taosKillSystem, exit!"); exit(0); #elif defined(_TD_DARWIN_64) @@ -705,7 +713,7 @@ void taosKillSystem() { } int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS GUID guid; CoCreateGuid(&guid); @@ -741,7 +749,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { } char *taosGetCmdlineByPID(int pid) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS return ""; #elif defined(_TD_DARWIN_64) static char cmdline[1024]; @@ -777,7 +785,7 @@ char *taosGetCmdlineByPID(int pid) { } void taosSetCoreDump(bool enable) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS SetUnhandledExceptionFilter(&FlCrashDump); #elif defined(_TD_DARWIN_64) #else @@ -857,7 +865,7 @@ void taosSetCoreDump(bool enable) { } SysNameInfo taosGetSysNameInfo() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #elif defined(_TD_DARWIN_64) SysNameInfo info = {0}; diff --git a/source/os/src/osSystem.c b/source/os/src/osSystem.c index 665f6370e1..cf3c95e658 100644 --- a/source/os/src/osSystem.c +++ b/source/os/src/osSystem.c @@ -183,16 +183,33 @@ void resetTerminalMode() { TdCmdPtr taosOpenCmd(const char *cmd) { if (cmd == NULL) return NULL; +#ifdef WINDOWS + return (TdCmdPtr)_popen(cmd, "r"); +#else return (TdCmdPtr)popen(cmd, "r"); +#endif } int64_t taosGetLineCmd(TdCmdPtr pCmd, char ** __restrict ptrBuf) { - if (pCmd == NULL) { + if (pCmd == NULL || ptrBuf == NULL ) { return -1; } - + if (*ptrBuf != NULL) { + taosMemoryFreeClear(*ptrBuf); + } +#ifdef WINDOWS + *ptrBuf = taosMemoryMalloc(1024); + if (*ptrBuf == NULL) return -1; + if (fgets(*ptrBuf, 1023, (FILE*)pCmd) == NULL) { + taosMemoryFreeClear(*ptrBuf); + return -1; + } + (*ptrBuf)[1023] = 0; + return strlen(*ptrBuf); +#else size_t len = 0; return getline(ptrBuf, &len, (FILE*)pCmd); +#endif } int32_t taosEOFCmd(TdCmdPtr pCmd) { @@ -206,7 +223,11 @@ int64_t taosCloseCmd(TdCmdPtr *ppCmd) { if (ppCmd == NULL || *ppCmd == NULL) { return 0; } +#ifdef WINDOWS + _pclose((FILE*)(*ppCmd)); +#else pclose((FILE*)(*ppCmd)); +#endif *ppCmd = NULL; return 0; } diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 656871fd3c..a2778e44c5 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -294,7 +294,7 @@ int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { } int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_destroy((pthread_mutex_t*)lock); #else return pthread_spin_destroy((pthread_spinlock_t*)lock); @@ -302,15 +302,16 @@ int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { } int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared) { -#ifndef __USE_XOPEN2K - return pthread_mutex_init((pthread_mutex_t*)lock, pshared); +#ifdef TD_USE_SPINLOCK_AS_MUTEX + assert(pshared == NULL); + return pthread_mutex_init((pthread_mutex_t*)lock, NULL); #else return pthread_spin_init((pthread_spinlock_t*)lock, pshared); #endif } int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_lock((pthread_mutex_t*)lock); #else return pthread_spin_lock((pthread_spinlock_t*)lock); @@ -318,7 +319,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { } int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_trylock((pthread_mutex_t*)lock); #else return pthread_spin_trylock((pthread_spinlock_t*)lock); @@ -326,7 +327,7 @@ int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { } int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock) { -#ifndef __USE_XOPEN2K +#ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_unlock((pthread_mutex_t*)lock); #else return pthread_spin_unlock((pthread_spinlock_t*)lock); @@ -337,10 +338,6 @@ void taosThreadTestCancel(void) { return pthread_testcancel(); } -int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset) { - return pthread_sigmask(how, set, oset); -} - -int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig) { - return sigwait(set, sig); +void taosThreadClear(TdThread *thread) { + memset(thread, 0, sizeof(TdThread)); } \ No newline at end of file diff --git a/source/os/src/osTime.c b/source/os/src/osTime.c index 9ea49b364e..5d6b4e7c2b 100644 --- a/source/os/src/osTime.c +++ b/source/os/src/osTime.c @@ -26,12 +26,11 @@ #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include #include #include -#include //#define TM_YEAR_BASE 1970 //origin #define TM_YEAR_BASE 1900 //slguan /* @@ -85,13 +84,43 @@ static const char *am_pm[2] = { "AM", "PM" }; +#define BILLION (1E9) + +static BOOL g_first_time = 1; +static LARGE_INTEGER g_counts_per_sec; + +int clock_gettime(int dummy, struct timespec *ct) +{ + LARGE_INTEGER count; + + if (g_first_time) + { + g_first_time = 0; + + if (0 == QueryPerformanceFrequency(&g_counts_per_sec)) + { + g_counts_per_sec.QuadPart = 0; + } + } + + if ((NULL == ct) || (g_counts_per_sec.QuadPart <= 0) || + (0 == QueryPerformanceCounter(&count))) + { + return -1; + } + + ct->tv_sec = count.QuadPart / g_counts_per_sec.QuadPart; + ct->tv_nsec = ((count.QuadPart % g_counts_per_sec.QuadPart) * BILLION) / g_counts_per_sec.QuadPart; + + return 0; +} #else #include #endif char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char c; const char *bp; size_t len = 0; @@ -391,7 +420,7 @@ char *taosStrpTime(const char *buf, const char *fmt, struct tm *tm) { } FORCE_INLINE int32_t taosGetTimeOfDay(struct timeval *tv) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS time_t t; t = taosGetTimestampSec(); SYSTEMTIME st; @@ -418,7 +447,7 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { if (result == NULL) { return localtime(timep); } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS localtime_s(result, timep); #else localtime_r(timep, result); @@ -427,3 +456,5 @@ struct tm *taosLocalTime(const time_t *timep, struct tm *result) { } int32_t taosGetTimestampSec() { return (int32_t)time(NULL); } + +int32_t taosClockGetTime(int clock_id, struct timespec *pTS) { return clock_gettime(clock_id, pTS); } \ No newline at end of file diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index e6e3ec7962..2992b29098 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #include #include #include @@ -143,7 +143,7 @@ static void * taosProcessAlarmSignal(void *tharg) { #endif int taosInitTimer(void (*callback)(int), int ms) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS DWORD_PTR param = *((int64_t *)&callback); timerId = timeSetEvent(ms, 1, (LPTIMECALLBACK)taosWinOnTimer, param, TIME_PERIODIC); @@ -190,7 +190,7 @@ int taosInitTimer(void (*callback)(int), int ms) { } void taosUninitTimer() { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS timeKillEvent(timerId); #elif defined(_TD_DARWIN_64) int r = 0; diff --git a/source/os/src/osTimezone.c b/source/os/src/osTimezone.c index dc23eaae1a..cbf20f02cd 100644 --- a/source/os/src/osTimezone.c +++ b/source/os/src/osTimezone.c @@ -17,7 +17,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS #if (_WIN64) #include #include @@ -59,11 +59,12 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 buf[i] = inTimezoneStr[i]; } -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char winStr[TD_LOCALE_LEN * 2]; sprintf(winStr, "TZ=%s", buf); putenv(winStr); tzset(); + /* * get CURRENT time zone. * system current time zone is affected by daylight saving time(DST) * @@ -116,7 +117,7 @@ void taosSetSystemTimezone(const char *inTimezoneStr, char *outTimezoneStr, int8 } void taosGetSystemTimezone(char *outTimezoneStr, enum TdTimezone *tsTimezone) { -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS char *tz = getenv("TZ"); if (tz == NULL || strlen(tz) == 0) { strcpy(outTimezoneStr, "not configured"); diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 8609c6eb21..8553d714fc 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( ) target_link_libraries( util - PRIVATE os + PUBLIC os PUBLIC lz4_static PUBLIC api cjson ) diff --git a/source/util/src/talgo.c b/source/util/src/talgo.c index 48a0f327c4..8675670cfe 100644 --- a/source/util/src/talgo.c +++ b/source/util/src/talgo.c @@ -229,22 +229,21 @@ void *taosbsearch(const void *key, const void *base, int64_t nmemb, int64_t size } void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, - __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + __ext_compar_fn_t compar, char* buf, bool maxroot) { int32_t parent; int32_t child; - char *buf; + + char* tmp = NULL; + if (buf == NULL) { + tmp = taosMemoryMalloc(size); + } else { + tmp = buf; + } if (base && size > 0 && compar) { parent = start; child = 2 * parent + 1; - if (swap == NULL) { - buf = taosMemoryCalloc(1, size); - if (buf == NULL) { - return; - } - } - if (maxroot) { while (child <= end) { if (child + 1 <= end && @@ -256,11 +255,7 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const break; } - if (swap == NULL) { - doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); - } else { - (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); - } + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp); parent = child; child = 2 * parent + 1; @@ -276,33 +271,35 @@ void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const break; } - if (swap == NULL) { - doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); - } else { - (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); - } + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, tmp); parent = child; child = 2 * parent + 1; } } + } - if (swap == NULL) { - taosMemoryFreeClear(buf); - } + if (buf == NULL) { + taosMemoryFree(tmp); } } void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, - const void *parswap, __ext_swap_fn_t swap, bool maxroot) { + bool maxroot) { int32_t i; + char* buf = taosMemoryCalloc(1, size); + if (buf == NULL) { + return; + } + if (base && size > 0) { for (i = len / 2 - 1; i >= 0; i--) { - taosheapadjust(base, size, i, len - 1, parcompar, compar, parswap, swap, maxroot); + taosheapadjust(base, size, i, len - 1, parcompar, compar, buf, maxroot); } } + taosMemoryFree(buf); /* char *buf = taosMemoryCalloc(1, size); diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index a69f11f285..5f838b9af6 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tcache.h" #include "taoserror.h" +#include "osThread.h" #include "tlog.h" #include "tutil.h" @@ -24,7 +25,7 @@ static TdThread cacheRefreshWorker = {0}; static TdThreadOnce cacheThreadInit = PTHREAD_ONCE_INIT; -static TdThreadMutex guard = PTHREAD_MUTEX_INITIALIZER; +static TdThreadMutex guard = TD_PTHREAD_MUTEX_INITIALIZER; static SArray *pCacheArrayList = NULL; static bool stopRefreshWorker = false; static bool refreshWorkerNormalStopped = false; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 9a0af24319..eea5c9fecc 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -139,7 +139,7 @@ static int32_t cfgCheckAndSetDir(SConfigItem *pItem, const char *inputDir) { return -1; } - if (taosRealPath(fullDir, PATH_MAX) != 0) { + if (taosRealPath(fullDir, NULL, PATH_MAX) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to get realpath of dir:%s since %s", inputDir, terrstr()); return -1; diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 6d3a259079..d2b07875a3 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -135,7 +135,7 @@ static FORCE_INLINE void taosHashEntryRUnlock(const SHashObj *pHashObj, SHashEnt } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { - int32_t len = MIN(length, HASH_MAX_CAPACITY); + int32_t len = (length < HASH_MAX_CAPACITY ? length : HASH_MAX_CAPACITY); int32_t i = 4; while (i < len) i = (i << 1u); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 6da4913114..23fe2e8301 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE +#include "os.h" #include "tlog.h" #include "tutil.h" @@ -222,7 +223,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - taosSsleep(3); + taosSsleep(10); taosCloseLogByFd(pOldFile); uInfo(" new log file:%d is opened", tsLogObj.flag); diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index ae7b3d6f1f..03c9c255b0 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -286,13 +286,13 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea pQueue->head = headLen + bodyLen; } else if (remain < 8 + headLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, remain - 8); - memcpy(pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); + memcpy((char*)pHead + remain - 8, pQueue->pBuffer, headLen - (remain - 8)); memcpy(pBody, pQueue->pBuffer + headLen - (remain - 8), bodyLen); pQueue->head = headLen - (remain - 8) + bodyLen; } else if (remain < 8 + headLen + bodyLen) { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); memcpy(pBody, pQueue->pBuffer + pQueue->head + 8 + headLen, remain - 8 - headLen); - memcpy(pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); + memcpy((char*)pBody + remain - 8 - headLen, pQueue->pBuffer, bodyLen - (remain - 8 - headLen)); pQueue->head = bodyLen - (remain - 8 - headLen); } else { memcpy(pHead, pQueue->pBuffer + pQueue->head + 8, headLen); @@ -434,7 +434,7 @@ void taosProcStop(SProcObj *pProc) { } tsem_post(&pQueue->sem); taosThreadJoin(pProc->thread, NULL); - pProc->thread = 0; + taosThreadClear(&pProc->thread); } void taosProcCleanup(SProcObj *pProc) { diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index 118fe58d2e..13637b8fe4 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -559,7 +559,7 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { const uint32_t factor = 4; int32_t n = 1; -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#ifdef WINDOWS while ((taosRand() % factor) == 0 && n <= pSkipList->maxLevel) { #else while ((taosRandR(&(pSkipList->seed)) % factor) == 0 && n <= pSkipList->maxLevel) { diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index fecc58c236..34a59a7d48 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -132,7 +132,7 @@ static timer_map_t timerMap; static uintptr_t getNextTimerId() { uintptr_t id; do { - id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, 1); + id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, (void*)1); } while (id == 0); return id; } diff --git a/source/util/test/freelistTest.cpp b/source/util/test/freelistTest.cpp index 440e997042..a445a16ad3 100644 --- a/source/util/test/freelistTest.cpp +++ b/source/util/test/freelistTest.cpp @@ -14,4 +14,4 @@ TEST(TD_UTIL_FREELIST_TEST, simple_test) { } tFreeListClear(&fl); -} \ No newline at end of file +} diff --git a/source/util/test/procTest.cpp b/source/util/test/procTest.cpp index e05008e8e1..3c014369fb 100644 --- a/source/util/test/procTest.cpp +++ b/source/util/test/procTest.cpp @@ -63,19 +63,19 @@ TEST_F(UtilTesProc, 00_Init_Cleanup) { ASSERT_EQ(taosCreateShm(&shm, 1234, 1024 * 1024 * 2), 0); shm.size = 1023; - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)NULL, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryMalloc, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryMalloc, - .shm = shm, - .parent = &shm, - .name = "1234"}; + SProcCfg cfg = {(ProcConsumeFp)NULL, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryMalloc, + shm, + &shm, + "1234"}; SProcObj *proc = taosProcInit(&cfg); ASSERT_EQ(proc, nullptr); @@ -104,19 +104,19 @@ void ConsumeChild1(void *parent, void *pHead, int16_t headLen, void *pBody, int3 TEST_F(UtilTesProc, 01_Push_Pop_Child) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1235, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)ConsumeChild1, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1235), - .name = "1235_c"}; + SProcCfg cfg = {(ProcConsumeFp)ConsumeChild1, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1235_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -162,19 +162,19 @@ void ConsumeParent1(void *parent, void *pHead, int16_t headLen, void *pBody, int TEST_F(UtilTesProc, 02_Push_Pop_Parent) { shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1236, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)NULL, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)ConsumeParent1, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1236), - .name = "1236_c"}; + SProcCfg cfg = {(ProcConsumeFp)NULL, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)ConsumeParent1, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1236), + "1236_c"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); @@ -216,19 +216,19 @@ TEST_F(UtilTesProc, 03_Handle) { // uDebugFlag = 207; shm.size = 3000; ASSERT_EQ(taosCreateShm(&shm, 1237, shm.size), 0); - SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)ConsumeChild3, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .childFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .parentConsumeFp = (ProcConsumeFp)NULL, - .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeBodyFp = (ProcFreeFp)taosMemoryFree, - .shm = shm, - .parent = (void *)((int64_t)1235), - .name = "1237_p"}; + SProcCfg cfg = {(ProcConsumeFp)ConsumeChild3, + (ProcMallocFp)taosAllocateQitem, + (ProcFreeFp)taosFreeQitem, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcConsumeFp)NULL, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + (ProcMallocFp)taosMemoryMalloc, + (ProcFreeFp)taosMemoryFree, + shm, + (void *)((int64_t)1235), + "1237_p"}; SProcObj *cproc = taosProcInit(&cfg); ASSERT_NE(cproc, nullptr); diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 3174d57aef..89561e61fe 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -36,7 +36,7 @@ void *addRef(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("a"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; if (pSpace->rid[id] <= 0) { pSpace->p[id] = taosMemoryMalloc(128); pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); @@ -53,7 +53,7 @@ void *removeRef(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("d"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; if (pSpace->rid[id] > 0) { code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]); if (code == 0) pSpace->rid[id] = 0; @@ -72,7 +72,7 @@ void *acquireRelease(void *param) { for (int i=0; i < pSpace->steps; ++i) { printf("a"); - id = random() % pSpace->refNum; + id = taosRand() % pSpace->refNum; void *p = taosAcquireRef(pSpace->rsetId, (int64_t) pSpace->p[id]); if (p) { taosUsleep(id % 5 + 1); diff --git a/tests/script/test.sh b/tests/script/test.sh index 14dc43beaf..e4191da0a9 100755 --- a/tests/script/test.sh +++ b/tests/script/test.sh @@ -131,8 +131,8 @@ if [ -n "$FILE_NAME" ]; then FLAG="-v" fi - echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG - valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + echo valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG + valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --child-silent-after-fork=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tsim.log $PROGRAM -c $CFG_DIR -f $FILE_NAME $FLAG else if [[ $MULTIPROCESS -eq 1 ]];then echo "ExcuteCmd(multiprocess):" $PROGRAM -m -c $CFG_DIR -f $FILE_NAME diff --git a/tests/script/tsim/query/session.sim b/tests/script/tsim/query/session.sim index d5e77c7b28..a69b6249fc 100644 --- a/tests/script/tsim/query/session.sim +++ b/tests/script/tsim/query/session.sim @@ -279,7 +279,8 @@ endi print ================> syntax error check not active ================> reactive sql_error select * from dev_001 session(ts,1w) -sql_error select count(*) from st session(ts,1w) +print disable this temporarily, session can not be directly applied to super table. +#sql_error select count(*) from st session(ts,1w) sql_error select count(*) from dev_001 group by tagtype session(ts,1w) sql_error sql select count(*) from dev_001 session(ts,1n) sql_error sql select count(*) from dev_001 session(ts,1y) diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index 60064b174e..df6a553d1a 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -8,196 +8,245 @@ # # notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). # -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 -system sh/exec.sh -n dnode1 -s start +run tsim/tmq/prepareBasicEnv.sim -$loop_cnt = 0 -check_dnode_ready: - $loop_cnt = $loop_cnt + 1 - sleep 200 - if $loop_cnt == 10 then - print ====> dnode not ready! - return -1 - endi -sql show dnodes -print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 -if $data00 != 1 then - return -1 -endi -if $data04 != ready then - goto check_dnode_ready -endi +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$showMsg = 1 +$showRow = 0 sql connect +sql use $dbName -$dbNamme = d0 -print =============== create database , vgroup 4 -sql create database $dbNamme vgroups 4 -sql use $dbNamme - -print =============== create super table -sql create table if not exists stb (ts timestamp, c1 int, c2 float, c3 binary(10)) tags (t1 int) - -sql show stables -if $rows != 1 then - return -1 -endi - -print =============== create child table -sql create table ct0 using stb tags(1000) -sql create table ct1 using stb tags(2000) -#sql create table ct3 using stb tags(3000) - -print =============== create normal table -sql create table ntb (ts timestamp, c1 int, c2 float, c3 binary(10)) - -print =============== create multi topics. notes: now only support: -print =============== 1. columns from stb; 2. * from ctb; 3. columns from ctb -print =============== will support: * from stb; function from stb/ctb - -sql create topic topic_stb_column as select ts, c1, c3 from stb -#sql create topic topic_stb_all as select * from stb +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb -sql create topic topic_ctb_column as select ts, c1, c3 from ct0 -sql create topic topic_ctb_all as select * from ct0 -sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 -sql create topic topic_ntb_column as select ts, c1, c3 from ntb -sql create topic topic_ntb_all as select * from ntb -sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 -sql show tables -if $rows != 3 then - return -1 -endi - -print =============== insert data - -$tbPrefix = ct -$tbNum = 2 -$rowNum = 10 -$tstart = 1640966400000 # 2022-01-01 00:00:00.000 - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - - $x = 0 - while $x < $rowNum - $c = $x / 10 - $c = $c * 10 - $c = $x - $c - - $binary = ' . binary - $binary = $binary . $c - $binary = $binary . ' - - sql insert into $tb values ($tstart , $c , $x , $binary ) - sql insert into ntb values ($tstart , $c , $x , $binary ) - $tstart = $tstart + 1 - $x = $x + 1 - endw - - $i = $i + 1 - $tstart = 1640966400000 -endw - -#root@trd02 /home $ tmq_sim --help -# -c Configuration directory, default is -# -d The name of the database for cosumer, no default -# -t The topic string for cosumer, no default -# -k The key-value string for cosumer, no default -# -g showMsgFlag, default is 0 -# - -$totalMsgCnt = $rowNum * $tbNum -print inserted totalMsgCnt: $totalMsgCnt - -# supported key: -# group.id: -# enable.auto.commit: -# auto.offset.reset: -# td.connect.ip: -# td.connect.user:root -# td.connect.pass:taosdata -# td.connect.port:6030 -# td.connect.db:db - -system_content echo -n \$BUILD_DIR -$tmq_sim = $system_content . /build/bin/tmq_sim -system_content echo -n \$SIM_DIR -$tsim_cfg = $system_content . /tsim/cfg - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_column" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then - return -1 -endi - -#print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -#system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_all" -k "group.id:tg2" -#print cmd result----> $system_content -#if $system_content != @{consume success: 20, 0}@ then +#sql show topics +#if $rows != 9 then # return -1 #endi -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then - print expect @{consume success: 20, 0}@, actual: $system_content - return -1 +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . ' + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end endi -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 10, 0}@ then +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +#$expectmsgcnt = $totalMsgOfStb + 1 +$expectmsgcnt = 110 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then return -1 endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_all" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 10, 0}@ then +if $data[0][2] != $expectmsgcnt then return -1 endi - -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 10, 0}@ then +if $data[0][3] != $expectmsgcnt then return -1 endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb1 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then return -1 endi +####################################################################################### -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_all" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then - return -1 + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end endi -print cmd===> system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -system_content $tmq_sim -c $tsim_cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" -print cmd result----> $system_content -if $system_content != @{consume success: 20, 0}@ then +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfCtb + 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then return -1 endi +if $data[0][2] != $totalMsgOfCtb then + return -1 +endi +if $data[0][3] != $totalMsgOfCtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: -print =============== create database , vgroup 4 -$dbNamme = d1 -sql create database $dbNamme vgroups 4 +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdbName = cdb2 +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb + 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $totalMsgOfNtb then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/tmq/clearConsume.sim b/tests/script/tsim/tmq/clearConsume.sim new file mode 100644 index 0000000000..8b55e0f8c3 --- /dev/null +++ b/tests/script/tsim/tmq/clearConsume.sim @@ -0,0 +1,28 @@ +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +sql use $dbName + +print == create consume info table and consume result table +sql drop table consumeinfo +sql drop table consumeresult +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + diff --git a/tests/script/tsim/tmq/consume.sh b/tests/script/tsim/tmq/consume.sh index ac500e6704..28e03d8fb9 100755 --- a/tests/script/tsim/tmq/consume.sh +++ b/tests/script/tsim/tmq/consume.sh @@ -11,16 +11,25 @@ set +e # set default value for parameters EXEC_OPTON=start DB_NAME=db +CDB_NAME=db POLL_DELAY=5 VALGRIND=0 SIGNAL=SIGINT +SHOW_MSG=0 +SHOW_ROW=0 -while getopts "d:s:v:y:x:" arg +while getopts "d:s:v:y:x:g:r:w:" arg do case $arg in d) DB_NAME=$OPTARG ;; + g) + SHOW_MSG=$OPTARG + ;; + r) + SHOW_ROW=$OPTARG + ;; s) EXEC_OPTON=$OPTARG ;; @@ -33,6 +42,9 @@ do x) SIGNAL=$OPTARG ;; + w) + CDB_NAME=$OPTARG + ;; ?) echo "unkown argument" ;; @@ -80,11 +92,11 @@ echo "DB_NAME: $DB_NAME echo "------------------------------------------------------------------------" if [ "$EXEC_OPTON" = "start" ]; then if [ $VALGRIND -eq 1 ]; then - echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 & - nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 & + echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 & + nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 & else - echo "nohup $PROGRAM -c $CFG_DIR -d $DB_NAME -y $POLL_DELAY > /dev/null 2>&1 &" - nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME > /dev/null 2>&1 & + echo "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &" + nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 & fi else PID=`ps -ef|grep tmq_sim | grep -v grep | awk '{print $2}'` diff --git a/tests/script/tsim/tmq/prepareBasicEnv.sim b/tests/script/tsim/tmq/prepareBasicEnv.sim new file mode 100644 index 0000000000..066d7d4ab0 --- /dev/null +++ b/tests/script/tsim/tmq/prepareBasicEnv.sim @@ -0,0 +1,88 @@ +# stop all dnodes before start this case +system sh/stop_dnodes.sh + +# deploy dnode 1 +system sh/deploy.sh -n dnode1 -i 1 + +# add some config items for this case +#system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 + +# start dnode 1 +system sh/exec.sh -n dnode1 -s start + +sql connect + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 100 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +print == create database $dbName vgroups $vgroups +sql create database $dbName vgroups $vgroups + +#wait database ready +$loop_cnt = 0 +check_db_ready: +if $loop_cnt == 10 then + print ====> database not ready! + return -1 +endi +sql show databases +print ==> rows: $rows +print ==> $data(db)[0] $data(db)[1] $data(db)[2] $data(db)[3] $data(db)[4] $data(db)[5] $data(db)[6] $data(db)[7] $data(db)[8] $data(db)[9] $data(db)[10] $data(db)[11] $data(db)[12] +print $data(db)[13] $data(db)[14] $data(db)[15] $data(db)[16] $data(db)[17] $data(db)[18] $data(db)[19] $data(db)[20] +if $data(db)[20] != nostrict then + sleep 100 + $loop_cnt = $loop_cnt + 1 + goto check_db_ready +endi + +sql use $dbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi + +print == create super table +sql create table $stbPrefix (ts timestamp, c1 int, c2 float, c3 binary(16)) tags (t1 int) +sql show stables +if $rows != 1 then + return -1 +endi + +print == create child table, normal table and insert data +$i = 0 +while $i < $ctbNum + $ctb = $ctbPrefix . $i + $ntb = $ntbPrefix . $i + sql create table $ctb using $stbPrefix tags( $i ) + sql create table $ntb (ts timestamp, c1 int, c2 float, c3 binary(16)) + + $x = 0 + while $x < $rowsPerCtb + $binary = ' . binary- + $binary = $binary . $i + $binary = $binary . ' + + sql insert into $ctb values ($tstart , $i , $x , $binary ) + sql insert into $ntb values ($tstart , $i , $x , $binary ) + $tstart = $tstart + 1 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 20803f657a..65e4785b5d 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -1,3 +1,6 @@ +#!/bin/bash +set -e + #python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 50b4a0800f..dc1a77c231 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -14,14 +14,13 @@ */ #include -#include #include #include #include #include #include #include -#include +// #include #include "taos.h" #include "taoserror.h" @@ -227,7 +226,7 @@ int64_t getDirectorySize(char* dir) { } } - taosCloseDir(pDir); + taosCloseDir(&pDir); return totalSize; } @@ -595,8 +594,8 @@ void printParaIntoFile() { taosOpenFile(g_stConfInfo.resultFileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", g_stConfInfo.resultFileName); - exit - 1; - }; + exit(-1); + } g_fp = pFile; time_t tTime = taosGetTimestampSec(); diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index d3bed600dd..f9ba0e418c 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -14,14 +14,13 @@ */ #include -#include #include #include #include #include #include #include -#include +// #include #include "taos.h" #include "taoserror.h" @@ -31,46 +30,49 @@ #define NC "\033[0m" #define min(a, b) (((a) < (b)) ? (a) : (b)) -#define MAX_SQL_STR_LEN (1024 * 1024) -#define MAX_ROW_STR_LEN (16 * 1024) -#define MAX_CONSUMER_THREAD_CNT (16) +#define MAX_SQL_STR_LEN (1024 * 1024) +#define MAX_ROW_STR_LEN (16 * 1024) +#define MAX_CONSUMER_THREAD_CNT (16) typedef struct { - TdThread thread; - int32_t consumerId; + TdThread thread; + int32_t consumerId; - int32_t ifCheckData; - int64_t expectMsgCnt; + int32_t ifCheckData; + int64_t expectMsgCnt; + + int64_t consumeMsgCnt; + int64_t consumeRowCnt; + int32_t checkresult; - int64_t consumeMsgCnt; - int32_t checkresult; + char topicString[1024]; + char keyString[1024]; - char topicString[1024]; - char keyString[1024]; + int32_t numOfTopic; + char topics[32][64]; - int32_t numOfTopic; - char topics[32][64]; - - int32_t numOfKey; - char key[32][64]; - char value[32][64]; + int32_t numOfKey; + char key[32][64]; + char value[32][64]; tmq_t* tmq; tmq_list_t* topicList; - + } SThreadInfo; typedef struct { // input from argvs - char dbName[32]; - int32_t showMsgFlag; - int32_t consumeDelay; // unit s - int32_t numOfThread; - SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + char cdbName[32]; + char dbName[32]; + int32_t showMsgFlag; + int32_t showRowFlag; + int32_t consumeDelay; // unit s + int32_t numOfThread; + SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; } SConfInfo; static SConfInfo g_stConfInfo; -TdFilePtr g_fp = NULL; +TdFilePtr g_fp = NULL; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -85,51 +87,62 @@ static void printHelp() { printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default "); printf("%s%s\n", indent, "-g"); printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); printf("%s%s\n", indent, "-y"); printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); exit(EXIT_SUCCESS); } + void initLogFile() { // FILE *fp = fopen(g_stConfInfo.resultFileName, "a"); - TdFilePtr pFile = taosOpenFile("./tmqlog.txt", TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); + char file[256]; + sprintf(file, "%s/../log/tmqlog.txt", configDir); + TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND | TD_FILE_STREAM); if (NULL == pFile) { fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt"); - exit - 1; - }; + exit(-1); + } g_fp = pFile; +} + +void saveConfigToLogFile() { time_t tTime = taosGetTimestampSec(); struct tm tm = *taosLocalTime(&tTime, NULL); - taosFprintfFile(pFile, "###################################################################\n"); - taosFprintfFile(pFile, "# configDir: %s\n", configDir); - taosFprintfFile(pFile, "# dbName: %s\n", g_stConfInfo.dbName); - taosFprintfFile(pFile, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); - taosFprintfFile(pFile, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); + taosFprintfFile(g_fp, "###################################################################\n"); + taosFprintfFile(g_fp, "# configDir: %s\n", configDir); + taosFprintfFile(g_fp, "# dbName: %s\n", g_stConfInfo.dbName); + taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); + taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); + taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { - taosFprintfFile(pFile, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); - taosFprintfFile(pFile, " Topics: "); - for (int i = 0; i < g_stConfInfo.stThreads[i].numOfTopic; i++) { - taosFprintfFile(pFile, "%s, ", g_stConfInfo.stThreads[i].topics[i]); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId); + taosFprintfFile(g_fp, " Topics: "); + for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfTopic; i++) { + taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[i]); } - taosFprintfFile(pFile, "\n"); - taosFprintfFile(pFile, " Key: "); - for (int i = 0; i < g_stConfInfo.stThreads[i].numOfKey; i++) { - taosFprintfFile(pFile, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]); + taosFprintfFile(g_fp, "\n"); + taosFprintfFile(g_fp, " Key: "); + for (int i = 0 ; i < g_stConfInfo.stThreads[i].numOfKey; i++) { + taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[i], g_stConfInfo.stThreads[i].value[i]); } - taosFprintfFile(pFile, "\n"); + taosFprintfFile(g_fp, "\n"); } - - taosFprintfFile(pFile, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, - tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - taosFprintfFile(pFile, "###################################################################\n"); + + taosFprintfFile(g_fp, "# Test time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, + tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + taosFprintfFile(g_fp, "###################################################################\n"); } void parseArgument(int32_t argc, char* argv[]) { memset(&g_stConfInfo, 0, sizeof(SConfInfo)); g_stConfInfo.showMsgFlag = 0; + g_stConfInfo.showRowFlag = 0; g_stConfInfo.consumeDelay = 5; for (int32_t i = 1; i < argc; i++) { @@ -138,10 +151,14 @@ void parseArgument(int32_t argc, char* argv[]) { exit(0); } else if (strcmp(argv[i], "-d") == 0) { strcpy(g_stConfInfo.dbName, argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + strcpy(g_stConfInfo.cdbName, argv[++i]); } else if (strcmp(argv[i], "-c") == 0) { strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-g") == 0) { g_stConfInfo.showMsgFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + g_stConfInfo.showRowFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else { @@ -150,11 +167,17 @@ void parseArgument(int32_t argc, char* argv[]) { } } + initLogFile(); + + taosFprintfFile(g_fp, "====parseArgument() success\n"); + #if 1 pPrint("%s configDir:%s %s", GREEN, configDir, NC); pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC); + pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC); pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); + pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); #endif } @@ -180,24 +203,29 @@ void ltrim(char* str) { // return str; } -static int running = 1; -static void msg_process(TAOS_RES* msg, int32_t msgIndex, int32_t threadLable) { +static int running = 1; +static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { char buf[1024]; + int32_t totalRows = 0; - // printf("topic: %s\n", tmq_get_topic_name(msg)); - // printf("vg:%d\n", tmq_get_vgroup_id(msg)); - taosFprintfFile(g_fp, "msg index:%d, threadLable: %d\n", msgIndex, threadLable); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); - + //printf("topic: %s\n", tmq_get_topic_name(msg)); + //printf("vg:%d\n", tmq_get_vgroup_id(msg)); + taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; - TAOS_FIELD* fields = taos_fetch_fields(msg); - int32_t numOfFields = taos_field_count(msg); - // taos_print_row(buf, row, fields, numOfFields); - // printf("%s\n", buf); - // taosFprintfFile(g_fp, "%s\n", buf); + if (0 != g_stConfInfo.showRowFlag) { + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + taos_print_row(buf, row, fields, numOfFields); + taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + } + totalRows++; } + + return totalRows; } int queryDB(TAOS* taos, char* command) { @@ -213,31 +241,43 @@ int queryDB(TAOS* taos, char* command) { return 0; } -void build_consumer(SThreadInfo* pInfo) { - char sqlStr[1024] = {0}; - - TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - sprintf(sqlStr, "use %s", g_stConfInfo.dbName); - TAOS_RES* pRes = taos_query(pConn, sqlStr); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - exit(-1); - } - taos_free_result(pRes); +static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets) { + printf("tmq_commit_cb_print() commit %d\n", resp); +} +void build_consumer(SThreadInfo *pInfo) { tmq_conf_t* conf = tmq_conf_new(); - // tmq_conf_set(conf, "group.id", "tg2"); + + //tmq_conf_set(conf, "td.connect.ip", "localhost"); + //tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + + tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + + tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); + + // tmq_conf_set(conf, "group.id", "cgrp1"); for (int32_t i = 0; i < pInfo->numOfKey; i++) { tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]); } + + //tmq_conf_set(conf, "client.id", "c-001"); + + //tmq_conf_set(conf, "enable.auto.commit", "true"); + //tmq_conf_set(conf, "enable.auto.commit", "false"); + + //tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + + //tmq_conf_set(conf, "auto.offset.reset", "none"); + //tmq_conf_set(conf, "auto.offset.reset", "earliest"); + //tmq_conf_set(conf, "auto.offset.reset", "latest"); + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); return; } -void build_topic_list(SThreadInfo* pInfo) { +void build_topic_list(SThreadInfo *pInfo) { pInfo->topicList = tmq_list_new(); // tmq_list_append(topic_list, "test_stb_topic_1"); for (int32_t i = 0; i < pInfo->numOfTopic; i++) { @@ -246,45 +286,49 @@ void build_topic_list(SThreadInfo* pInfo) { return; } -int32_t saveConsumeResult(SThreadInfo* pInfo) { +int32_t saveConsumeResult(SThreadInfo *pInfo) { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %d)", g_stConfInfo.dbName, - pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->checkresult); - + sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", + g_stConfInfo.cdbName, + pInfo->consumerId, + pInfo->consumeMsgCnt, + pInfo->consumeRowCnt, + pInfo->checkresult); + TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); exit(-1); } - + taos_free_result(pRes); return 0; } -void loop_consume(SThreadInfo* pInfo) { +void loop_consume(SThreadInfo *pInfo) { tmq_resp_err_t err; - + int64_t totalMsgs = 0; - // int64_t totalRows = 0; + int64_t totalRows = 0; while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); - if (tmqMsg) { + if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { - msg_process(tmqMsg, totalMsgs, 0); + totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); } taos_free_result(tmqMsg); totalMsgs++; - + if (totalMsgs >= pInfo->expectMsgCnt) { break; } @@ -292,7 +336,7 @@ void loop_consume(SThreadInfo* pInfo) { break; } } - + err = tmq_consumer_close(pInfo->tmq); if (err) { printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); @@ -300,34 +344,38 @@ void loop_consume(SThreadInfo* pInfo) { } pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + + taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %"PRId64", consumeRowCnt: %"PRId64"\n", pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt); + } -void* consumeThreadFunc(void* param) { +void *consumeThreadFunc(void *param) { int32_t totalMsgs = 0; - SThreadInfo* pInfo = (SThreadInfo*)param; + SThreadInfo *pInfo = (SThreadInfo *)param; build_consumer(pInfo); build_topic_list(pInfo); - if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)){ return NULL; } - + tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); if (err) { printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); exit(-1); } - + loop_consume(pInfo); err = tmq_unsubscribe(pInfo->tmq); if (err) { printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); - pInfo->consumeMsgCnt = -1; + pInfo->consumeMsgCnt = -1; return NULL; - } - + } + // save consume result into consumeresult table saveConsumeResult(pInfo); @@ -339,7 +387,7 @@ void parseConsumeInfo() { const char delim[2] = ","; const char ch = ':'; - for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { token = strtok(g_stConfInfo.stThreads[i].topicString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -347,10 +395,10 @@ void parseConsumeInfo() { ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]); // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]); g_stConfInfo.stThreads[i].numOfTopic++; - + token = strtok(NULL, delim); } - + token = strtok(g_stConfInfo.stThreads[i].keyString, delim); while (token != NULL) { // printf("%s\n", token ); @@ -364,7 +412,7 @@ void parseConsumeInfo() { // g_stConfInfo.value[g_stConfInfo.numOfKey]); g_stConfInfo.stThreads[i].numOfKey++; } - + token = strtok(NULL, delim); } } @@ -372,47 +420,48 @@ void parseConsumeInfo() { int32_t getConsumeInfo() { char sqlStr[1024] = {0}; - + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); - - sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.dbName); + + sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName); TAOS_RES* pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); taos_free_result(pRes); exit(-1); - } - - TAOS_ROW row = NULL; - int num_fields = taos_num_fields(pRes); - TAOS_FIELD* fields = taos_fetch_fields(pRes); - - // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, - // ifcheckdata int - + } + + TAOS_ROW row = NULL; + int num_fields = taos_num_fields(pRes); + TAOS_FIELD* fields = taos_fetch_fields(pRes); + + // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int + int32_t numOfThread = 0; while ((row = taos_fetch_row(pRes))) { - int32_t* lengths = taos_fetch_lengths(pRes); - - for (int i = 0; i < num_fields; ++i) { + int32_t* lengths = taos_fetch_lengths(pRes); + + for (int i = 0; i < num_fields; ++i) { if (row[i] == NULL || 0 == i) { continue; } - + if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]); + g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t *)row[i]); } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]); } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) { memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]); } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) { - g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]); + g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t *)row[i]); } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) { - g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]); + g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t *)row[i]); } } - numOfThread++; + numOfThread ++; } g_stConfInfo.numOfThread = numOfThread; @@ -423,10 +472,11 @@ int32_t getConsumeInfo() { return 0; } + int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); getConsumeInfo(); - initLogFile(); + saveConfigToLogFile(); TdThreadAttr thattr; taosThreadAttrInit(&thattr); @@ -434,19 +484,18 @@ int main(int32_t argc, char* argv[]) { // pthread_create one thread to consume for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { - taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, - (void*)(&(g_stConfInfo.stThreads[i]))); + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc, (void *)(&(g_stConfInfo.stThreads[i]))); } for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); } - // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); - - taosFprintfFile(g_fp, "\n"); - taosCloseFile(&g_fp); - + //printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + + taosFprintfFile(g_fp, "\n"); + taosCloseFile(&g_fp); + return 0; } diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index ca5bf69795..12206ce290 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -29,7 +29,6 @@ #include "tutil.h" #include -#include /**************** Global variables ****************/ #ifdef _TD_POWER_ @@ -224,27 +223,63 @@ int32_t shellRunCommand(TAOS *con, char *command) { } } - char quote = 0, *cmd = command; + bool esc = false; + char quote = 0, *cmd = command, *p = command; for (char c = *command++; c != 0; c = *command++) { - if (c == '\\' && (*command == '\'' || *command == '"' || *command == '`')) { - command ++; + if (esc) { + switch (c) { + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'G': + *p++ = '\\'; + break; + case '\'': + case '"': + if (quote) { + *p++ = '\\'; + } + break; + } + *p++ = c; + esc = false; continue; } + if (c == '\\') { + if (quote != 0 && (*command == '_' || *command == '\\')) { + // DO nothing + } else { + esc = true; + continue; + } + } + if (quote == c) { quote = 0; - } else if (quote == 0 && (c == '\'' || c == '"' || c == '`')) { + } else if (quote == 0 && (c == '\'' || c == '"')) { quote = c; - } else if (c == ';' && quote == 0) { - c = *command; - *command = 0; + } + + *p++ = c; + if (c == ';' && quote == 0) { + c = *p; + *p = 0; if (shellRunSingleCommand(con, cmd) < 0) { return -1; } - *command = c; - cmd = command; + *p = c; + p = cmd; } } + + *p = 0; return shellRunSingleCommand(con, cmd); } @@ -538,23 +573,19 @@ static void shellPrintNChar(const char *str, int length, int width) { while (pos < length) { TdWchar wc; int bytes = taosMbToWchar(&wc, str + pos, MB_CUR_MAX); - if (bytes <= 0) { + if (bytes == 0) { break; } - if (pos + bytes > length) { - break; - } - int w = 0; -#ifdef WINDOWS - w = bytes; -#else - if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){ - w = bytes; - }else{ - w = taosWcharWidth(wc); - } -#endif pos += bytes; + if (pos > length) { + break; + } + +#ifdef WINDOWS + int w = bytes; +#else + int w = taosWcharWidth(wc); +#endif if (w <= 0) { continue; } @@ -648,7 +679,6 @@ static void printField(const char *val, TAOS_FIELD *field, int width, int32_t le break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON: shellPrintNChar(val, length, width); break; case TSDB_DATA_TYPE_TIMESTAMP: @@ -761,8 +791,7 @@ static int calcColWidth(TAOS_FIELD *field, int precision) { return TMAX(field->bytes, width); } - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON:{ + case TSDB_DATA_TYPE_NCHAR: { int16_t bytes = field->bytes * TSDB_NCHAR_SIZE; if (bytes > tsMaxBinaryDisplayWidth) { return TMAX(tsMaxBinaryDisplayWidth, width); diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 1ee42358f6..db73f2fc5c 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -18,9 +18,11 @@ #include "tglobal.h" #include "tlog.h" +#ifndef WINDOWS #include #include #include +#endif #define OPT_ABORT 1 /* abort */ @@ -37,6 +39,7 @@ TdThread pid; static tsem_t cancelSem; extern void taos_init(); +#ifndef WINDOWS static struct argp_option options[] = { {"host", 'h', "HOST", 0, "TDengine server FQDN to connect. The default host is localhost."}, {"password", 'p', NULL, 0, "The password to use when connecting to the server."}, @@ -213,7 +216,10 @@ static void parse_args(int argc, char *argv[], SShellArguments *arguments) { } } +#endif void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { +#ifdef WINDOWS +#else static char verType[32] = {0}; sprintf(verType, "version: %s\n", version); @@ -233,9 +239,12 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { abort(); #endif } +#endif } int32_t shellReadCommand(TAOS *con, char *command) { +#ifdef WINDOWS +#else unsigned hist_counter = history.hend; char utf8_array[10] = "\0"; Command cmd; @@ -390,6 +399,7 @@ int32_t shellReadCommand(TAOS *con, char *command) { } } +#endif return 0; } @@ -435,6 +445,8 @@ void *shellLoopQuery(void *arg) { void get_history_path(char *_history) { snprintf(_history, TSDB_FILENAME_LEN, "%s/%s", getenv("HOME"), HISTORY_FILE); } void clearScreen(int ecmd_pos, int cursor_pos) { +#ifdef WINDOWS +#else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { // fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); @@ -453,9 +465,12 @@ void clearScreen(int ecmd_pos, int cursor_pos) { fprintf(stdout, "\033[2K"); } fflush(stdout); +#endif } void showOnScreen(Command *cmd) { +#ifdef WINDOWS +#else struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { // fprintf(stderr, "No stream device\n"); @@ -526,6 +541,7 @@ void showOnScreen(Command *cmd) { positionCursor(cursor_x, DOWN); positionCursor(cursor_y, RIGHT); fflush(stdout); +#endif } void cleanup_handler(void *arg) { resetTerminalMode(); }