xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/db

This commit is contained in:
Shengliang Guan 2022-04-28 14:25:52 +08:00
parent 621290ab79 d5ae5fc667
commit 3cc90a8cec
290 changed files with 16241 additions and 15425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.devcontainer/Dockerfile
View File

@ -7,4 +7,4 @@ FROM mcr.microsoft.com/vscode/devcontainers/cpp:0-${VARIANT}
# [Optional] Uncomment this section to install additional packages. # [Optional] Uncomment this section to install additional packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here> # && apt-get -y install --no-install-recommends <your-package-list-here>
RUN apt-get update && apt-get -y install tree vim tmux RUN apt-get update && apt-get -y install tree vim tmux python3-pip

2
.gitignore vendored
View File

@ -40,6 +40,7 @@ taosdalipu/
Target/ Target/
*.failed *.failed
*.sql *.sql
sim
sim/ sim/
psim/ psim/
pysim/ pysim/
@ -107,3 +108,4 @@ TAGS
contrib/* contrib/*
!contrib/CMakeLists.txt !contrib/CMakeLists.txt
!contrib/test !contrib/test
sql

115
contrib/CMakeLists.txt
View File

@ -14,9 +14,24 @@ if(${BUILD_PTHREAD})
cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif() endif()
# gnu regex # iconv
if(${BUILD_GNUREGEX}) if(${BUILD_WITH_ICONV})
cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# msvc regex
if(${BUILD_MSVCREGEX})
cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wcwidth
if(${BUILD_WCWIDTH})
cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wingetopt
if(${BUILD_WINGETOPT})
cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif() endif()
# googletest # googletest
@ -95,12 +110,31 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
# ================================================================================================ # ================================================================================================
# googletest # googletest
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(googletest) add_subdirectory(googletest EXCLUDE_FROM_ALL)
target_include_directories( target_include_directories(
gtest gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
) )
if(${TD_WINDOWS})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
)
endif(${TD_WINDOWS})
if(${TD_LINUX})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
endif(${TD_LINUX})
if(${TD_DARWIN})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
)
endif(${TD_DARWIN})
endif(${BUILD_TEST}) endif(${BUILD_TEST})
# cJson # cJson
@ -109,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF) option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF)
option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON) option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON)
option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF) option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF)
add_subdirectory(cJson) add_subdirectory(cJson EXCLUDE_FROM_ALL)
target_include_directories( target_include_directories(
cjson cjson
# see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source
@ -118,7 +152,7 @@ target_include_directories(
unset(CMAKE_PROJECT_INCLUDE_BEFORE) unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# lz4 # lz4
add_subdirectory(lz4/build/cmake) add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL)
target_include_directories( target_include_directories(
lz4_static lz4_static
PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib
@ -126,7 +160,7 @@ target_include_directories(
# zlib # zlib
set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in")
add_subdirectory(zlib) add_subdirectory(zlib EXCLUDE_FROM_ALL)
target_include_directories( target_include_directories(
zlibstatic zlibstatic
PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib
@ -142,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE)
# leveldb # leveldb
if(${BUILD_WITH_LEVELDB}) if(${BUILD_WITH_LEVELDB})
option(LEVELDB_BUILD_TESTS "" OFF) option(LEVELDB_BUILD_TESTS "" OFF)
add_subdirectory(leveldb) add_subdirectory(leveldb EXCLUDE_FROM_ALL)
target_include_directories( target_include_directories(
leveldb leveldb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/leveldb/include>
@ -158,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB})
option(WITH_TOOLS "" OFF) option(WITH_TOOLS "" OFF)
option(WITH_LIBURING "" OFF) option(WITH_LIBURING "" OFF)
option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF)
add_subdirectory(rocksdb) add_subdirectory(rocksdb EXCLUDE_FROM_ALL)
target_include_directories( target_include_directories(
rocksdb rocksdb
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/include>
@ -169,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB})
# To support build on ubuntu: sudo apt-get install libboost-all-dev # To support build on ubuntu: sudo apt-get install libboost-all-dev
if(${BUILD_WITH_LUCENE}) if(${BUILD_WITH_LUCENE})
option(ENABLE_TEST "Enable the tests" OFF) option(ENABLE_TEST "Enable the tests" OFF)
add_subdirectory(lucene) add_subdirectory(lucene EXCLUDE_FROM_ALL)
target_include_directories( target_include_directories(
lucene++ lucene++
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lucene/include>
@ -179,9 +213,56 @@ endif(${BUILD_WITH_LUCENE})
# NuRaft # NuRaft
if(${BUILD_WITH_NURAFT}) if(${BUILD_WITH_NURAFT})
add_subdirectory(nuraft) add_subdirectory(nuraft EXCLUDE_FROM_ALL)
endif(${BUILD_WITH_NURAFT}) endif(${BUILD_WITH_NURAFT})
# pthread
if(${BUILD_PTHREAD})
set(CMAKE_BUILD_TYPE release)
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
add_library(pthread STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
endif()
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv EXCLUDE_FROM_ALL)
endif(${BUILD_WITH_ICONV})
# wingetopt
if(${BUILD_WINGETOPT})
add_subdirectory(wingetopt EXCLUDE_FROM_ALL)
endif(${BUILD_WINGETOPT})
# msvcregex
if(${BUILD_MSVCREGEX})
add_library(msvcregex STATIC "")
target_sources(msvcregex
PRIVATE "msvcregex/regex.c"
)
target_include_directories(msvcregex
PRIVATE "msvcregex"
)
target_link_libraries(msvcregex
INTERFACE Shell32
)
SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
endif(${BUILD_MSVCREGEX})
# msvcregex
if(${BUILD_WCWIDTH})
add_library(wcwidth STATIC "")
target_sources(wcwidth
PRIVATE "wcwidth/wcwidth.c"
)
target_include_directories(wcwidth
PRIVATE "wcwidth"
)
SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
endif(${BUILD_WCWIDTH})
# CRAFT # CRAFT
if(${BUILD_WITH_CRAFT}) if(${BUILD_WITH_CRAFT})
add_library(craft STATIC IMPORTED GLOBAL) add_library(craft STATIC IMPORTED GLOBAL)
@ -212,7 +293,7 @@ if(${BUILD_WITH_UV})
MESSAGE("Windows need set no-sign-compare") MESSAGE("Windows need set no-sign-compare")
add_compile_options(-Wno-sign-compare) add_compile_options(-Wno-sign-compare)
endif () endif ()
add_subdirectory(libuv) add_subdirectory(libuv EXCLUDE_FROM_ALL)
endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV})
# BDB # BDB
@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE})
target_link_libraries(sqlite target_link_libraries(sqlite
INTERFACE m INTERFACE m
INTERFACE pthread INTERFACE pthread
INTERFACE dl
) )
if(NOT TD_WINDOWS)
target_link_libraries(sqlite
INTERFACE dl
)
endif(NOT TD_WINDOWS)
endif(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE})
# pthread # pthread
@ -249,5 +334,5 @@ endif(${BUILD_WITH_SQLITE})
# Build test # Build test
# ================================================================================================ # ================================================================================================
if(${BUILD_DEPENDENCY_TESTS}) if(${BUILD_DEPENDENCY_TESTS})
add_subdirectory(test) add_subdirectory(test EXCLUDE_FROM_ALL)
endif(${BUILD_DEPENDENCY_TESTS}) endif(${BUILD_DEPENDENCY_TESTS})

9
contrib/test/craft/CMakeLists.txt
View File

@ -1,2 +1,9 @@
add_executable(simulate_vnode "simulate_vnode.c") add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
if(${BUILD_WINGETOPT})
target_link_libraries(simulate_vnode PUBLIC wingetopt)
target_include_directories(
simulate_vnode
PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
)
endif()

72
contrib/test/tdev/src/main.c
View File

@ -6,43 +6,39 @@
#define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
#define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
#define tPutA(buf, val) \ static inline void tPutA(void **buf, uint64_t val) {
({ \ memcpy(buf, &val, sizeof(val));
memcpy(buf, &val, sizeof(val)); \ *buf = POINTER_SHIFT(buf, sizeof(val));
POINTER_SHIFT(buf, sizeof(val)); \ }
})
#define tPutB(buf, val) \ static inline void tPutB(void **buf, uint64_t val) {
({ \ ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \ ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \ ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \ ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \ ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ ((uint8_t *)buf)[0] = (val)&0xff;
((uint8_t *)buf)[0] = (val)&0xff; \ *buf = POINTER_SHIFT(buf, sizeof(val));
POINTER_SHIFT(buf, sizeof(val)); \ }
})
#define tPutC(buf, val) \ static inline void tPutC(void **buf, uint64_t val) {
({ \ if (buf) {
if (buf) { \ ((uint64_t *)buf)[0] = (val);
((uint64_t *)buf)[0] = (val); \ POINTER_SHIFT(buf, sizeof(val));
POINTER_SHIFT(buf, sizeof(val)); \ }
} \ *buf = NULL;
NULL; \ }
})
#define tPutD(buf, val) \ static inline void tPutD(void **buf, uint64_t val) {
({ \ uint64_t tmp = val;
uint64_t tmp = val; \ for (size_t i = 0; i < sizeof(val); i++) {
for (size_t i = 0; i < sizeof(val); i++) { \ ((uint8_t *)buf)[i] = tmp & 0xff;
((uint8_t *)buf)[i] = tmp & 0xff; \ tmp >>= 8;
tmp >>= 8; \ }
} \ *buf = POINTER_SHIFT(buf, sizeof(val));
POINTER_SHIFT(buf, sizeof(val)); \ }
})
static inline void tPutE(void **buf, uint64_t val) { static inline void tPutE(void **buf, uint64_t val) {
if (buf) { if (buf) {
@ -61,7 +57,7 @@ static void func(T t) {
switch (t) { switch (t) {
case A: case A:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutA(pBuf, val); tPutA(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }
@ -69,7 +65,7 @@ static void func(T t) {
break; break;
case B: case B:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutB(pBuf, val); tPutB(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }
@ -77,7 +73,7 @@ static void func(T t) {
break; break;
case C: case C:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutC(pBuf, val); tPutC(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }
@ -85,7 +81,7 @@ static void func(T t) {
break; break;
case D: case D:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutD(pBuf, val); tPutD(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }

1
example/CMakeLists.txt
View File

@ -30,6 +30,7 @@ target_link_libraries(demoapi
) )
target_include_directories(tmq target_include_directories(tmq
PUBLIC "${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )

2
example/src/tmq.c
View File

@ -17,7 +17,6 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <time.h> #include <time.h>
#include <unistd.h>
#include "taos.h" #include "taos.h"
static int running = 1; static int running = 1;
@ -48,7 +47,6 @@ int32_t init_env() {
return -1; return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
sleep(1);
pRes = taos_query(pConn, "use abc1"); pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {

50
include/client/taos.h
View File

@ -27,10 +27,7 @@ typedef void TAOS;
typedef void TAOS_STMT; typedef void TAOS_STMT;
typedef void TAOS_RES; typedef void TAOS_RES;
typedef void **TAOS_ROW; typedef void **TAOS_ROW;
#if 0 typedef void TAOS_SUB;
typedef void TAOS_STREAM;
#endif
typedef void TAOS_SUB;
// Data type definition // Data type definition
#define TSDB_DATA_TYPE_NULL 0 // 1 bytes #define TSDB_DATA_TYPE_NULL 0 // 1 bytes
@ -88,18 +85,22 @@ typedef struct taosField {
int32_t bytes; int32_t bytes;
} TAOS_FIELD; } TAOS_FIELD;
#define DLL_EXPORT #ifdef WINDOWS
#define DLL_EXPORT __declspec(dllexport)
#else
#define DLL_EXPORT
#endif
typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code);
typedef struct TAOS_BIND_v2 { typedef struct TAOS_MULTI_BIND {
int buffer_type; int buffer_type;
void *buffer; void *buffer;
int32_t buffer_length; uintptr_t buffer_length;
int32_t *length; int32_t *length;
char *is_null; char *is_null;
int num; int num;
} TAOS_BIND_v2; } TAOS_MULTI_BIND;
typedef enum { typedef enum {
SET_CONF_RET_SUCC = 0, SET_CONF_RET_SUCC = 0,
@ -120,26 +121,27 @@ typedef struct setConfRet {
DLL_EXPORT void taos_cleanup(void); DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT setConfRet taos_set_config(const char *config);
DLL_EXPORT int taos_init(void);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen,
const char *db, int dbLen, uint16_t port); const char *db, int dbLen, uint16_t port);
DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port);
DLL_EXPORT void taos_close(TAOS *taos); DLL_EXPORT void taos_close(TAOS *taos);
const char *taos_data_type(int type); const char *taos_data_type(int type);
DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags); DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags);
DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);
DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);
DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes);
DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind); DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind);
DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx); DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx);
DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt);
DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt);
DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt);
@ -192,12 +194,6 @@ DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub);
DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress);
#endif #endif
#if 0
DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),
int64_t stime, void *param, void (*callback)(void *));
DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
#endif
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList);
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
@ -234,15 +230,11 @@ DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errst
DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
/* ------------------------TMQ CONSUMER INTERFACE------------------------ */ /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t wait_time);
DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq);
#if 0
DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups);
DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups);
#endif
DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async); DLL_EXPORT tmq_resp_err_t tmq_commit(tmq_t *tmq, const tmq_topic_vgroup_list_t *offsets, int32_t async);
#if 0 #if 0
DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async); DLL_EXPORT tmq_resp_err_t tmq_commit_message(tmq_t* tmq, const tmq_message_t* tmqmessage, int32_t async);
@ -269,7 +261,7 @@ DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res);
DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res);
// TODO // TODO
#if 0 #if 0
DLL_EXPORT char *tmq_get_block_table_name(TAOS_RES *res); DLL_EXPORT char *tmq_get_table_name(TAOS_RES *res);
#endif #endif
#if 0 #if 0

2
include/common/taosdef.h
View File

@ -82,6 +82,8 @@ extern char *qtypeStr[];
#define TSDB_PORT_HTTP 11 #define TSDB_PORT_HTTP 11
#undef TD_DEBUG_PRINT_ROW
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

99
include/common/tcommon.h
View File

@ -66,13 +66,13 @@ typedef struct SDataBlockInfo {
int32_t rows; int32_t rows;
int32_t rowSize; int32_t rowSize;
union { union {
int64_t uid; // from which table of uid, comes from this data block int64_t uid; // from which table of uid, comes from this data block
int64_t blockId; int64_t blockId;
}; };
uint64_t groupId; // no need to serialize uint64_t groupId; // no need to serialize
int16_t numOfCols; int16_t numOfCols;
int16_t hasVarCol; int16_t hasVarCol;
int16_t capacity; int16_t capacity;
} SDataBlockInfo; } SDataBlockInfo;
typedef struct SSDataBlock { typedef struct SSDataBlock {
@ -99,6 +99,15 @@ typedef struct SColumnInfoData {
}; };
} SColumnInfoData; } SColumnInfoData;
typedef struct SQueryTableDataCond {
STimeWindow twindow;
int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo *colList;
bool loadExternalRows; // load external rows or not
int32_t type; // data block load type:
} SQueryTableDataCond;
void* blockDataDestroy(SSDataBlock* pBlock); void* blockDataDestroy(SSDataBlock* pBlock);
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
@ -122,59 +131,6 @@ static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) {
static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); } static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); }
static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
int32_t tlen = 0;
int32_t sz = 0;
// tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics);
if (pRsp->numOfTopics == 0) return tlen;
tlen += taosEncodeSSchemaWrapper(buf, pRsp->schema);
if (pRsp->pBlockData) {
sz = taosArrayGetSize(pRsp->pBlockData);
}
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i);
tlen += tEncodeDataBlock(buf, pBlock);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
int32_t sz;
// buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics);
if (pRsp->numOfTopics == 0) return buf;
pRsp->schema = (SSchemaWrapper*)taosMemoryCalloc(1, sizeof(SSchemaWrapper));
if (pRsp->schema == NULL) return NULL;
buf = taosDecodeSSchemaWrapper(buf, pRsp->schema);
buf = taosDecodeFixedI32(buf, &sz);
pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock));
for (int32_t i = 0; i < sz; i++) {
SSDataBlock block = {0};
tDecodeDataBlock(buf, &block);
taosArrayPush(pRsp->pBlockData, &block);
}
return buf;
}
static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
if (pRsp->schema) {
if (pRsp->schema->nCols) {
taosMemoryFreeClear(pRsp->schema->pSchema);
}
taosMemoryFree(pRsp->schema);
}
taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner);
pRsp->pBlockData = NULL;
}
//====================================================================================================================== //======================================================================================================================
// the following structure shared by parser and executor // the following structure shared by parser and executor
typedef struct SColumn { typedef struct SColumn {
@ -195,22 +151,22 @@ typedef struct SColumn {
} SColumn; } SColumn;
typedef struct STableBlockDistInfo { typedef struct STableBlockDistInfo {
uint16_t rowSize; uint16_t rowSize;
uint16_t numOfFiles; uint16_t numOfFiles;
uint32_t numOfTables; uint32_t numOfTables;
uint64_t totalSize; uint64_t totalSize;
uint64_t totalRows; uint64_t totalRows;
int32_t maxRows; int32_t maxRows;
int32_t minRows; int32_t minRows;
int32_t firstSeekTimeUs; int32_t firstSeekTimeUs;
uint32_t numOfRowsInMemTable; uint32_t numOfRowsInMemTable;
uint32_t numOfSmallBlocks; uint32_t numOfSmallBlocks;
SArray *dataBlockInfos; SArray* dataBlockInfos;
} STableBlockDistInfo; } STableBlockDistInfo;
enum { enum {
FUNC_PARAM_TYPE_VALUE = 0x1, FUNC_PARAM_TYPE_VALUE = 0x1,
FUNC_PARAM_TYPE_COLUMN= 0x2, FUNC_PARAM_TYPE_COLUMN = 0x2,
}; };
typedef struct SFunctParam { typedef struct SFunctParam {
@ -229,7 +185,6 @@ typedef struct SResSchame {
char name[TSDB_COL_NAME_LEN]; char name[TSDB_COL_NAME_LEN];
} SResSchema; } SResSchema;
// TODO move away to executor.h
typedef struct SExprBasicInfo { typedef struct SExprBasicInfo {
SResSchema resSchema; SResSchema resSchema;
int16_t numOfParams; // argument value of each function int16_t numOfParams; // argument value of each function
@ -241,7 +196,7 @@ typedef struct SExprInfo {
struct tExprNode* pExpr; struct tExprNode* pExpr;
} SExprInfo; } SExprInfo;
#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_ASC_FORWARD_STEP 1
#define QUERY_DESC_FORWARD_STEP -1 #define QUERY_DESC_FORWARD_STEP -1
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)

374
include/common/tmsg.h
View File

@ -177,6 +177,7 @@ typedef struct SField {
char name[TSDB_COL_NAME_LEN]; char name[TSDB_COL_NAME_LEN];
uint8_t type; uint8_t type;
int32_t bytes; int32_t bytes;
int8_t flags;
} SField; } SField;
typedef struct SRetention { typedef struct SRetention {
@ -230,12 +231,12 @@ typedef struct {
int32_t totalLen; int32_t totalLen;
int32_t len; int32_t len;
// head of SSubmitBlk // head of SSubmitBlk
// int64_t uid; // table unique id int64_t uid; // table unique id
// int64_t suid; // stable id int64_t suid; // stable id
// int32_t sversion; // data schema version int32_t sversion; // data schema version
// int32_t dataLen; // data part length, not including the SSubmitBlk head int32_t dataLen; // data part length, not including the SSubmitBlk head
// int32_t schemaLen; // schema length, if length is 0, no schema exists int32_t schemaLen; // schema length, if length is 0, no schema exists
// int16_t numOfRows; // total number of rows in current submit block int16_t numOfRows; // total number of rows in current submit block
// head of SSubmitBlk // head of SSubmitBlk
const void* pMsg; const void* pMsg;
} SSubmitMsgIter; } SSubmitMsgIter;
@ -249,10 +250,10 @@ STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts // 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts
// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later // 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later
// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter // 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter
// int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter);
// int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
// int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
// STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter);
typedef struct { typedef struct {
int32_t index; // index of failed block in submit blocks int32_t index; // index of failed block in submit blocks
@ -286,24 +287,27 @@ typedef struct SSchema {
#define SSCHMEA_BYTES(s) ((s)->bytes) #define SSCHMEA_BYTES(s) ((s)->bytes)
#define SSCHMEA_NAME(s) ((s)->name) #define SSCHMEA_NAME(s) ((s)->name)
typedef struct {
int32_t nCols;
int32_t sver;
SSchema* pSchema;
} SSchemaWrapper;
STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols);
typedef struct { typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; char name[TSDB_TABLE_FNAME_LEN];
int8_t igExists; int8_t igExists;
float xFilesFactor; float xFilesFactor;
int32_t aggregationMethod;
int32_t delay; int32_t delay;
int32_t ttl; int32_t ttl;
int32_t numOfColumns; int32_t numOfColumns;
int32_t numOfTags; int32_t numOfTags;
int32_t numOfSmas;
int32_t commentLen; int32_t commentLen;
int32_t ast1Len; int32_t ast1Len;
int32_t ast2Len; int32_t ast2Len;
SArray* pColumns; // array of SField SArray* pColumns; // array of SField
SArray* pTags; // array of SField SArray* pTags; // array of SField
SArray* pSmas; // array of SField
char* comment; char* comment;
char* pAst1; char* pAst1;
char* pAst2; char* pAst2;
@ -434,6 +438,7 @@ typedef struct {
int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
typedef struct { typedef struct {
int16_t colId; // column id int16_t colId; // column id
@ -1342,12 +1347,12 @@ int32_t tDeserializeSCMCreateTopicRsp(void* buf, int32_t bufLen, SCMCreateTopicR
typedef struct { typedef struct {
int64_t consumerId; int64_t consumerId;
} SMqConsumerLostMsg; } SMqConsumerLostMsg, SMqConsumerRecoverMsg;
typedef struct { typedef struct {
int64_t consumerId; int64_t consumerId;
char cgroup[TSDB_CGROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
SArray* topicNames; // SArray<char*> SArray* topicNames; // SArray<char**>
} SCMSubscribeReq; } SCMSubscribeReq;
static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
@ -1522,69 +1527,93 @@ typedef struct {
} SDDropTopicReq; } SDDropTopicReq;
typedef struct { typedef struct {
float xFilesFactor; float xFilesFactor;
int32_t delay; int32_t delay;
int32_t qmsg1Len; int32_t qmsg1Len;
int32_t qmsg2Len; int32_t qmsg2Len;
func_id_t* pFuncIds; char* qmsg1; // pAst1:qmsg1:SRetention1 => trigger aggr task1
char* qmsg1; // pAst1:qmsg1:SRetention1 => trigger aggr task1 char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2
char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2
int8_t nFuncIds;
} SRSmaParam; } SRSmaParam;
int32_t tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam);
int32_t tDecodeSRSmaParam(SCoder* pCoder, SRSmaParam* pRSmaParam);
typedef struct SVCreateStbReq {
const char* name;
tb_uid_t suid;
int8_t rollup;
SSchemaWrapper schema;
SSchemaWrapper schemaTag;
SRSmaParam pRSmaParam;
} SVCreateStbReq;
int tEncodeSVCreateStbReq(SCoder* pCoder, const SVCreateStbReq* pReq);
int tDecodeSVCreateStbReq(SCoder* pCoder, SVCreateStbReq* pReq);
typedef struct SVDropStbReq {
// data
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
} SVDropStbReq;
typedef struct SVCreateStbRsp {
int code;
} SVCreateStbRsp;
typedef struct SVCreateTbReq { typedef struct SVCreateTbReq {
char* name; tb_uid_t uid;
uint32_t ttl; int64_t ctime;
uint32_t keep; const char* name;
union { int32_t ttl;
uint8_t info; int8_t type;
struct {
uint8_t rollup : 1; // 1 means rollup sma
uint8_t type : 7;
};
};
union { union {
struct { struct {
tb_uid_t suid; tb_uid_t suid;
col_id_t nCols; const void* pTag;
col_id_t nBSmaCols; } ctb;
SSchema* pSchema;
col_id_t nTagCols;
SSchema* pTagSchema;
SRSmaParam* pRSmaParam;
} stbCfg;
struct { struct {
tb_uid_t suid; SSchemaWrapper schema;
SKVRow pTag; } ntb;
} ctbCfg;
struct {
col_id_t nCols;
col_id_t nBSmaCols;
SSchema* pSchema;
SRSmaParam* pRSmaParam;
} ntbCfg;
}; };
} SVCreateTbReq, SVUpdateTbReq; } SVCreateTbReq;
int tEncodeSVCreateTbReq(SCoder* pCoder, const SVCreateTbReq* pReq);
int tDecodeSVCreateTbReq(SCoder* pCoder, SVCreateTbReq* pReq);
typedef struct {
int32_t nReqs;
union {
SVCreateTbReq* pReqs;
SArray* pArray;
};
} SVCreateTbBatchReq;
int tEncodeSVCreateTbBatchReq(SCoder* pCoder, const SVCreateTbBatchReq* pReq);
int tDecodeSVCreateTbBatchReq(SCoder* pCoder, SVCreateTbBatchReq* pReq);
typedef struct { typedef struct {
int32_t code; int32_t code;
} SVCreateTbRsp, SVUpdateTbRsp; } SVCreateTbRsp, SVUpdateTbRsp;
int tEncodeSVCreateTbRsp(SCoder* pCoder, const SVCreateTbRsp* pRsp);
int tDecodeSVCreateTbRsp(SCoder* pCoder, SVCreateTbRsp* pRsp);
int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq); int32_t tSerializeSVCreateTbReq(void** buf, SVCreateTbReq* pReq);
void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq); void* tDeserializeSVCreateTbReq(void* buf, SVCreateTbReq* pReq);
typedef struct { typedef struct {
int64_t ver; // use a general definition int32_t nRsps;
SArray* pArray; union {
} SVCreateTbBatchReq; SVCreateTbRsp* pRsps;
SArray* pArray;
int32_t tSerializeSVCreateTbBatchReq(void** buf, SVCreateTbBatchReq* pReq); };
void* tDeserializeSVCreateTbBatchReq(void* buf, SVCreateTbBatchReq* pReq);
typedef struct {
SArray* rspList; // SArray<SVCreateTbRsp>
} SVCreateTbBatchRsp; } SVCreateTbBatchRsp;
int tEncodeSVCreateTbBatchRsp(SCoder* pCoder, const SVCreateTbBatchRsp* pRsp);
int tDecodeSVCreateTbBatchRsp(SCoder* pCoder, SVCreateTbBatchRsp* pRsp);
int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); int32_t tSerializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp); int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatchRsp* pRsp);
@ -1639,103 +1668,11 @@ typedef struct {
char data[]; char data[];
} SVShowTablesFetchRsp; } SVShowTablesFetchRsp;
typedef struct SMqCMGetSubEpReq { typedef struct {
int64_t consumerId; int64_t consumerId;
int32_t epoch; int32_t epoch;
char cgroup[TSDB_CGROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
} SMqCMGetSubEpReq; } SMqAskEpReq;
static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pMsg->contLen);
tlen += taosEncodeFixedI32(buf, pMsg->vgId);
return tlen;
}
typedef struct SMqHbRsp {
int8_t status; // idle or not
int8_t vnodeChanged;
int8_t epChanged; // should use new epset
int8_t reserved;
SEpSet epSet;
} SMqHbRsp;
static FORCE_INLINE int32_t taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pRsp->status);
tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged);
tlen += taosEncodeFixedI8(buf, pRsp->epChanged);
tlen += taosEncodeSEpSet(buf, &pRsp->epSet);
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) {
buf = taosDecodeFixedI8(buf, &pRsp->status);
buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged);
buf = taosDecodeFixedI8(buf, &pRsp->epChanged);
buf = taosDecodeSEpSet(buf, &pRsp->epSet);
return buf;
}
typedef struct SMqHbOneTopicBatchRsp {
char topicName[TSDB_TOPIC_FNAME_LEN];
SArray* rsps; // SArray<SMqHbRsp>
} SMqHbOneTopicBatchRsp;
static FORCE_INLINE int32_t taosEncodeSMqHbOneTopicBatchRsp(void** buf, const SMqHbOneTopicBatchRsp* pBatchRsp) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pBatchRsp->topicName);
int32_t sz = taosArrayGetSize(pBatchRsp->rsps);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbRsp* pRsp = (SMqHbRsp*)taosArrayGet(pBatchRsp->rsps, i);
tlen += taosEncodeSMqHbRsp(buf, pRsp);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTopicBatchRsp* pBatchRsp) {
int32_t sz;
buf = taosDecodeStringTo(buf, pBatchRsp->topicName);
buf = taosDecodeFixedI32(buf, &sz);
pBatchRsp->rsps = taosArrayInit(sz, sizeof(SMqHbRsp));
for (int32_t i = 0; i < sz; i++) {
SMqHbRsp rsp;
buf = taosDecodeSMqHbRsp(buf, &rsp);
buf = taosArrayPush(pBatchRsp->rsps, &rsp);
}
return buf;
}
typedef struct SMqHbBatchRsp {
int64_t consumerId;
SArray* batchRsps; // SArray<SMqHbOneTopicBatchRsp>
} SMqHbBatchRsp;
static FORCE_INLINE int32_t taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pBatchRsp->consumerId);
int32_t sz;
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*)taosArrayGet(pBatchRsp->batchRsps, i);
tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp);
}
return tlen;
}
static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBatchRsp) {
buf = taosDecodeFixedI64(buf, &pBatchRsp->consumerId);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp));
for (int32_t i = 0; i < sz; i++) {
SMqHbOneTopicBatchRsp rsp;
buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp);
buf = taosArrayPush(pBatchRsp->batchRsps, &rsp);
}
return buf;
}
typedef struct { typedef struct {
int32_t key; int32_t key;
@ -2104,15 +2041,11 @@ int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset);
int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq);
int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq);
typedef struct {
uint32_t nCols;
SSchema* pSchema;
} SSchemaWrapper;
static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
if (pSW == NULL) return pSW; if (pSW == NULL) return pSW;
pSW->nCols = pSchemaWrapper->nCols; pSW->nCols = pSchemaWrapper->nCols;
pSW->sver = pSchemaWrapper->sver;
pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
if (pSW->pSchema == NULL) { if (pSW->pSchema == NULL) {
taosMemoryFree(pSW); taosMemoryFree(pSW);
@ -2149,8 +2082,8 @@ static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) {
static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1;
if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1; if (tEncodeI8(pEncoder, pSchema->flags) < 0) return -1;
if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1; if (tEncodeI32v(pEncoder, pSchema->bytes) < 0) return -1;
if (tEncodeI16(pEncoder, pSchema->colId) < 0) return -1; if (tEncodeI16v(pEncoder, pSchema->colId) < 0) return -1;
if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1;
return 0; return 0;
} }
@ -2158,15 +2091,16 @@ static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSch
static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) {
if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1;
if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1; if (tDecodeI8(pDecoder, &pSchema->flags) < 0) return -1;
if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1; if (tDecodeI32v(pDecoder, &pSchema->bytes) < 0) return -1;
if (tDecodeI16(pDecoder, &pSchema->colId) < 0) return -1; if (tDecodeI16v(pDecoder, &pSchema->colId) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1;
return 0; return 0;
} }
static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedU32(buf, pSW->nCols); tlen += taosEncodeVariantI32(buf, pSW->nCols);
tlen += taosEncodeVariantI32(buf, pSW->sver);
for (int32_t i = 0; i < pSW->nCols; i++) { for (int32_t i = 0; i < pSW->nCols; i++) {
tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]);
} }
@ -2174,7 +2108,8 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr
} }
static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) { static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) {
buf = taosDecodeFixedU32(buf, &pSW->nCols); buf = taosDecodeVariantI32(buf, &pSW->nCols);
buf = taosDecodeVariantI32(buf, &pSW->sver);
pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
if (pSW->pSchema == NULL) { if (pSW->pSchema == NULL) {
return NULL; return NULL;
@ -2187,23 +2122,25 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapp
} }
static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) { static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) {
if (tEncodeU32(pEncoder, pSW->nCols) < 0) return -1; if (tEncodeI32v(pEncoder, pSW->nCols) < 0) return -1;
if (tEncodeI32v(pEncoder, pSW->sver) < 0) return -1;
for (int32_t i = 0; i < pSW->nCols; i++) { for (int32_t i = 0; i < pSW->nCols; i++) {
if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1; if (tEncodeSSchema(pEncoder, &pSW->pSchema[i]) < 0) return -1;
} }
return pEncoder->pos;
return 0;
} }
static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapper* pSW) { static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapper* pSW) {
if (tDecodeU32(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1;
void* ptr = taosMemoryRealloc(pSW->pSchema, pSW->nCols * sizeof(SSchema)); if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1;
if (ptr == NULL) {
return -1; pSW->pSchema = (SSchema*)tCoderMalloc(pDecoder, sizeof(SSchema) * pSW->nCols);
} if (pSW->pSchema == NULL) return -1;
pSW->pSchema = (SSchema*)ptr;
for (int32_t i = 0; i < pSW->nCols; i++) { for (int32_t i = 0; i < pSW->nCols; i++) {
if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1;
} }
return 0; return 0;
} }
@ -2459,31 +2396,15 @@ typedef struct {
int64_t consumerId; int64_t consumerId;
} SMqRspHead; } SMqRspHead;
#if 0
typedef struct {
SMsgHead head;
int64_t consumerId;
int64_t blockingTime;
int32_t epoch;
int8_t withSchema;
char cgroup[TSDB_CGROUP_LEN];
int64_t currentOffset;
uint64_t reqId;
char topic[TSDB_TOPIC_FNAME_LEN];
} SMqPollReq;
#endif
typedef struct { typedef struct {
SMsgHead head; SMsgHead head;
char subKey[TSDB_SUBSCRIBE_KEY_LEN]; char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int32_t epoch; int32_t epoch;
uint64_t reqId; uint64_t reqId;
int64_t consumerId; int64_t consumerId;
int64_t blockingTime; int64_t waitTime;
int64_t currentOffset; int64_t currentOffset;
} SMqPollReqV2; } SMqPollReq;
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
@ -2498,65 +2419,6 @@ typedef struct {
SSchemaWrapper schema; SSchemaWrapper schema;
} SMqSubTopicEp; } SMqSubTopicEp;
typedef struct {
SMqRspHead head;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
// TODO: replace with topic name
int32_t numOfTopics;
// TODO: remove from msg
SSchemaWrapper* schema;
SArray* pBlockData; // SArray<SSDataBlock>
} SMqPollRsp;
typedef struct {
SMqRspHead head;
int64_t reqOffset;
int64_t rspOffset;
int32_t skipLogNum;
int32_t dataLen;
SArray* blockPos; // beginning pos for each SRetrieveTableRsp
void* blockData; // serialized batched SRetrieveTableRsp
} SMqPollRspV2;
static FORCE_INLINE int32_t tEncodeSMqPollRspV2(void** buf, const SMqPollRspV2* pRsp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pRsp->reqOffset);
tlen += taosEncodeFixedI64(buf, pRsp->rspOffset);
tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum);
tlen += taosEncodeFixedI32(buf, pRsp->dataLen);
if (pRsp->dataLen != 0) {
int32_t sz = taosArrayGetSize(pRsp->blockPos);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
int32_t blockPos = *(int32_t*)taosArrayGet(pRsp->blockPos, i);
tlen += taosEncodeFixedI32(buf, blockPos);
}
tlen += taosEncodeBinary(buf, pRsp->blockData, pRsp->dataLen);
}
return tlen;
}
static FORCE_INLINE void* tDecodeSMqPollRspV2(const void* buf, SMqPollRspV2* pRsp) {
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
buf = taosDecodeFixedI64(buf, &pRsp->rspOffset);
buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum);
buf = taosDecodeFixedI32(buf, &pRsp->dataLen);
if (pRsp->dataLen != 0) {
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pRsp->blockPos = taosArrayInit(sz, sizeof(int32_t));
for (int32_t i = 0; i < sz; i++) {
int32_t blockPos;
buf = taosDecodeFixedI32(buf, &blockPos);
taosArrayPush(pRsp->blockPos, &blockPos);
}
buf = taosDecodeBinary(buf, &pRsp->blockData, pRsp->dataLen);
}
return (void*)buf;
}
typedef struct { typedef struct {
SMqRspHead head; SMqRspHead head;
int64_t reqOffset; int64_t reqOffset;
@ -2633,7 +2495,7 @@ typedef struct {
SMqRspHead head; SMqRspHead head;
char cgroup[TSDB_CGROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
SArray* topics; // SArray<SMqSubTopicEp> SArray* topics; // SArray<SMqSubTopicEp>
} SMqCMGetSubEpRsp; } SMqAskEpRsp;
static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) {
// taosMemoryFree(pSubTopicEp->schema.pSchema); // taosMemoryFree(pSubTopicEp->schema.pSchema);
@ -2655,10 +2517,6 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) {
return buf; return buf;
} }
static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) {
taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
}
static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeString(buf, pTopicEp->topic); tlen += taosEncodeString(buf, pTopicEp->topic);
@ -2691,7 +2549,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE
return buf; return buf;
} }
static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pRsp) {
int32_t tlen = 0; int32_t tlen = 0;
// tlen += taosEncodeString(buf, pRsp->cgroup); // tlen += taosEncodeString(buf, pRsp->cgroup);
int32_t sz = taosArrayGetSize(pRsp->topics); int32_t sz = taosArrayGetSize(pRsp->topics);
@ -2703,7 +2561,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu
return tlen; return tlen;
} }
static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) {
// buf = taosDecodeStringTo(buf, pRsp->cgroup); // buf = taosDecodeStringTo(buf, pRsp->cgroup);
int32_t sz; int32_t sz;
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
@ -2719,6 +2577,10 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
return buf; return buf;
} }
static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp);
}
#pragma pack(pop) #pragma pack(pop)
#ifdef __cplusplus #ifdef __cplusplus

9
include/common/tmsgdef.h
View File

@ -145,10 +145,11 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "mnode-mq-ask-ep", SMqAskEpReq, SMqAskEpRsp)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMqConsumerLostMsg, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "mnode-mq-consumer-recover", SMqConsumerRecoverMsg, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-mq-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STREAM, "mnode-create-stream", SCMCreateStreamReq, SCMCreateStreamRsp)
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STREAM, "mnode-alter-stream", NULL, NULL)

41
include/common/trow.h
View File

@ -23,7 +23,6 @@
#include "tbuffer.h" #include "tbuffer.h"
#include "tdataformat.h" #include "tdataformat.h"
#include "tdef.h" #include "tdef.h"
#include "tschema.h"
#include "ttypes.h" #include "ttypes.h"
#include "tutil.h" #include "tutil.h"
@ -58,12 +57,12 @@ extern "C" {
#define TD_ROWS_ALL_NORM 0x00U #define TD_ROWS_ALL_NORM 0x00U
#define TD_ROWS_NULL_NORM 0x01U #define TD_ROWS_NULL_NORM 0x01U
#define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM #define TD_COL_ROWS_NORM(c) ((c)->bitmap == TD_ROWS_ALL_NORM) // all rows of SDataCol/SBlockCol is NORM
#define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v)) #define TD_SET_COL_ROWS_BTIMAP(c, v) ((c)->bitmap = (v))
#define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM) #define TD_SET_COL_ROWS_NORM(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_ALL_NORM)
#define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM) #define TD_SET_COL_ROWS_MISC(c) TD_SET_COL_ROWS_BTIMAP((c), TD_ROWS_NULL_NORM)
#define KvConvertRatio (0.9f) #define KvConvertRatio (0.9f)
#define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio)) #define isSelectKVRow(klen, tlen) ((klen) < ((tlen)*KvConvertRatio))
#ifdef TD_SUPPORT_BITMAP #ifdef TD_SUPPORT_BITMAP
@ -215,6 +214,16 @@ STSRow *tdRowDup(STSRow *row);
static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) { static FORCE_INLINE SKvRowIdx *tdKvRowColIdxAt(STSRow *pRow, col_id_t idx) {
return (SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx; return (SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx;
} }
static FORCE_INLINE int16_t tdKvRowColIdAt(STSRow *pRow, col_id_t idx) {
ASSERT(idx >= 0);
if (idx == 0) {
return PRIMARYKEY_TIMESTAMP_COL_ID;
}
return ((SKvRowIdx *)TD_ROW_COL_IDX(pRow) + idx - 1)->colId;
}
static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return POINTER_SHIFT(pRow, pIdx->offset); } static FORCE_INLINE void *tdKVRowColVal(STSRow *pRow, SKvRowIdx *pIdx) { return POINTER_SHIFT(pRow, pIdx->offset); }
#define TD_ROW_OFFSET(p) ((p)->toffset); // During ParseInsert when without STSchema, how to get the offset for STpRow? #define TD_ROW_OFFSET(p) ((p)->toffset); // During ParseInsert when without STSchema, how to get the offset for STpRow?
@ -341,7 +350,8 @@ static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx,
return false; return false;
} }
static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
int8_t bitmapMode) {
switch (bitmapMode) { switch (bitmapMode) {
case 0: case 0:
tdGetBitmapValTypeII(pBitmap, colIdx, pValType); tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
@ -668,7 +678,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) {
case TD_ROW_KV: case TD_ROW_KV:
#ifdef TD_SUPPORT_BITMAP #ifdef TD_SUPPORT_BITMAP
pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols); pBuilder->pBitmap = tdGetBitmapAddrKv(pBuilder->pBuf, pBuilder->nBoundCols);
memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBitmaps); memset(pBuilder->pBitmap, TD_VTYPE_NONE_BYTE_II, pBuilder->nBoundBitmaps);
#endif #endif
len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) + len = TD_ROW_HEAD_LEN + TD_ROW_NCOLS_LEN + (pBuilder->nBoundCols - 1) * sizeof(SKvRowIdx) +
pBuilder->nBoundBitmaps; // add pBuilder->nBoundBitmaps; // add
@ -1092,7 +1102,7 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
STSRow *pRow = pIter->pRow; STSRow *pRow = pIter->pRow;
SKvRowIdx *pKvIdx = NULL; SKvRowIdx *pKvIdx = NULL;
bool colFound = false; bool colFound = false;
col_id_t kvNCols = tdRowGetNCols(pRow); col_id_t kvNCols = tdRowGetNCols(pRow) - 1;
while (*nIdx < kvNCols) { while (*nIdx < kvNCols) {
pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx)); pKvIdx = (SKvRowIdx *)POINTER_SHIFT(TD_ROW_COL_IDX(pRow), *nIdx * sizeof(SKvRowIdx));
if (pKvIdx->colId == colId) { if (pKvIdx->colId == colId) {
@ -1108,7 +1118,14 @@ static FORCE_INLINE bool tdGetKvRowValOfColEx(STSRowIter *pIter, col_id_t colId,
} }
} }
if (!colFound) return false; if (!colFound) {
if (colId <= pIter->maxColId) {
pVal->valType = TD_VTYPE_NONE;
return true;
} else {
return false;
}
}
#ifdef TD_SUPPORT_BITMAP #ifdef TD_SUPPORT_BITMAP
int16_t colIdx = -1; int16_t colIdx = -1;
@ -1352,14 +1369,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) {
} }
} }
static void tdSRowPrint(STSRow *row, STSchema *pSchema) { static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) {
STSRowIter iter = {0}; STSRowIter iter = {0};
tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterInit(&iter, pSchema);
tdSTSRowIterReset(&iter, row); tdSTSRowIterReset(&iter, row);
printf(">>>"); printf("%s >>>", tag);
for (int i = 0; i < pSchema->numOfCols; ++i) { for (int i = 0; i < pSchema->numOfCols; ++i) {
STColumn *stCol = pSchema->columns + i; STColumn *stCol = pSchema->columns + i;
SCellVal sVal = { 255, NULL}; SCellVal sVal = {.valType = 255, .val = NULL};
if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
break; break;
} }

81
include/common/tschema.h
View File

@ -1,81 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_SCHEMA_H_
#define _TD_COMMON_SCHEMA_H_
#include "os.h"
#include "tarray.h"
#include "ttypes.h"
#ifdef __cplusplus
extern "C" {
#endif
#if 0
typedef struct STColumn {
/// column name
char *cname;
union {
/// for encode purpose
uint64_t info;
struct {
uint64_t sma : 1;
/// column data type
uint64_t type : 7;
/// column id
uint64_t cid : 16;
/// max bytes of the column
uint64_t bytes : 32;
/// reserved
uint64_t reserve : 8;
};
};
/// comment about the column
char *comment;
} STColumn;
typedef struct STSchema {
/// schema version
uint16_t sver;
/// number of columns
uint16_t ncols;
/// sma attributes
struct {
bool sma;
SArray *smaArray;
};
/// column info
STColumn cols[];
} STSchema;
typedef struct {
uint64_t size;
STSchema *pSchema;
} STShemaBuilder;
#define tSchemaBuilderInit(target, capacity) \
{ .size = (capacity), .pSchema = (target) }
void tSchemaBuilderSetSver(STShemaBuilder *pSchemaBuilder, uint16_t sver);
void tSchemaBuilderSetSMA(bool sma, SArray *smaArray);
int32_t tSchemaBuilderPutColumn(char *cname, bool sma, uint8_t type, col_id_t cid, uint32_t bytes, char *comment);
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_COMMON_SCHEMA_H_*/

474
include/common/ttokendef.h
View File

@ -16,244 +16,244 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_USER 33 #define TK_USER 33
#define TK_PRIVILEGE 34 #define TK_PRIVILEGE 34
#define TK_DROP 35 #define TK_DROP 35
#define TK_DNODE 36 #define TK_DNODE 36
#define TK_PORT 37 #define TK_PORT 37
#define TK_NK_INTEGER 38 #define TK_NK_INTEGER 38
#define TK_DNODES 39 #define TK_DNODES 39
#define TK_NK_IPTOKEN 40 #define TK_NK_IPTOKEN 40
#define TK_LOCAL 41 #define TK_LOCAL 41
#define TK_QNODE 42 #define TK_QNODE 42
#define TK_ON 43 #define TK_ON 43
#define TK_BNODE 44 #define TK_BNODE 44
#define TK_SNODE 45 #define TK_SNODE 45
#define TK_MNODE 46 #define TK_MNODE 46
#define TK_DATABASE 47 #define TK_DATABASE 47
#define TK_USE 48 #define TK_USE 48
#define TK_IF 49 #define TK_IF 49
#define TK_NOT 50 #define TK_NOT 50
#define TK_EXISTS 51 #define TK_EXISTS 51
#define TK_BLOCKS 52 #define TK_BUFFER 52
#define TK_CACHE 53 #define TK_CACHELAST 53
#define TK_CACHELAST 54 #define TK_COMP 54
#define TK_COMP 55 #define TK_DAYS 55
#define TK_DAYS 56 #define TK_NK_VARIABLE 56
#define TK_NK_VARIABLE 57 #define TK_FSYNC 57
#define TK_FSYNC 58 #define TK_MAXROWS 58
#define TK_MAXROWS 59 #define TK_MINROWS 59
#define TK_MINROWS 60 #define TK_KEEP 60
#define TK_KEEP 61 #define TK_PAGES 61
#define TK_PRECISION 62 #define TK_PAGESIZE 62
#define TK_QUORUM 63 #define TK_PRECISION 63
#define TK_REPLICA 64 #define TK_REPLICA 64
#define TK_TTL 65 #define TK_STRICT 65
#define TK_WAL 66 #define TK_WAL 66
#define TK_VGROUPS 67 #define TK_VGROUPS 67
#define TK_SINGLE_STABLE 68 #define TK_SINGLE_STABLE 68
#define TK_STREAM_MODE 69 #define TK_RETENTIONS 69
#define TK_RETENTIONS 70 #define TK_NK_COMMA 70
#define TK_STRICT 71 #define TK_NK_COLON 71
#define TK_NK_COMMA 72 #define TK_TABLE 72
#define TK_NK_COLON 73 #define TK_NK_LP 73
#define TK_TABLE 74 #define TK_NK_RP 74
#define TK_NK_LP 75 #define TK_STABLE 75
#define TK_NK_RP 76 #define TK_ADD 76
#define TK_STABLE 77 #define TK_COLUMN 77
#define TK_ADD 78 #define TK_MODIFY 78
#define TK_COLUMN 79 #define TK_RENAME 79
#define TK_MODIFY 80 #define TK_TAG 80
#define TK_RENAME 81 #define TK_SET 81
#define TK_TAG 82 #define TK_NK_EQ 82
#define TK_SET 83 #define TK_USING 83
#define TK_NK_EQ 84 #define TK_TAGS 84
#define TK_USING 85 #define TK_NK_DOT 85
#define TK_TAGS 86 #define TK_COMMENT 86
#define TK_NK_DOT 87 #define TK_BOOL 87
#define TK_COMMENT 88 #define TK_TINYINT 88
#define TK_BOOL 89 #define TK_SMALLINT 89
#define TK_TINYINT 90 #define TK_INT 90
#define TK_SMALLINT 91 #define TK_INTEGER 91
#define TK_INT 92 #define TK_BIGINT 92
#define TK_INTEGER 93 #define TK_FLOAT 93
#define TK_BIGINT 94 #define TK_DOUBLE 94
#define TK_FLOAT 95 #define TK_BINARY 95
#define TK_DOUBLE 96 #define TK_TIMESTAMP 96
#define TK_BINARY 97 #define TK_NCHAR 97
#define TK_TIMESTAMP 98 #define TK_UNSIGNED 98
#define TK_NCHAR 99 #define TK_JSON 99
#define TK_UNSIGNED 100 #define TK_VARCHAR 100
#define TK_JSON 101 #define TK_MEDIUMBLOB 101
#define TK_VARCHAR 102 #define TK_BLOB 102
#define TK_MEDIUMBLOB 103 #define TK_VARBINARY 103
#define TK_BLOB 104 #define TK_DECIMAL 104
#define TK_VARBINARY 105 #define TK_DELAY 105
#define TK_DECIMAL 106 #define TK_FILE_FACTOR 106
#define TK_SMA 107 #define TK_NK_FLOAT 107
#define TK_ROLLUP 108 #define TK_ROLLUP 108
#define TK_FILE_FACTOR 109 #define TK_TTL 109
#define TK_NK_FLOAT 110 #define TK_SMA 110
#define TK_DELAY 111 #define TK_SHOW 111
#define TK_SHOW 112 #define TK_DATABASES 112
#define TK_DATABASES 113 #define TK_TABLES 113
#define TK_TABLES 114 #define TK_STABLES 114
#define TK_STABLES 115 #define TK_MNODES 115
#define TK_MNODES 116 #define TK_MODULES 116
#define TK_MODULES 117 #define TK_QNODES 117
#define TK_QNODES 118 #define TK_FUNCTIONS 118
#define TK_FUNCTIONS 119 #define TK_INDEXES 119
#define TK_INDEXES 120 #define TK_FROM 120
#define TK_FROM 121 #define TK_ACCOUNTS 121
#define TK_ACCOUNTS 122 #define TK_APPS 122
#define TK_APPS 123 #define TK_CONNECTIONS 123
#define TK_CONNECTIONS 124 #define TK_LICENCE 124
#define TK_LICENCE 125 #define TK_GRANTS 125
#define TK_GRANTS 126 #define TK_QUERIES 126
#define TK_QUERIES 127 #define TK_SCORES 127
#define TK_SCORES 128 #define TK_TOPICS 128
#define TK_TOPICS 129 #define TK_VARIABLES 129
#define TK_VARIABLES 130 #define TK_BNODES 130
#define TK_BNODES 131 #define TK_SNODES 131
#define TK_SNODES 132 #define TK_CLUSTER 132
#define TK_CLUSTER 133 #define TK_LIKE 133
#define TK_LIKE 134 #define TK_INDEX 134
#define TK_INDEX 135 #define TK_FULLTEXT 135
#define TK_FULLTEXT 136 #define TK_FUNCTION 136
#define TK_FUNCTION 137 #define TK_INTERVAL 137
#define TK_INTERVAL 138 #define TK_TOPIC 138
#define TK_TOPIC 139 #define TK_AS 139
#define TK_AS 140 #define TK_WITH 140
#define TK_WITH 141 #define TK_SCHEMA 141
#define TK_SCHEMA 142 #define TK_DESC 142
#define TK_DESC 143 #define TK_DESCRIBE 143
#define TK_DESCRIBE 144 #define TK_RESET 144
#define TK_RESET 145 #define TK_QUERY 145
#define TK_QUERY 146 #define TK_CACHE 146
#define TK_EXPLAIN 147 #define TK_EXPLAIN 147
#define TK_ANALYZE 148 #define TK_ANALYZE 148
#define TK_VERBOSE 149 #define TK_VERBOSE 149
#define TK_NK_BOOL 150 #define TK_NK_BOOL 150
#define TK_RATIO 151 #define TK_RATIO 151
#define TK_COMPACT 152 #define TK_COMPACT 152
#define TK_VNODES 153 #define TK_VNODES 153
#define TK_IN 154 #define TK_IN 154
#define TK_OUTPUTTYPE 155 #define TK_OUTPUTTYPE 155
#define TK_AGGREGATE 156 #define TK_AGGREGATE 156
#define TK_BUFSIZE 157 #define TK_BUFSIZE 157
#define TK_STREAM 158 #define TK_STREAM 158
#define TK_INTO 159 #define TK_INTO 159
#define TK_TRIGGER 160 #define TK_TRIGGER 160
#define TK_AT_ONCE 161 #define TK_AT_ONCE 161
#define TK_WINDOW_CLOSE 162 #define TK_WINDOW_CLOSE 162
#define TK_WATERMARK 163 #define TK_WATERMARK 163
#define TK_KILL 164 #define TK_KILL 164
#define TK_CONNECTION 165 #define TK_CONNECTION 165
#define TK_MERGE 166 #define TK_MERGE 166
#define TK_VGROUP 167 #define TK_VGROUP 167
#define TK_REDISTRIBUTE 168 #define TK_REDISTRIBUTE 168
#define TK_SPLIT 169 #define TK_SPLIT 169
#define TK_SYNCDB 170 #define TK_SYNCDB 170
#define TK_NULL 171 #define TK_NULL 171
#define TK_NK_QUESTION 172 #define TK_NK_QUESTION 172
#define TK_NK_ARROW 173 #define TK_NK_ARROW 173
#define TK_ROWTS 174 #define TK_ROWTS 174
#define TK_TBNAME 175 #define TK_TBNAME 175
#define TK_QSTARTTS 176 #define TK_QSTARTTS 176
#define TK_QENDTS 177 #define TK_QENDTS 177
#define TK_WSTARTTS 178 #define TK_WSTARTTS 178
#define TK_WENDTS 179 #define TK_WENDTS 179
#define TK_WDURATION 180 #define TK_WDURATION 180
#define TK_CAST 181 #define TK_CAST 181
#define TK_NOW 182 #define TK_NOW 182
#define TK_TODAY 183 #define TK_TODAY 183
#define TK_TIMEZONE 184 #define TK_TIMEZONE 184
#define TK_COUNT 185 #define TK_COUNT 185
#define TK_FIRST 186 #define TK_FIRST 186
#define TK_LAST 187 #define TK_LAST 187
#define TK_LAST_ROW 188 #define TK_LAST_ROW 188
#define TK_BETWEEN 189 #define TK_BETWEEN 189
#define TK_IS 190 #define TK_IS 190
#define TK_NK_LT 191 #define TK_NK_LT 191
#define TK_NK_GT 192 #define TK_NK_GT 192
#define TK_NK_LE 193 #define TK_NK_LE 193
#define TK_NK_GE 194 #define TK_NK_GE 194
#define TK_NK_NE 195 #define TK_NK_NE 195
#define TK_MATCH 196 #define TK_MATCH 196
#define TK_NMATCH 197 #define TK_NMATCH 197
#define TK_CONTAINS 198 #define TK_CONTAINS 198
#define TK_JOIN 199 #define TK_JOIN 199
#define TK_INNER 200 #define TK_INNER 200
#define TK_SELECT 201 #define TK_SELECT 201
#define TK_DISTINCT 202 #define TK_DISTINCT 202
#define TK_WHERE 203 #define TK_WHERE 203
#define TK_PARTITION 204 #define TK_PARTITION 204
#define TK_BY 205 #define TK_BY 205
#define TK_SESSION 206 #define TK_SESSION 206
#define TK_STATE_WINDOW 207 #define TK_STATE_WINDOW 207
#define TK_SLIDING 208 #define TK_SLIDING 208
#define TK_FILL 209 #define TK_FILL 209
#define TK_VALUE 210 #define TK_VALUE 210
#define TK_NONE 211 #define TK_NONE 211
#define TK_PREV 212 #define TK_PREV 212
#define TK_LINEAR 213 #define TK_LINEAR 213
#define TK_NEXT 214 #define TK_NEXT 214
#define TK_GROUP 215 #define TK_GROUP 215
#define TK_HAVING 216 #define TK_HAVING 216
#define TK_ORDER 217 #define TK_ORDER 217
#define TK_SLIMIT 218 #define TK_SLIMIT 218
#define TK_SOFFSET 219 #define TK_SOFFSET 219
#define TK_LIMIT 220 #define TK_LIMIT 220
#define TK_OFFSET 221 #define TK_OFFSET 221
#define TK_ASC 222 #define TK_ASC 222
#define TK_NULLS 223 #define TK_NULLS 223
#define TK_ID 224 #define TK_ID 224
#define TK_NK_BITNOT 225 #define TK_NK_BITNOT 225
#define TK_INSERT 226 #define TK_INSERT 226
#define TK_VALUES 227 #define TK_VALUES 227
#define TK_IMPORT 228 #define TK_IMPORT 228
#define TK_NK_SEMI 229 #define TK_NK_SEMI 229
#define TK_FILE 230 #define TK_FILE 230
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301
#define TK_NK_ILLEGAL 302 #define TK_NK_ILLEGAL 302
#define TK_NK_HEX 303 // hex number 0x123 #define TK_NK_HEX 303 // hex number 0x123
#define TK_NK_OCT 304 // oct number #define TK_NK_OCT 304 // oct number
#define TK_NK_BIN 305 // bin format data 0b111 #define TK_NK_BIN 305 // bin format data 0b111
#define TK_NK_NIL 65535 #define TK_NK_NIL 65535
#endif /*_TD_COMMON_TOKEN_H_*/ #endif /*_TD_COMMON_TOKEN_H_*/

31
include/dnode/mnode/sdb/sdb.h
View File

@ -18,6 +18,11 @@
#include "os.h" #include "os.h"
#include "thash.h"
#include "tlockfree.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -135,7 +140,7 @@ typedef enum {
typedef struct SSdb SSdb; typedef struct SSdb SSdb;
typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj);
typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj); typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj);
typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc);
typedef int32_t (*SdbDeployFp)(SMnode *pMnode); typedef int32_t (*SdbDeployFp)(SMnode *pMnode);
typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw);
typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj);
@ -221,7 +226,7 @@ int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw);
* @param pRaw The raw data. * @param pRaw The raw data.
* @return int32_t 0 for success, -1 for failure. * @return int32_t 0 for success, -1 for failure.
*/ */
int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw); int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw);
/** /**
* @brief Acquire a row from sdb * @brief Acquire a row from sdb
@ -326,9 +331,29 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw);
SSdbRow *sdbAllocRow(int32_t objSize); SSdbRow *sdbAllocRow(int32_t objSize);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow); void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc);
void *sdbGetRowObj(SSdbRow *pRow); void *sdbGetRowObj(SSdbRow *pRow);
typedef struct SSdb {
SMnode *pMnode;
char *currDir;
char *syncDir;
char *tmpDir;
int64_t lastCommitVer;
int64_t curVer;
int64_t tableVer[SDB_MAX];
int64_t maxId[SDB_MAX];
EKeyType keyTypes[SDB_MAX];
SHashObj *hashObjs[SDB_MAX];
SRWLatch locks[SDB_MAX];
SdbInsertFp insertFps[SDB_MAX];
SdbUpdateFp updateFps[SDB_MAX];
SdbDeleteFp deleteFps[SDB_MAX];
SdbDeployFp deployFps[SDB_MAX];
SdbEncodeFp encodeFps[SDB_MAX];
SdbDecodeFp decodeFps[SDB_MAX];
} SSdb;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
include/libs/executor/executor.h
View File

@ -32,6 +32,7 @@ typedef struct SReadHandle {
void* reader; void* reader;
void* meta; void* meta;
void* config; void* config;
void* vnode;
} SReadHandle; } SReadHandle;
#define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1 #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1

6
include/libs/function/function.h
View File

@ -202,10 +202,12 @@ typedef struct SqlFunctionCtx {
SPoint1 end; SPoint1 end;
SFuncExecFuncs fpSet; SFuncExecFuncs fpSet;
SScalarFuncExecFuncs sfp; SScalarFuncExecFuncs sfp;
SExprInfo *pExpr; struct SExprInfo *pExpr;
struct SDiskbasedBuf *pBuf; struct SDiskbasedBuf *pBuf;
struct SSDataBlock *pSrcBlock; struct SSDataBlock *pSrcBlock;
int32_t curBufPage; int32_t curBufPage;
char* udfName[TSDB_FUNC_NAME_LEN];
} SqlFunctionCtx; } SqlFunctionCtx;
enum { enum {
@ -334,8 +336,6 @@ int32_t udfcOpen();
*/ */
int32_t udfcClose(); int32_t udfcClose();
typedef void *UdfcFuncHandle;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

13
include/libs/function/functionMgt.h
View File

@ -20,8 +20,8 @@
extern "C" { extern "C" {
#endif #endif
#include "querynodes.h"
#include "function.h" #include "function.h"
#include "querynodes.h"
typedef enum EFunctionType { typedef enum EFunctionType {
// aggregate function // aggregate function
@ -123,17 +123,17 @@ struct SCatalog;
typedef struct SFmGetFuncInfoParam { typedef struct SFmGetFuncInfoParam {
struct SCatalog* pCtg; struct SCatalog* pCtg;
void *pRpc; void* pRpc;
const SEpSet* pMgmtEps; const SEpSet* pMgmtEps;
char* pErrBuf;
int32_t errBufLen;
} SFmGetFuncInfoParam; } SFmGetFuncInfoParam;
int32_t fmFuncMgtInit(); int32_t fmFuncMgtInit();
void fmFuncMgtDestroy(); void fmFuncMgtDestroy();
int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc);
int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
bool fmIsAggFunc(int32_t funcId); bool fmIsAggFunc(int32_t funcId);
bool fmIsScalarFunc(int32_t funcId); bool fmIsScalarFunc(int32_t funcId);
@ -143,6 +143,7 @@ bool fmIsDatetimeFunc(int32_t funcId);
bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId);
bool fmIsTimeorderFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId);
bool fmIsPseudoColumnFunc(int32_t funcId); bool fmIsPseudoColumnFunc(int32_t funcId);
bool fmIsScanPseudoColumnFunc(int32_t funcId);
bool fmIsWindowPseudoColumnFunc(int32_t funcId); bool fmIsWindowPseudoColumnFunc(int32_t funcId);
bool fmIsWindowClauseFunc(int32_t funcId); bool fmIsWindowClauseFunc(int32_t funcId);
bool fmIsSpecialDataRequiredFunc(int32_t funcId); bool fmIsSpecialDataRequiredFunc(int32_t funcId);

16
source/libs/function/inc/tudf.h → include/libs/function/tudf.h
View File

@ -42,8 +42,7 @@ enum {
UDFC_CODE_INVALID_STATE = -5 UDFC_CODE_INVALID_STATE = -5
}; };
typedef void *UdfcFuncHandle;
/** /**
* setup udf * setup udf
@ -95,6 +94,7 @@ typedef struct SUdfDataBlock {
typedef struct SUdfInterBuf { typedef struct SUdfInterBuf {
int32_t bufLen; int32_t bufLen;
char* buf; char* buf;
int8_t numOfResult; //zero or one
} SUdfInterBuf; } SUdfInterBuf;
// output: interBuf // output: interBuf
@ -118,6 +118,10 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
*/ */
int32_t teardownUdf(UdfcFuncHandle handle); int32_t teardownUdf(UdfcFuncHandle handle);
bool udfAggGetEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
int32_t udfAggProcess(struct SqlFunctionCtx *pCtx);
int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
// end API to taosd and qworker // end API to taosd and qworker
//============================================================================================================================= //=============================================================================================================================
// begin API to UDF writer. // begin API to UDF writer.
@ -133,11 +137,11 @@ typedef int32_t (*TUdfTeardownFunc)();
//typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data); //typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data);
typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column);
typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol);
typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock block, SUdfColumn *resultCol); typedef int32_t (*TUdfAggStartFunc)(SUdfInterBuf *buf);
typedef int32_t (*TUdfAggInitFunc)(SUdfInterBuf *buf); typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock* block, SUdfInterBuf *interBuf);
typedef int32_t (*TUdfAggProcessFunc)(SUdfDataBlock block, SUdfInterBuf *interBuf); typedef int32_t (*TUdfAggFinishFunc)(SUdfInterBuf* buf, SUdfInterBuf *resultData);
typedef int32_t (*TUdfAggFinalizeFunc)(SUdfInterBuf buf, SUdfInterBuf *resultData);
// end API to UDF writer // end API to UDF writer

269
include/libs/nodes/cmdnodes.h
View File

@ -23,294 +23,291 @@ extern "C" {
#include "query.h" #include "query.h"
#include "querynodes.h" #include "querynodes.h"
#define DESCRIBE_RESULT_COLS 4 #define DESCRIBE_RESULT_COLS 4
#define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE)
typedef struct SDatabaseOptions { typedef struct SDatabaseOptions {
ENodeType type; ENodeType type;
SValueNode* pNumOfBlocks; int32_t buffer;
SValueNode* pCacheBlockSize; int8_t cachelast;
SValueNode* pCachelast; int8_t compressionLevel;
SValueNode* pCompressionLevel; int32_t daysPerFile;
SValueNode* pDaysPerFile; SValueNode* pDaysPerFile;
SValueNode* pFsyncPeriod; int32_t fsyncPeriod;
SValueNode* pMaxRowsPerBlock; int32_t maxRowsPerBlock;
SValueNode* pMinRowsPerBlock; int32_t minRowsPerBlock;
SNodeList* pKeep; SNodeList* pKeep;
SValueNode* pPrecision; int32_t keep[3];
SValueNode* pQuorum; int32_t pages;
SValueNode* pReplica; int32_t pagesize;
SValueNode* pTtl; char precisionStr[3];
SValueNode* pWalLevel; int8_t precision;
SValueNode* pNumOfVgroups; int8_t replica;
SValueNode* pSingleStable; int8_t strict;
SValueNode* pStreamMode; int8_t walLevel;
SValueNode* pStrict; int32_t numOfVgroups;
SNodeList* pRetentions; int8_t singleStable;
SNodeList* pRetentions;
} SDatabaseOptions; } SDatabaseOptions;
typedef struct SCreateDatabaseStmt { typedef struct SCreateDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SDatabaseOptions* pOptions; SDatabaseOptions* pOptions;
} SCreateDatabaseStmt; } SCreateDatabaseStmt;
typedef struct SUseDatabaseStmt { typedef struct SUseDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
} SUseDatabaseStmt; } SUseDatabaseStmt;
typedef struct SDropDatabaseStmt { typedef struct SDropDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropDatabaseStmt; } SDropDatabaseStmt;
typedef struct SAlterDatabaseStmt { typedef struct SAlterDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
SDatabaseOptions* pOptions; SDatabaseOptions* pOptions;
} SAlterDatabaseStmt; } SAlterDatabaseStmt;
typedef struct STableOptions { typedef struct STableOptions {
ENodeType type; ENodeType type;
SNodeList* pKeep; char comment[TSDB_STB_COMMENT_LEN];
SValueNode* pTtl; int32_t delay;
SValueNode* pComments; float filesFactor;
SNodeList* pRollupFuncs;
int32_t ttl;
SNodeList* pSma; SNodeList* pSma;
SNodeList* pFuncs;
SValueNode* pFilesFactor;
SValueNode* pDelay;
} STableOptions; } STableOptions;
typedef struct SColumnDefNode { typedef struct SColumnDefNode {
ENodeType type; ENodeType type;
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
SDataType dataType; SDataType dataType;
char comments[TSDB_STB_COMMENT_LEN]; char comments[TSDB_STB_COMMENT_LEN];
bool sma; bool sma;
} SColumnDefNode; } SColumnDefNode;
typedef struct SCreateTableStmt { typedef struct SCreateTableStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SNodeList* pCols; SNodeList* pCols;
SNodeList* pTags; SNodeList* pTags;
STableOptions* pOptions; STableOptions* pOptions;
} SCreateTableStmt; } SCreateTableStmt;
typedef struct SCreateSubTableClause { typedef struct SCreateSubTableClause {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
char useDbName[TSDB_DB_NAME_LEN]; char useDbName[TSDB_DB_NAME_LEN];
char useTableName[TSDB_TABLE_NAME_LEN]; char useTableName[TSDB_TABLE_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SNodeList* pSpecificTags; SNodeList* pSpecificTags;
SNodeList* pValsOfTags; SNodeList* pValsOfTags;
} SCreateSubTableClause; } SCreateSubTableClause;
typedef struct SCreateMultiTableStmt { typedef struct SCreateMultiTableStmt {
ENodeType type; ENodeType type;
SNodeList* pSubTables; SNodeList* pSubTables;
} SCreateMultiTableStmt; } SCreateMultiTableStmt;
typedef struct SDropTableClause { typedef struct SDropTableClause {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropTableClause; } SDropTableClause;
typedef struct SDropTableStmt { typedef struct SDropTableStmt {
ENodeType type; ENodeType type;
SNodeList* pTables; SNodeList* pTables;
} SDropTableStmt; } SDropTableStmt;
typedef struct SDropSuperTableStmt { typedef struct SDropSuperTableStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropSuperTableStmt; } SDropSuperTableStmt;
typedef struct SAlterTableStmt { typedef struct SAlterTableStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
int8_t alterType; int8_t alterType;
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
char newColName[TSDB_COL_NAME_LEN]; char newColName[TSDB_COL_NAME_LEN];
STableOptions* pOptions; STableOptions* pOptions;
SDataType dataType; SDataType dataType;
SValueNode* pVal; SValueNode* pVal;
} SAlterTableStmt; } SAlterTableStmt;
typedef struct SCreateUserStmt { typedef struct SCreateUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
} SCreateUserStmt; } SCreateUserStmt;
typedef struct SAlterUserStmt { typedef struct SAlterUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
int8_t alterType; int8_t alterType;
} SAlterUserStmt; } SAlterUserStmt;
typedef struct SDropUserStmt { typedef struct SDropUserStmt {
ENodeType type; ENodeType type;
char useName[TSDB_USER_LEN]; char useName[TSDB_USER_LEN];
} SDropUserStmt; } SDropUserStmt;
typedef struct SCreateDnodeStmt { typedef struct SCreateDnodeStmt {
ENodeType type; ENodeType type;
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
int32_t port; int32_t port;
} SCreateDnodeStmt; } SCreateDnodeStmt;
typedef struct SDropDnodeStmt { typedef struct SDropDnodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
int32_t port; int32_t port;
} SDropDnodeStmt; } SDropDnodeStmt;
typedef struct SAlterDnodeStmt { typedef struct SAlterDnodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN]; char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN]; char value[TSDB_DNODE_VALUE_LEN];
} SAlterDnodeStmt; } SAlterDnodeStmt;
typedef struct SShowStmt { typedef struct SShowStmt {
ENodeType type; ENodeType type;
SNode* pDbName; // SValueNode SNode* pDbName; // SValueNode
SNode* pTbNamePattern; // SValueNode SNode* pTbNamePattern; // SValueNode
} SShowStmt; } SShowStmt;
typedef struct SShowCreatStmt { typedef struct SShowCreatStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
} SShowCreatStmt; } SShowCreatStmt;
typedef enum EIndexType { typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType;
INDEX_TYPE_SMA = 1,
INDEX_TYPE_FULLTEXT
} EIndexType;
typedef struct SIndexOptions { typedef struct SIndexOptions {
ENodeType type; ENodeType type;
SNodeList* pFuncs; SNodeList* pFuncs;
SNode* pInterval; SNode* pInterval;
SNode* pOffset; SNode* pOffset;
SNode* pSliding; SNode* pSliding;
} SIndexOptions; } SIndexOptions;
typedef struct SCreateIndexStmt { typedef struct SCreateIndexStmt {
ENodeType type; ENodeType type;
EIndexType indexType; EIndexType indexType;
bool ignoreExists; bool ignoreExists;
char indexName[TSDB_INDEX_NAME_LEN]; char indexName[TSDB_INDEX_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
SNodeList* pCols; SNodeList* pCols;
SIndexOptions* pOptions; SIndexOptions* pOptions;
} SCreateIndexStmt; } SCreateIndexStmt;
typedef struct SDropIndexStmt { typedef struct SDropIndexStmt {
ENodeType type; ENodeType type;
bool ignoreNotExists; bool ignoreNotExists;
char indexName[TSDB_INDEX_NAME_LEN]; char indexName[TSDB_INDEX_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
} SDropIndexStmt; } SDropIndexStmt;
typedef struct SCreateComponentNodeStmt { typedef struct SCreateComponentNodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
} SCreateComponentNodeStmt; } SCreateComponentNodeStmt;
typedef struct SDropComponentNodeStmt { typedef struct SDropComponentNodeStmt {
ENodeType type; ENodeType type;
int32_t dnodeId; int32_t dnodeId;
} SDropComponentNodeStmt; } SDropComponentNodeStmt;
typedef struct STopicOptions { typedef struct STopicOptions {
ENodeType type; ENodeType type;
bool withTable; bool withTable;
bool withSchema; bool withSchema;
bool withTag; bool withTag;
} STopicOptions; } STopicOptions;
typedef struct SCreateTopicStmt { typedef struct SCreateTopicStmt {
ENodeType type; ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN]; char topicName[TSDB_TABLE_NAME_LEN];
char subscribeDbName[TSDB_DB_NAME_LEN]; char subscribeDbName[TSDB_DB_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SNode* pQuery; SNode* pQuery;
STopicOptions* pOptions; STopicOptions* pOptions;
} SCreateTopicStmt; } SCreateTopicStmt;
typedef struct SDropTopicStmt { typedef struct SDropTopicStmt {
ENodeType type; ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN]; char topicName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropTopicStmt; } SDropTopicStmt;
typedef struct SAlterLocalStmt { typedef struct SAlterLocalStmt {
ENodeType type; ENodeType type;
char config[TSDB_DNODE_CONFIG_LEN]; char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_DNODE_VALUE_LEN]; char value[TSDB_DNODE_VALUE_LEN];
} SAlterLocalStmt; } SAlterLocalStmt;
typedef struct SDescribeStmt { typedef struct SDescribeStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
STableMeta* pMeta; STableMeta* pMeta;
} SDescribeStmt; } SDescribeStmt;
typedef struct SKillStmt { typedef struct SKillStmt {
ENodeType type; ENodeType type;
int32_t targetId; int32_t targetId;
} SKillStmt; } SKillStmt;
typedef struct SStreamOptions { typedef struct SStreamOptions {
ENodeType type; ENodeType type;
int8_t triggerType; int8_t triggerType;
SNode* pWatermark; SNode* pWatermark;
} SStreamOptions; } SStreamOptions;
typedef struct SCreateStreamStmt { typedef struct SCreateStreamStmt {
ENodeType type; ENodeType type;
char streamName[TSDB_TABLE_NAME_LEN]; char streamName[TSDB_TABLE_NAME_LEN];
char targetDbName[TSDB_DB_NAME_LEN]; char targetDbName[TSDB_DB_NAME_LEN];
char targetTabName[TSDB_TABLE_NAME_LEN]; char targetTabName[TSDB_TABLE_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SStreamOptions* pOptions; SStreamOptions* pOptions;
SNode* pQuery; SNode* pQuery;
} SCreateStreamStmt; } SCreateStreamStmt;
typedef struct SDropStreamStmt { typedef struct SDropStreamStmt {
ENodeType type; ENodeType type;
char streamName[TSDB_TABLE_NAME_LEN]; char streamName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropStreamStmt; } SDropStreamStmt;
typedef struct SCreateFunctionStmt { typedef struct SCreateFunctionStmt {
ENodeType type; ENodeType type;
bool ignoreExists; bool ignoreExists;
char funcName[TSDB_FUNC_NAME_LEN]; char funcName[TSDB_FUNC_NAME_LEN];
bool isAgg; bool isAgg;
char libraryPath[PATH_MAX]; char libraryPath[PATH_MAX];
SDataType outputDt; SDataType outputDt;
int32_t bufSize; int32_t bufSize;
} SCreateFunctionStmt; } SCreateFunctionStmt;
#ifdef __cplusplus #ifdef __cplusplus

295
include/libs/nodes/plannodes.h
View File

@ -20,50 +20,46 @@
extern "C" { extern "C" {
#endif #endif
#include "querynodes.h"
#include "query.h" #include "query.h"
#include "querynodes.h"
#include "tname.h" #include "tname.h"
typedef struct SLogicNode { typedef struct SLogicNode {
ENodeType type; ENodeType type;
SNodeList* pTargets; // SColumnNode SNodeList* pTargets; // SColumnNode
SNode* pConditions; SNode* pConditions;
SNodeList* pChildren; SNodeList* pChildren;
struct SLogicNode* pParent; struct SLogicNode* pParent;
int32_t optimizedFlag; int32_t optimizedFlag;
} SLogicNode; } SLogicNode;
typedef enum EScanType { typedef enum EScanType { SCAN_TYPE_TAG = 1, SCAN_TYPE_TABLE, SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM } EScanType;
SCAN_TYPE_TAG,
SCAN_TYPE_TABLE,
SCAN_TYPE_SYSTEM_TABLE,
SCAN_TYPE_STREAM
} EScanType;
typedef struct SScanLogicNode { typedef struct SScanLogicNode {
SLogicNode node; SLogicNode node;
SNodeList* pScanCols; SNodeList* pScanCols;
SNodeList* pScanPseudoCols;
struct STableMeta* pMeta; struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList; SVgroupsInfo* pVgroupList;
EScanType scanType; EScanType scanType;
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange; STimeWindow scanRange;
SName tableName; SName tableName;
bool showRewrite; bool showRewrite;
double ratio; double ratio;
SNodeList* pDynamicScanFuncs; SNodeList* pDynamicScanFuncs;
int32_t dataRequired; int32_t dataRequired;
int64_t interval; int64_t interval;
int64_t offset; int64_t offset;
int64_t sliding; int64_t sliding;
int8_t intervalUnit; int8_t intervalUnit;
int8_t slidingUnit; int8_t slidingUnit;
} SScanLogicNode; } SScanLogicNode;
typedef struct SJoinLogicNode { typedef struct SJoinLogicNode {
SLogicNode node; SLogicNode node;
EJoinType joinType; EJoinType joinType;
SNode* pOnConditions; SNode* pOnConditions;
} SJoinLogicNode; } SJoinLogicNode;
typedef struct SAggLogicNode { typedef struct SAggLogicNode {
@ -75,47 +71,43 @@ typedef struct SAggLogicNode {
typedef struct SProjectLogicNode { typedef struct SProjectLogicNode {
SLogicNode node; SLogicNode node;
SNodeList* pProjections; SNodeList* pProjections;
char stmtName[TSDB_TABLE_NAME_LEN]; char stmtName[TSDB_TABLE_NAME_LEN];
int64_t limit; int64_t limit;
int64_t offset; int64_t offset;
int64_t slimit; int64_t slimit;
int64_t soffset; int64_t soffset;
} SProjectLogicNode; } SProjectLogicNode;
typedef struct SVnodeModifLogicNode { typedef struct SVnodeModifLogicNode {
SLogicNode node; SLogicNode node;
int32_t msgType; int32_t msgType;
SArray* pDataBlocks; SArray* pDataBlocks;
SVgDataBlocks* pVgDataBlocks; SVgDataBlocks* pVgDataBlocks;
} SVnodeModifLogicNode; } SVnodeModifLogicNode;
typedef struct SExchangeLogicNode { typedef struct SExchangeLogicNode {
SLogicNode node; SLogicNode node;
int32_t srcGroupId; int32_t srcGroupId;
uint8_t precision; uint8_t precision;
} SExchangeLogicNode; } SExchangeLogicNode;
typedef enum EWindowType { typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType;
WINDOW_TYPE_INTERVAL = 1,
WINDOW_TYPE_SESSION,
WINDOW_TYPE_STATE
} EWindowType;
typedef struct SWindowLogicNode { typedef struct SWindowLogicNode {
SLogicNode node; SLogicNode node;
EWindowType winType; EWindowType winType;
SNodeList* pFuncs; SNodeList* pFuncs;
int64_t interval; int64_t interval;
int64_t offset; int64_t offset;
int64_t sliding; int64_t sliding;
int8_t intervalUnit; int8_t intervalUnit;
int8_t slidingUnit; int8_t slidingUnit;
SFillNode* pFill; SFillNode* pFill;
int64_t sessionGap; int64_t sessionGap;
SNode* pTspk; SNode* pTspk;
SNode* pStateExpr; SNode* pStateExpr;
int8_t triggerType; int8_t triggerType;
int64_t watermark; int64_t watermark;
} SWindowLogicNode; } SWindowLogicNode;
typedef struct SSortLogicNode { typedef struct SSortLogicNode {
@ -137,59 +129,60 @@ typedef enum ESubplanType {
typedef struct SSubplanId { typedef struct SSubplanId {
uint64_t queryId; uint64_t queryId;
int32_t groupId; int32_t groupId;
int32_t subplanId; int32_t subplanId;
} SSubplanId; } SSubplanId;
typedef struct SLogicSubplan { typedef struct SLogicSubplan {
ENodeType type; ENodeType type;
SSubplanId id; SSubplanId id;
SNodeList* pChildren; SNodeList* pChildren;
SNodeList* pParents; SNodeList* pParents;
SLogicNode* pNode; SLogicNode* pNode;
ESubplanType subplanType; ESubplanType subplanType;
SVgroupsInfo* pVgroupList; SVgroupsInfo* pVgroupList;
int32_t level; int32_t level;
int32_t splitFlag; int32_t splitFlag;
} SLogicSubplan; } SLogicSubplan;
typedef struct SQueryLogicPlan { typedef struct SQueryLogicPlan {
ENodeType type; ENodeType type;
SNodeList* pTopSubplans; SNodeList* pTopSubplans;
} SQueryLogicPlan; } SQueryLogicPlan;
typedef struct SSlotDescNode { typedef struct SSlotDescNode {
ENodeType type; ENodeType type;
int16_t slotId; int16_t slotId;
SDataType dataType; SDataType dataType;
bool reserve; bool reserve;
bool output; bool output;
bool tag; bool tag;
} SSlotDescNode; } SSlotDescNode;
typedef struct SDataBlockDescNode { typedef struct SDataBlockDescNode {
ENodeType type; ENodeType type;
int16_t dataBlockId; int16_t dataBlockId;
SNodeList* pSlots; SNodeList* pSlots;
int32_t totalRowSize; int32_t totalRowSize;
int32_t outputRowSize; int32_t outputRowSize;
uint8_t precision; uint8_t precision;
} SDataBlockDescNode; } SDataBlockDescNode;
typedef struct SPhysiNode { typedef struct SPhysiNode {
ENodeType type; ENodeType type;
SDataBlockDescNode* pOutputDataBlockDesc; SDataBlockDescNode* pOutputDataBlockDesc;
SNode* pConditions; SNode* pConditions;
SNodeList* pChildren; SNodeList* pChildren;
struct SPhysiNode* pParent; struct SPhysiNode* pParent;
} SPhysiNode; } SPhysiNode;
typedef struct SScanPhysiNode { typedef struct SScanPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pScanCols; SNodeList* pScanCols;
uint64_t uid; // unique id of the table SNodeList* pScanPseudoCols;
int8_t tableType; uint64_t uid; // unique id of the table
SName tableName; int8_t tableType;
SName tableName;
} SScanPhysiNode; } SScanPhysiNode;
typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode STagScanPhysiNode;
@ -197,23 +190,23 @@ typedef SScanPhysiNode SStreamScanPhysiNode;
typedef struct SSystemTableScanPhysiNode { typedef struct SSystemTableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;
SEpSet mgmtEpSet; SEpSet mgmtEpSet;
bool showRewrite; bool showRewrite;
int32_t accountId; int32_t accountId;
} SSystemTableScanPhysiNode; } SSystemTableScanPhysiNode;
typedef struct STableScanPhysiNode { typedef struct STableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;
uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange; STimeWindow scanRange;
double ratio; double ratio;
int32_t dataRequired; int32_t dataRequired;
SNodeList* pDynamicScanFuncs; SNodeList* pDynamicScanFuncs;
int64_t interval; int64_t interval;
int64_t offset; int64_t offset;
int64_t sliding; int64_t sliding;
int8_t intervalUnit; int8_t intervalUnit;
int8_t slidingUnit; int8_t slidingUnit;
} STableScanPhysiNode; } STableScanPhysiNode;
typedef STableScanPhysiNode STableSeqScanPhysiNode; typedef STableScanPhysiNode STableSeqScanPhysiNode;
@ -221,89 +214,89 @@ typedef STableScanPhysiNode STableSeqScanPhysiNode;
typedef struct SProjectPhysiNode { typedef struct SProjectPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pProjections; SNodeList* pProjections;
int64_t limit; int64_t limit;
int64_t offset; int64_t offset;
int64_t slimit; int64_t slimit;
int64_t soffset; int64_t soffset;
} SProjectPhysiNode; } SProjectPhysiNode;
typedef struct SJoinPhysiNode { typedef struct SJoinPhysiNode {
SPhysiNode node; SPhysiNode node;
EJoinType joinType; EJoinType joinType;
SNode* pOnConditions; // in or out tuple ? SNode* pOnConditions; // in or out tuple ?
SNodeList* pTargets; SNodeList* pTargets;
} SJoinPhysiNode; } SJoinPhysiNode;
typedef struct SAggPhysiNode { typedef struct SAggPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function SNodeList* pExprs; // these are expression list of group_by_clause and parameter expression of aggregate function
SNodeList* pGroupKeys; SNodeList* pGroupKeys;
SNodeList* pAggFuncs; SNodeList* pAggFuncs;
} SAggPhysiNode; } SAggPhysiNode;
typedef struct SDownstreamSourceNode { typedef struct SDownstreamSourceNode {
ENodeType type; ENodeType type;
SQueryNodeAddr addr; SQueryNodeAddr addr;
uint64_t taskId; uint64_t taskId;
uint64_t schedId; uint64_t schedId;
} SDownstreamSourceNode; } SDownstreamSourceNode;
typedef struct SExchangePhysiNode { typedef struct SExchangePhysiNode {
SPhysiNode node; SPhysiNode node;
int32_t srcGroupId; // group id of datasource suplans int32_t srcGroupId; // group id of datasource suplans
SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode
} SExchangePhysiNode; } SExchangePhysiNode;
typedef struct SWinodwPhysiNode { typedef struct SWinodwPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pExprs; // these are expression list of parameter expression of function SNodeList* pExprs; // these are expression list of parameter expression of function
SNodeList* pFuncs; SNodeList* pFuncs;
SNode* pTspk; // timestamp primary key SNode* pTspk; // timestamp primary key
int8_t triggerType; int8_t triggerType;
int64_t watermark; int64_t watermark;
} SWinodwPhysiNode; } SWinodwPhysiNode;
typedef struct SIntervalPhysiNode { typedef struct SIntervalPhysiNode {
SWinodwPhysiNode window; SWinodwPhysiNode window;
int64_t interval; int64_t interval;
int64_t offset; int64_t offset;
int64_t sliding; int64_t sliding;
int8_t intervalUnit; int8_t intervalUnit;
int8_t slidingUnit; int8_t slidingUnit;
SFillNode* pFill; SFillNode* pFill;
} SIntervalPhysiNode; } SIntervalPhysiNode;
typedef struct SMultiTableIntervalPhysiNode { typedef struct SMultiTableIntervalPhysiNode {
SIntervalPhysiNode interval; SIntervalPhysiNode interval;
SNodeList* pPartitionKeys; SNodeList* pPartitionKeys;
} SMultiTableIntervalPhysiNode; } SMultiTableIntervalPhysiNode;
typedef struct SSessionWinodwPhysiNode { typedef struct SSessionWinodwPhysiNode {
SWinodwPhysiNode window; SWinodwPhysiNode window;
int64_t gap; int64_t gap;
} SSessionWinodwPhysiNode; } SSessionWinodwPhysiNode;
typedef struct SStateWinodwPhysiNode { typedef struct SStateWinodwPhysiNode {
SWinodwPhysiNode window; SWinodwPhysiNode window;
SNode* pStateKey; SNode* pStateKey;
} SStateWinodwPhysiNode; } SStateWinodwPhysiNode;
typedef struct SSortPhysiNode { typedef struct SSortPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function
SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
SNodeList* pTargets; SNodeList* pTargets;
} SSortPhysiNode; } SSortPhysiNode;
typedef struct SPartitionPhysiNode { typedef struct SPartitionPhysiNode {
SPhysiNode node; SPhysiNode node;
SNodeList* pExprs; // these are expression list of partition_by_clause SNodeList* pExprs; // these are expression list of partition_by_clause
SNodeList* pPartitionKeys; SNodeList* pPartitionKeys;
SNodeList* pTargets; SNodeList* pTargets;
} SPartitionPhysiNode; } SPartitionPhysiNode;
typedef struct SDataSinkNode { typedef struct SDataSinkNode {
ENodeType type; ENodeType type;
SDataBlockDescNode* pInputDataBlockDesc; SDataBlockDescNode* pInputDataBlockDesc;
} SDataSinkNode; } SDataSinkNode;
@ -313,45 +306,41 @@ typedef struct SDataDispatcherNode {
typedef struct SDataInserterNode { typedef struct SDataInserterNode {
SDataSinkNode sink; SDataSinkNode sink;
int32_t numOfTables; int32_t numOfTables;
uint32_t size; uint32_t size;
char *pData; char* pData;
} SDataInserterNode; } SDataInserterNode;
typedef struct SSubplan { typedef struct SSubplan {
ENodeType type; ENodeType type;
SSubplanId id; // unique id of the subplan SSubplanId id; // unique id of the subplan
ESubplanType subplanType; ESubplanType subplanType;
int32_t msgType; // message type for subplan, used to denote the send message type to vnode. int32_t msgType; // message type for subplan, used to denote the send message type to vnode.
int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner. int32_t level; // the execution level of current subplan, starting from 0 in a top-down manner.
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node SQueryNodeAddr execNode; // for the scan/modify subplan, the optional execution node
SQueryNodeStat execNodeStat; // only for scan subplan SQueryNodeStat execNodeStat; // only for scan subplan
SNodeList* pChildren; // the datasource subplan,from which to fetch the result SNodeList* pChildren; // the datasource subplan,from which to fetch the result
SNodeList* pParents; // the data destination subplan, get data from current subplan SNodeList* pParents; // the data destination subplan, get data from current subplan
SPhysiNode* pNode; // physical plan of current subplan SPhysiNode* pNode; // physical plan of current subplan
SDataSinkNode* pDataSink; // data of the subplan flow into the datasink SDataSinkNode* pDataSink; // data of the subplan flow into the datasink
} SSubplan; } SSubplan;
typedef enum EExplainMode { typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;
EXPLAIN_MODE_DISABLE = 1,
EXPLAIN_MODE_STATIC,
EXPLAIN_MODE_ANALYZE
} EExplainMode;
typedef struct SExplainInfo { typedef struct SExplainInfo {
EExplainMode mode; EExplainMode mode;
bool verbose; bool verbose;
double ratio; double ratio;
} SExplainInfo; } SExplainInfo;
typedef struct SQueryPlan { typedef struct SQueryPlan {
ENodeType type; ENodeType type;
uint64_t queryId; uint64_t queryId;
int32_t numOfSubplans; int32_t numOfSubplans;
SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0. SNodeList* pSubplans; // Element is SNodeListNode. The execution level of subplan, starting from 0.
SExplainInfo explainInfo; SExplainInfo explainInfo;
SNodeList* pPlaceholderValues; SArray* pPlaceholderValues;
} SQueryPlan; } SQueryPlan;
void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext); void nodesWalkPhysiPlan(SNode* pNode, FNodeWalker walker, void* pContext);

269
include/libs/nodes/querynodes.h
View File

@ -25,14 +25,16 @@ extern "C" {
#include "tvariant.h" #include "tvariant.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) #define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema))) #define TABLE_META_SIZE(pMeta) \
#define VGROUPS_INFO_SIZE(pInfo) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo))) (NULL == (pMeta) ? 0 : (sizeof(STableMeta) + TABLE_TOTAL_COL_NUM((pMeta)) * sizeof(SSchema)))
#define VGROUPS_INFO_SIZE(pInfo) \
(NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
typedef struct SRawExprNode { typedef struct SRawExprNode {
ENodeType nodeType; ENodeType nodeType;
char* p; char* p;
uint32_t n; uint32_t n;
SNode* pNode; SNode* pNode;
} SRawExprNode; } SRawExprNode;
typedef struct SDataType { typedef struct SDataType {
@ -45,170 +47,155 @@ typedef struct SDataType {
typedef struct SExprNode { typedef struct SExprNode {
ENodeType type; ENodeType type;
SDataType resType; SDataType resType;
char aliasName[TSDB_COL_NAME_LEN]; char aliasName[TSDB_COL_NAME_LEN];
SNodeList* pAssociationList; SArray* pAssociation;
} SExprNode; } SExprNode;
typedef enum EColumnType { typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG } EColumnType;
COLUMN_TYPE_COLUMN = 1,
COLUMN_TYPE_TAG
} EColumnType;
typedef struct SColumnNode { typedef struct SColumnNode {
SExprNode node; // QUERY_NODE_COLUMN SExprNode node; // QUERY_NODE_COLUMN
uint64_t tableId; uint64_t tableId;
int8_t tableType; int8_t tableType;
col_id_t colId; col_id_t colId;
EColumnType colType; // column or tag EColumnType colType; // column or tag
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
char tableAlias[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN];
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
SNode* pProjectRef; SNode* pProjectRef;
int16_t dataBlockId; int16_t dataBlockId;
int16_t slotId; int16_t slotId;
} SColumnNode; } SColumnNode;
typedef struct STargetNode { typedef struct STargetNode {
ENodeType type; ENodeType type;
int16_t dataBlockId; int16_t dataBlockId;
int16_t slotId; int16_t slotId;
SNode* pExpr; SNode* pExpr;
} STargetNode; } STargetNode;
typedef struct SValueNode { typedef struct SValueNode {
SExprNode node; // QUERY_NODE_VALUE SExprNode node; // QUERY_NODE_VALUE
char* literal; char* literal;
bool isDuration; bool isDuration;
bool translate; bool translate;
bool genByCalc; int16_t placeholderNo;
int16_t placeholderNo;
union { union {
bool b; bool b;
int64_t i; int64_t i;
uint64_t u; uint64_t u;
double d; double d;
char* p; char* p;
} datum; } datum;
char unit; char unit;
} SValueNode; } SValueNode;
typedef struct SOperatorNode { typedef struct SOperatorNode {
SExprNode node; // QUERY_NODE_OPERATOR SExprNode node; // QUERY_NODE_OPERATOR
EOperatorType opType; EOperatorType opType;
SNode* pLeft; SNode* pLeft;
SNode* pRight; SNode* pRight;
} SOperatorNode; } SOperatorNode;
typedef struct SLogicConditionNode { typedef struct SLogicConditionNode {
SExprNode node; // QUERY_NODE_LOGIC_CONDITION SExprNode node; // QUERY_NODE_LOGIC_CONDITION
ELogicConditionType condType; ELogicConditionType condType;
SNodeList* pParameterList; SNodeList* pParameterList;
} SLogicConditionNode; } SLogicConditionNode;
typedef struct SNodeListNode { typedef struct SNodeListNode {
ENodeType type; // QUERY_NODE_NODE_LIST ENodeType type; // QUERY_NODE_NODE_LIST
SDataType dataType; SDataType dataType;
SNodeList* pNodeList; SNodeList* pNodeList;
} SNodeListNode; } SNodeListNode;
typedef struct SFunctionNode { typedef struct SFunctionNode {
SExprNode node; // QUERY_NODE_FUNCTION SExprNode node; // QUERY_NODE_FUNCTION
char functionName[TSDB_FUNC_NAME_LEN]; char functionName[TSDB_FUNC_NAME_LEN];
int32_t funcId; int32_t funcId;
int32_t funcType; int32_t funcType;
SNodeList* pParameterList; SNodeList* pParameterList;
int32_t udfBufSize;
} SFunctionNode; } SFunctionNode;
typedef struct STableNode { typedef struct STableNode {
SExprNode node; SExprNode node;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
char tableAlias[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN];
uint8_t precision; uint8_t precision;
} STableNode; } STableNode;
struct STableMeta; struct STableMeta;
typedef struct SRealTableNode { typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta; struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList; SVgroupsInfo* pVgroupList;
char useDbName[TSDB_DB_NAME_LEN]; char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES
double ratio; double ratio;
} SRealTableNode; } SRealTableNode;
typedef struct STempTableNode { typedef struct STempTableNode {
STableNode table; // QUERY_NODE_TEMP_TABLE STableNode table; // QUERY_NODE_TEMP_TABLE
SNode* pSubquery; SNode* pSubquery;
} STempTableNode; } STempTableNode;
typedef enum EJoinType { typedef enum EJoinType { JOIN_TYPE_INNER = 1 } EJoinType;
JOIN_TYPE_INNER = 1
} EJoinType;
typedef struct SJoinTableNode { typedef struct SJoinTableNode {
STableNode table; // QUERY_NODE_JOIN_TABLE STableNode table; // QUERY_NODE_JOIN_TABLE
EJoinType joinType; EJoinType joinType;
SNode* pLeft; SNode* pLeft;
SNode* pRight; SNode* pRight;
SNode* pOnCond; SNode* pOnCond;
} SJoinTableNode; } SJoinTableNode;
typedef enum EGroupingSetType { typedef enum EGroupingSetType { GP_TYPE_NORMAL = 1 } EGroupingSetType;
GP_TYPE_NORMAL = 1
} EGroupingSetType;
typedef struct SGroupingSetNode { typedef struct SGroupingSetNode {
ENodeType type; // QUERY_NODE_GROUPING_SET ENodeType type; // QUERY_NODE_GROUPING_SET
EGroupingSetType groupingSetType; EGroupingSetType groupingSetType;
SNodeList* pParameterList; SNodeList* pParameterList;
} SGroupingSetNode; } SGroupingSetNode;
typedef enum EOrder { typedef enum EOrder { ORDER_ASC = 1, ORDER_DESC } EOrder;
ORDER_ASC = 1,
ORDER_DESC
} EOrder;
typedef enum ENullOrder { typedef enum ENullOrder { NULL_ORDER_DEFAULT = 1, NULL_ORDER_FIRST, NULL_ORDER_LAST } ENullOrder;
NULL_ORDER_DEFAULT = 1,
NULL_ORDER_FIRST,
NULL_ORDER_LAST
} ENullOrder;
typedef struct SOrderByExprNode { typedef struct SOrderByExprNode {
ENodeType type; // QUERY_NODE_ORDER_BY_EXPR ENodeType type; // QUERY_NODE_ORDER_BY_EXPR
SNode* pExpr; SNode* pExpr;
EOrder order; EOrder order;
ENullOrder nullOrder; ENullOrder nullOrder;
} SOrderByExprNode; } SOrderByExprNode;
typedef struct SLimitNode { typedef struct SLimitNode {
ENodeType type; // QUERY_NODE_LIMIT ENodeType type; // QUERY_NODE_LIMIT
int64_t limit; int64_t limit;
int64_t offset; int64_t offset;
} SLimitNode; } SLimitNode;
typedef struct SStateWindowNode { typedef struct SStateWindowNode {
ENodeType type; // QUERY_NODE_STATE_WINDOW ENodeType type; // QUERY_NODE_STATE_WINDOW
SNode* pCol; // timestamp primary key SNode* pCol; // timestamp primary key
SNode* pExpr; SNode* pExpr;
} SStateWindowNode; } SStateWindowNode;
typedef struct SSessionWindowNode { typedef struct SSessionWindowNode {
ENodeType type; // QUERY_NODE_SESSION_WINDOW ENodeType type; // QUERY_NODE_SESSION_WINDOW
SColumnNode* pCol; // timestamp primary key SColumnNode* pCol; // timestamp primary key
SValueNode* pGap; // gap between two session window(in microseconds) SValueNode* pGap; // gap between two session window(in microseconds)
} SSessionWindowNode; } SSessionWindowNode;
typedef struct SIntervalWindowNode { typedef struct SIntervalWindowNode {
ENodeType type; // QUERY_NODE_INTERVAL_WINDOW ENodeType type; // QUERY_NODE_INTERVAL_WINDOW
SNode* pCol; // timestamp primary key SNode* pCol; // timestamp primary key
SNode* pInterval; // SValueNode SNode* pInterval; // SValueNode
SNode* pOffset; // SValueNode SNode* pOffset; // SValueNode
SNode* pSliding; // SValueNode SNode* pSliding; // SValueNode
SNode* pFill; SNode* pFill;
} SIntervalWindowNode; } SIntervalWindowNode;
typedef enum EFillMode { typedef enum EFillMode {
@ -221,42 +208,40 @@ typedef enum EFillMode {
} EFillMode; } EFillMode;
typedef struct SFillNode { typedef struct SFillNode {
ENodeType type; // QUERY_NODE_FILL ENodeType type; // QUERY_NODE_FILL
EFillMode mode; EFillMode mode;
SNode* pValues; // SNodeListNode SNode* pValues; // SNodeListNode
} SFillNode; } SFillNode;
typedef struct SSelectStmt { typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT ENodeType type; // QUERY_NODE_SELECT_STMT
bool isDistinct; bool isDistinct;
SNodeList* pProjectionList; SNodeList* pProjectionList;
SNode* pFromTable; SNode* pFromTable;
SNode* pWhere; SNode* pWhere;
SNodeList* pPartitionByList; SNodeList* pPartitionByList;
SNode* pWindow; SNode* pWindow;
SNodeList* pGroupByList; // SGroupingSetNode SNodeList* pGroupByList; // SGroupingSetNode
SNode* pHaving; SNode* pHaving;
SNodeList* pOrderByList; // SOrderByExprNode SNodeList* pOrderByList; // SOrderByExprNode
SLimitNode* pLimit; SLimitNode* pLimit;
SLimitNode* pSlimit; SLimitNode* pSlimit;
char stmtName[TSDB_TABLE_NAME_LEN]; char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision; uint8_t precision;
bool isEmptyResult; bool isEmptyResult;
bool hasAggFuncs;
} SSelectStmt; } SSelectStmt;
typedef enum ESetOperatorType { typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
SET_OP_TYPE_UNION_ALL = 1,
SET_OP_TYPE_UNION
} ESetOperatorType;
typedef struct SSetOperator { typedef struct SSetOperator {
ENodeType type; // QUERY_NODE_SET_OPERATOR ENodeType type; // QUERY_NODE_SET_OPERATOR
ESetOperatorType opType; ESetOperatorType opType;
SNodeList* pProjectionList; SNodeList* pProjectionList;
SNode* pLeft; SNode* pLeft;
SNode* pRight; SNode* pRight;
SNodeList* pOrderByList; // SOrderByExprNode SNodeList* pOrderByList; // SOrderByExprNode
SNode* pLimit; SNode* pLimit;
} SSetOperator; } SSetOperator;
typedef enum ESqlClause { typedef enum ESqlClause {
@ -271,7 +256,6 @@ typedef enum ESqlClause {
SQL_CLAUSE_ORDER_BY SQL_CLAUSE_ORDER_BY
} ESqlClause; } ESqlClause;
typedef enum { typedef enum {
PAYLOAD_TYPE_KV = 0, PAYLOAD_TYPE_KV = 0,
PAYLOAD_TYPE_RAW = 1, PAYLOAD_TYPE_RAW = 1,
@ -281,35 +265,38 @@ typedef struct SVgDataBlocks {
SVgroupInfo vg; SVgroupInfo vg;
int32_t numOfTables; // number of tables in current submit block int32_t numOfTables; // number of tables in current submit block
uint32_t size; uint32_t size;
char *pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... char* pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ...
} SVgDataBlocks; } SVgDataBlocks;
typedef struct SVnodeModifOpStmt { typedef struct SVnodeModifOpStmt {
ENodeType nodeType; ENodeType nodeType;
ENodeType sqlNodeType; ENodeType sqlNodeType;
SArray* pDataBlocks; // data block for each vgroup, SArray<SVgDataBlocks*>. SArray* pDataBlocks; // data block for each vgroup, SArray<SVgDataBlocks*>.
uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
uint32_t insertType; // insert data from [file|sql statement| bound statement] uint32_t insertType; // insert data from [file|sql statement| bound statement]
const char* sql; // current sql statement position const char* sql; // current sql statement position
} SVnodeModifOpStmt; } SVnodeModifOpStmt;
typedef struct SExplainOptions { typedef struct SExplainOptions {
ENodeType type; ENodeType type;
bool verbose; bool verbose;
double ratio; double ratio;
} SExplainOptions; } SExplainOptions;
typedef struct SExplainStmt { typedef struct SExplainStmt {
ENodeType type; ENodeType type;
bool analyze; bool analyze;
SExplainOptions* pOptions; SExplainOptions* pOptions;
SNode* pQuery; SNode* pQuery;
} SExplainStmt; } SExplainStmt;
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext); void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, SNodeList** pCols); typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId); typedef bool (*FFuncClassifier)(int32_t funcId);
int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs); int32_t nodesCollectFuncs(SSelectStmt* pSelect, FFuncClassifier classifier, SNodeList** pFuncs);
@ -324,10 +311,10 @@ bool nodesIsJsonOp(const SOperatorNode* pOp);
bool nodesIsTimeorderQuery(const SNode* pQuery); bool nodesIsTimeorderQuery(const SNode* pQuery);
bool nodesIsTimelineQuery(const SNode* pQuery); bool nodesIsTimelineQuery(const SNode* pQuery);
void* nodesGetValueFromNode(SValueNode *pNode); void* nodesGetValueFromNode(SValueNode* pNode);
char* nodesGetStrValueFromNode(SValueNode *pNode); char* nodesGetStrValueFromNode(SValueNode* pNode);
char *getFillModeString(EFillMode mode); char* getFillModeString(EFillMode mode);
void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal); void valueNodeToVariant(const SValueNode* pNode, SVariant* pVal);
#ifdef __cplusplus #ifdef __cplusplus
} }

7
include/libs/parser/parser.h
View File

@ -73,6 +73,7 @@ typedef struct SQuery {
SArray* pDbList; SArray* pDbList;
SArray* pTableList; SArray* pTableList;
bool showRewrite; bool showRewrite;
int32_t placeholderNum;
} SQuery; } SQuery;
int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery); int32_t qParseQuerySql(SParseContext* pCxt, SQuery** pQuery);
@ -88,11 +89,11 @@ int32_t qCloneStmtDataBlock(void** pDst, void* pSrc);
void qFreeStmtDataBlock(void* pDataBlock); void qFreeStmtDataBlock(void* pDataBlock);
int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc); int32_t qRebuildStmtDataBlock(void** pDst, void* pSrc);
void qDestroyStmtDataBlock(void* pBlock); void qDestroyStmtDataBlock(void* pBlock);
int32_t qBindStmtColsValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen);
int32_t qBindStmtSingleColValue(void *pBlock, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum); int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum);
int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtColFields(void *pDataBlock, int32_t *fieldNum, TAOS_FIELD** fields);
int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields); int32_t qBuildStmtTagFields(void *pBlock, void *boundTags, int32_t *fieldNum, TAOS_FIELD** fields);
int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_BIND_v2 *bind, char *msgBuf, int32_t msgBufLen); int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen);
void destroyBoundColumnInfo(void* pBoundInfo); void destroyBoundColumnInfo(void* pBoundInfo);
int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen); int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char *msgBuf, int32_t msgBufLen);

4
include/libs/planner/planner.h
View File

@ -34,7 +34,7 @@ typedef struct SPlanContext {
bool showRewrite; bool showRewrite;
int8_t triggerType; int8_t triggerType;
int64_t watermark; int64_t watermark;
bool isStmtQuery; int32_t placeholderNum;
void* pTransporter; void* pTransporter;
struct SCatalog* pCatalog; struct SCatalog* pCatalog;
char* pMsg; char* pMsg;
@ -50,7 +50,7 @@ int32_t qCreateQueryPlan(SPlanContext* pCxt, SQueryPlan** pPlan, SArray* pExecNo
// @pSource one execution location of this group of datasource subplans // @pSource one execution location of this group of datasource subplans
int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource); int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstreamSourceNode* pSource);
int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_BIND_v2* pParams); int32_t qStmtBindParam(SQueryPlan* pPlan, TAOS_MULTI_BIND* pParams, int32_t colIdx, uint64_t queryId);
// Convert to subplan to string for the scheduler to send to the executor // Convert to subplan to string for the scheduler to send to the executor
int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen);

46
include/libs/stream/tstreamUpdate.h Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TSTREAMUPDATE_H_
#define _TSTREAMUPDATE_H_
#include "taosdef.h"
#include "tarray.h"
#include "tmsg.h"
#include "tscalablebf.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SUpdateInfo {
SArray *pTsBuckets;
uint64_t numBuckets;
SArray *pTsSBFs;
uint64_t numSBFs;
int64_t interval;
int64_t watermark;
TSKEY minTS;
} SUpdateInfo;
SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark);
SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark);
bool isUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts);
void updateInfoDestroy(SUpdateInfo *pInfo);
#ifdef __cplusplus
}
#endif
#endif /* ifndef _TSTREAMUPDATE_H_ */

2
include/libs/sync/sync.h
View File

@ -158,6 +158,8 @@ typedef enum {
int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
bool syncEnvIsStart();
extern int32_t sDebugFlag; extern int32_t sDebugFlag;
//----------------------------------------- //-----------------------------------------

8
include/libs/transport/trpc.h
View File

@ -59,9 +59,13 @@ typedef struct {
void * pNode; void * pNode;
} SNodeMsg; } SNodeMsg;
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *); typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *rf);
typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey); typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *); ///
// // SRpcMsg code
// REDIERE,
// NOT READY, EpSet
typedef bool (*RpcRfp)(int32_t code);
typedef struct SRpcInit { typedef struct SRpcInit {
uint16_t localPort; // local port uint16_t localPort; // local port

2
include/libs/wal/wal.h
View File

@ -72,8 +72,6 @@ extern "C" {
#define WAL_FILE_LEN (WAL_PATH_LEN + 32) #define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_MAGIC 0xFAFBFCFDULL #define WAL_MAGIC 0xFAFBFCFDULL
#define WAL_CUR_FAILED 1
#pragma pack(push, 1) #pragma pack(push, 1)
typedef enum { typedef enum {
TAOS_WAL_NOLOG = 0, TAOS_WAL_NOLOG = 0,

1
include/os/os.h
View File

@ -59,6 +59,7 @@ extern "C" {
#include <winsock.h> #include <winsock.h>
#endif #endif
#define __typeof(a) auto
#endif #endif

32
include/os/osMath.h
View File

@ -23,27 +23,21 @@ extern "C" {
#define TPOW2(x) ((x) * (x)) #define TPOW2(x) ((x) * (x))
#define TABS(x) ((x) > 0 ? (x) : -(x)) #define TABS(x) ((x) > 0 ? (x) : -(x))
#define TSWAP(a, b) \
do { \
__typeof(a) __tmp = (a); \
(a) = (b); \
(b) = __tmp; \
} while (0)
#ifdef WINDOWS #ifdef WINDOWS
#define TSWAP(a, b, c) \
do { \
c __tmp = (c)(a); \
(a) = (c)(b); \
(b) = __tmp; \
} while (0)
#define TMAX(a, b) (((a) > (b)) ? (a) : (b)) #define TMAX(a, b) (((a) > (b)) ? (a) : (b))
#define TMIN(a, b) (((a) < (b)) ? (a) : (b)) #define TMIN(a, b) (((a) < (b)) ? (a) : (b))
#define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc))) #define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
#else #else
#define TSWAP(a, b, c) \
do { \
__typeof(a) __tmp = (a); \
(a) = (b); \
(b) = __tmp; \
} while (0)
#define TMAX(a, b) \ #define TMAX(a, b) \
({ \ ({ \
__typeof(a) __a = (a); \ __typeof(a) __a = (a); \
@ -51,12 +45,12 @@ extern "C" {
(__a > __b) ? __a : __b; \ (__a > __b) ? __a : __b; \
}) })
#define TMIN(a, b) \ #define TMIN(a, b) \
({ \ ({ \
__typeof(a) __a = (a); \ __typeof(a) __a = (a); \
__typeof(b) __b = (b); \ __typeof(b) __b = (b); \
(__a < __b) ? __a : __b; \ (__a < __b) ? __a : __b; \
}) })
#define TRANGE(a, b, c) \ #define TRANGE(a, b, c) \
({ \ ({ \

163
include/os/osSocket.h
View File

@ -19,53 +19,53 @@
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section. // When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC #ifndef ALLOW_FORBID_FUNC
#define socket SOCKET_FUNC_TAOS_FORBID #define socket SOCKET_FUNC_TAOS_FORBID
#define bind BIND_FUNC_TAOS_FORBID #define bind BIND_FUNC_TAOS_FORBID
#define listen LISTEN_FUNC_TAOS_FORBID #define listen LISTEN_FUNC_TAOS_FORBID
#define accept ACCEPT_FUNC_TAOS_FORBID #define accept ACCEPT_FUNC_TAOS_FORBID
#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
#define inet_addr INET_ADDR_FUNC_TAOS_FORBID #define inet_addr INET_ADDR_FUNC_TAOS_FORBID
#define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
#endif #endif
#if defined(WINDOWS) #if defined(WINDOWS)
#if BYTE_ORDER == LITTLE_ENDIAN #if BYTE_ORDER == LITTLE_ENDIAN
#include <stdlib.h> #include <stdlib.h>
#define htobe16(x) _byteswap_ushort(x) #define htobe16(x) _byteswap_ushort(x)
#define htole16(x) (x) #define htole16(x) (x)
#define be16toh(x) _byteswap_ushort(x) #define be16toh(x) _byteswap_ushort(x)
#define le16toh(x) (x) #define le16toh(x) (x)
#define htobe32(x) _byteswap_ulong(x)
#define htole32(x) (x)
#define be32toh(x) _byteswap_ulong(x)
#define le32toh(x) (x)
#define htobe64(x) _byteswap_uint64(x)
#define htole64(x) (x)
#define be64toh(x) _byteswap_uint64(x)
#define le64toh(x) (x)
#else
#error byte order not supported
#endif
#define __BYTE_ORDER BYTE_ORDER #define htobe32(x) _byteswap_ulong(x)
#define __BIG_ENDIAN BIG_ENDIAN #define htole32(x) (x)
#define __LITTLE_ENDIAN LITTLE_ENDIAN #define be32toh(x) _byteswap_ulong(x)
#define __PDP_ENDIAN PDP_ENDIAN #define le32toh(x) (x)
#define htobe64(x) _byteswap_uint64(x)
#define htole64(x) (x)
#define be64toh(x) _byteswap_uint64(x)
#define le64toh(x) (x)
#else
#error byte order not supported
#endif
#define __BYTE_ORDER BYTE_ORDER
#define __BIG_ENDIAN BIG_ENDIAN
#define __LITTLE_ENDIAN LITTLE_ENDIAN
#define __PDP_ENDIAN PDP_ENDIAN
#else #else
#include <netinet/in.h> #include <netinet/in.h>
#include <sys/socket.h> #include <sys/socket.h>
#if defined(_TD_DARWIN_64) #if defined(_TD_DARWIN_64)
#include <osEok.h> #include <osEok.h>
#else #else
#include <netinet/in.h> #include <netinet/in.h>
#include <sys/epoll.h> #include <sys/epoll.h>
#endif #endif
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus
@ -73,24 +73,24 @@ extern "C" {
#endif #endif
#if defined(WINDOWS) #if defined(WINDOWS)
typedef int socklen_t; typedef int socklen_t;
#define TAOS_EPOLL_WAIT_TIME 100 #define TAOS_EPOLL_WAIT_TIME 100
typedef SOCKET eventfd_t; typedef SOCKET eventfd_t;
#define eventfd(a, b) -1 #define eventfd(a, b) -1
#define EpollClose(pollFd) epoll_close(pollFd) #define EpollClose(pollFd) epoll_close(pollFd)
#ifndef EPOLLWAKEUP #ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29) #define EPOLLWAKEUP (1u << 29)
#endif #endif
#elif defined(_TD_DARWIN_64) #elif defined(_TD_DARWIN_64)
#define TAOS_EPOLL_WAIT_TIME 500 #define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SOCKET; typedef int32_t SOCKET;
typedef SOCKET EpollFd; typedef SOCKET EpollFd;
#define EpollClose(pollFd) epoll_close(pollFd) #define EpollClose(pollFd) epoll_close(pollFd)
#else #else
#define TAOS_EPOLL_WAIT_TIME 500 #define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SOCKET; typedef int32_t SOCKET;
typedef SOCKET EpollFd; typedef SOCKET EpollFd;
#define EpollClose(pollFd) taosCloseSocket(pollFd) #define EpollClose(pollFd) taosCloseSocket(pollFd)
#endif #endif
#if defined(_TD_DARWIN_64) #if defined(_TD_DARWIN_64)
@ -119,8 +119,8 @@ extern "C" {
#define __PDP_ENDIAN PDP_ENDIAN #define __PDP_ENDIAN PDP_ENDIAN
#endif #endif
typedef int32_t SocketFd; typedef int32_t SocketFd;
typedef SocketFd EpollFd; typedef SocketFd EpollFd;
typedef struct TdSocket { typedef struct TdSocket {
#if SOCKET_WITH_LOCK #if SOCKET_WITH_LOCK
@ -128,16 +128,17 @@ typedef struct TdSocket {
#endif #endif
int refId; int refId;
SocketFd fd; SocketFd fd;
} *TdSocketPtr, TdSocket; } * TdSocketPtr, TdSocket;
typedef struct TdSocketServer *TdSocketServerPtr; typedef struct TdSocketServer *TdSocketServerPtr;
typedef struct TdSocket *TdSocketPtr; typedef struct TdSocket * TdSocketPtr;
typedef struct TdEpoll *TdEpollPtr; typedef struct TdEpoll * TdEpollPtr;
int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen); int32_t taosSendto(TdSocketPtr pSocket, void *msg, int len, unsigned int flags, const struct sockaddr *to, int tolen);
int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len); int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen); int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr,
int *addrLen);
int32_t taosCloseSocketNoCheck1(SocketFd fd); int32_t taosCloseSocketNoCheck1(SocketFd fd);
int32_t taosCloseSocket(TdSocketPtr *ppSocket); int32_t taosCloseSocket(TdSocketPtr *ppSocket);
int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer); int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@ -154,30 +155,32 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes); int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
void taosWinSocketInit(); void taosWinSocketInit();
TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec);
TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int32_t taosKeepTcpAlive(TdSocketPtr pSocket); int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen); int32_t taosGetSocketName(TdSocketPtr pSocket, struct sockaddr *destAddr, int *addrLen);
void taosBlockSIGPIPE(); void taosBlockSIGPIPE();
uint32_t taosGetIpv4FromFqdn(const char *); uint32_t taosGetIpv4FromFqdn(const char *);
int32_t taosGetFqdn(char *); int32_t taosGetFqdn(char *);
void tinet_ntoa(char *ipstr, uint32_t ip); void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr); uint32_t ip2uint(const char *const ip_addr);
void taosIgnSIGPIPE(); void taosIgnSIGPIPE();
void taosSetMaskSIGPIPE(); void taosSetMaskSIGPIPE();
uint32_t taosInetAddr(const char *ipAddr); uint32_t taosInetAddr(const char *ipAddr);
const char *taosInetNtoa(struct in_addr ipInt); const char *taosInetNtoa(struct in_addr ipInt);
TdEpollPtr taosCreateEpoll(int32_t size); TdEpollPtr taosCreateEpoll(int32_t size);
int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event); int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout); int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
int32_t taosCloseEpoll(TdEpollPtr *ppEpoll); int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
#ifdef __cplusplus #ifdef __cplusplus
} }

32
include/util/taoserror.h
View File

@ -62,6 +62,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014) #define TSDB_CODE_APP_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0014)
#define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015) #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0015)
#define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016) #define TSDB_CODE_RPC_INVALID_VERSION TAOS_DEF_ERROR_CODE(0, 0x0016)
#define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0017)
//common & util //common & util
#define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0100) #define TSDB_CODE_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0100)
@ -139,16 +140,9 @@ int32_t* taosGetErrno();
// mnode-common // mnode-common
#define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300)
#define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301) #define TSDB_CODE_MND_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0301)
#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0302) #define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0302)
#define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0303) #define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303)
#define TSDB_CODE_MND_ACTION_NEED_REPROCESSED TAOS_DEF_ERROR_CODE(0, 0x0304) #define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0304)
#define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0305)
#define TSDB_CODE_MND_INVALID_OPTIONS TAOS_DEF_ERROR_CODE(0, 0x0306)
#define TSDB_CODE_MND_INVALID_CONNECTION TAOS_DEF_ERROR_CODE(0, 0x0307)
#define TSDB_CODE_MND_INVALID_MSG_VERSION TAOS_DEF_ERROR_CODE(0, 0x0308)
#define TSDB_CODE_MND_INVALID_MSG_LEN TAOS_DEF_ERROR_CODE(0, 0x0309)
#define TSDB_CODE_MND_INVALID_MSG_TYPE TAOS_DEF_ERROR_CODE(0, 0x030A)
#define TSDB_CODE_MND_TOO_MANY_SHELL_CONNS TAOS_DEF_ERROR_CODE(0, 0x030B)
// mnode-show // mnode-show
#define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310) #define TSDB_CODE_MND_INVALID_SHOWOBJ TAOS_DEF_ERROR_CODE(0, 0x0310)
@ -275,14 +269,13 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TOPIC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E1) #define TSDB_CODE_MND_TOPIC_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E1)
#define TSDB_CODE_MND_TOO_MANY_TOPICS TAOS_DEF_ERROR_CODE(0, 0x03E2) #define TSDB_CODE_MND_TOO_MANY_TOPICS TAOS_DEF_ERROR_CODE(0, 0x03E2)
#define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E3) #define TSDB_CODE_MND_INVALID_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E3)
#define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E4) #define TSDB_CODE_MND_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x03E4)
#define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E5) #define TSDB_CODE_MND_INVALID_TOPIC_OPTION TAOS_DEF_ERROR_CODE(0, 0x03E5)
#define TSDB_CODE_MND_NAME_CONFLICT_WITH_STB TAOS_DEF_ERROR_CODE(0, 0x03E6) #define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E6)
#define TSDB_CODE_MND_CONSUMER_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E7) #define TSDB_CODE_MND_TOPIC_OPTION_UNCHNAGED TAOS_DEF_ERROR_CODE(0, 0x03E7)
#define TSDB_CODE_MND_UNSUPPORTED_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03E8) #define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E8)
#define TSDB_CODE_MND_SUBSCRIBE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9) #define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03E9)
#define TSDB_CODE_MND_OFFSET_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03EA) #define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EA)
#define TSDB_CODE_MND_CONSUMER_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x03EB)
// mnode-stream // mnode-stream
#define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0)
@ -573,7 +566,6 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) #define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602)
#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603)
#define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) #define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604)
#define TSDB_CODE_PAR_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2605)
#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608)
#define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609)
#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A)
@ -594,7 +586,6 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INVALID_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619) #define TSDB_CODE_PAR_INVALID_RANGE_OPTION TAOS_DEF_ERROR_CODE(0, 0x2619)
#define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x261A) #define TSDB_CODE_PAR_INVALID_STR_OPTION TAOS_DEF_ERROR_CODE(0, 0x261A)
#define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x261B) #define TSDB_CODE_PAR_INVALID_ENUM_OPTION TAOS_DEF_ERROR_CODE(0, 0x261B)
#define TSDB_CODE_PAR_INVALID_TTL_OPTION TAOS_DEF_ERROR_CODE(0, 0x261C)
#define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x261D) #define TSDB_CODE_PAR_INVALID_KEEP_NUM TAOS_DEF_ERROR_CODE(0, 0x261D)
#define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x261E) #define TSDB_CODE_PAR_INVALID_KEEP_ORDER TAOS_DEF_ERROR_CODE(0, 0x261E)
#define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x261F) #define TSDB_CODE_PAR_INVALID_KEEP_VALUE TAOS_DEF_ERROR_CODE(0, 0x261F)
@ -633,6 +624,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801) #define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801)
#define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802)
#define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803)
#define TSDB_CODE_FUNC_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2604)
#ifdef __cplusplus #ifdef __cplusplus
} }

10
include/util/tarray.h
View File

@ -205,7 +205,6 @@ SArray* taosArrayDup(const SArray* pSrc);
*/ */
SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy); SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy);
/** /**
* clear the array (remove all element) * clear the array (remove all element)
* @param pArray * @param pArray
@ -219,6 +218,13 @@ void taosArrayClear(SArray* pArray);
*/ */
void taosArrayClearEx(SArray* pArray, void (*fp)(void*)); void taosArrayClearEx(SArray* pArray, void (*fp)(void*));
/**
* clear the array (remove all element)
* @param pArray
* @param fp
*/
void taosArrayClearP(SArray* pArray, FDelete fp);
void* taosArrayDestroy(SArray* pArray); void* taosArrayDestroy(SArray* pArray);
void taosArrayDestroyP(SArray* pArray, FDelete fp); void taosArrayDestroyP(SArray* pArray, FDelete fp);
void taosArrayDestroyEx(SArray* pArray, FDelete fp); void taosArrayDestroyEx(SArray* pArray, FDelete fp);
@ -272,6 +278,8 @@ void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* par
int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode); int32_t taosEncodeArray(void** buf, const SArray* pArray, FEncode encode);
void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz); void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t dataSz);
char* taosShowStrArray(const SArray* pArray);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

50
include/util/tbloomfilter.h Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_BLOOMFILTER_H_
#define _TD_UTIL_BLOOMFILTER_H_
#include "os.h"
#include "thash.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SBloomFilter {
uint32_t hashFunctions;
uint64_t expectedEntries;
uint64_t numUnits;
uint64_t numBits;
uint64_t size;
_hash_fn_t hashFn1;
_hash_fn_t hashFn2;
void *buffer;
double errorRate;
} SBloomFilter;
SBloomFilter *tBloomFilterInit(uint64_t expectedEntries, double errorRate);
int32_t tBloomFilterPut(SBloomFilter *pBF, const void *keyBuf, uint32_t len);
int32_t tBloomFilterNoContain(const SBloomFilter *pBF, const void *keyBuf,
uint32_t len);
void tBloomFilterDestroy(SBloomFilter *pBF);
void tBloomFilterDump(const SBloomFilter *pBF);
bool tBloomFilterIsFull(const SBloomFilter *pBF);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_BLOOMFILTER_H_*/

157
include/util/tdef.h
View File

@ -125,12 +125,13 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_INS_TABLE_QUERIES "queries" #define TSDB_INS_TABLE_QUERIES "queries"
#define TSDB_INS_TABLE_VNODES "vnodes" #define TSDB_INS_TABLE_VNODES "vnodes"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_CONNECTIONS "connections" #define TSDB_PERFS_TABLE_CONNECTIONS "connections"
#define TSDB_PERFS_TABLE_QUERIES "queries" #define TSDB_PERFS_TABLE_QUERIES "queries"
#define TSDB_PERFS_TABLE_TOPICS "topics" #define TSDB_PERFS_TABLE_TOPICS "topics"
#define TSDB_PERFS_TABLE_CONSUMERS "consumers" #define TSDB_PERFS_TABLE_CONSUMERS "consumers"
#define TSDB_PERFS_TABLE_SUBSCRIBES "subscribes" #define TSDB_PERFS_TABLE_SUBSCRIPTIONS "subscriptions"
#define TSDB_PERFS_TABLE_OFFSETS "offsets"
#define TSDB_INDEX_TYPE_SMA "SMA" #define TSDB_INDEX_TYPE_SMA "SMA"
#define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT"
@ -195,6 +196,8 @@ typedef enum EOperatorType {
OP_TYPE_JSON_CONTAINS OP_TYPE_JSON_CONTAINS
} EOperatorType; } EOperatorType;
#define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR
typedef enum ELogicConditionType { typedef enum ELogicConditionType {
LOGIC_COND_TYPE_AND = 1, LOGIC_COND_TYPE_AND = 1,
LOGIC_COND_TYPE_OR, LOGIC_COND_TYPE_OR,
@ -270,7 +273,7 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_TAGS 128 #define TSDB_MAX_TAGS 128
#define TSDB_MAX_TAG_CONDITIONS 1024 #define TSDB_MAX_TAG_CONDITIONS 1024
#define TSDB_MAX_JSON_TAG_LEN 16384 #define TSDB_MAX_JSON_TAG_LEN 16384
#define TSDB_AUTH_LEN 16 #define TSDB_AUTH_LEN 16
#define TSDB_PASSWORD_LEN 32 #define TSDB_PASSWORD_LEN 32
@ -284,8 +287,9 @@ typedef enum ELogicConditionType {
#define TSDB_IPv4ADDR_LEN 16 #define TSDB_IPv4ADDR_LEN 16
#define TSDB_FILENAME_LEN 128 #define TSDB_FILENAME_LEN 128
#define TSDB_SHOW_SQL_LEN 512 #define TSDB_SHOW_SQL_LEN 512
#define TSDB_SHOW_SUBQUERY_LEN 1000
#define TSDB_SLOW_QUERY_SQL_LEN 512 #define TSDB_SLOW_QUERY_SQL_LEN 512
#define TSDB_SHOW_SUBQUERY_LEN 1000
#define TSDB_SHOW_LIST_LEN 1000
#define TSDB_TRANS_STAGE_LEN 12 #define TSDB_TRANS_STAGE_LEN 12
#define TSDB_TRANS_TYPE_LEN 16 #define TSDB_TRANS_TYPE_LEN 16
@ -319,78 +323,73 @@ typedef enum ELogicConditionType {
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta
#define TSDB_MIN_VNODES_PER_DB 1 #define TSDB_MIN_VNODES_PER_DB 1
#define TSDB_MAX_VNODES_PER_DB 4096 #define TSDB_MAX_VNODES_PER_DB 4096
#define TSDB_DEFAULT_VN_PER_DB 2 #define TSDB_DEFAULT_VN_PER_DB 2
#define TSDB_MIN_STBS_PER_DB 0 #define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB
#define TSDB_MAX_STBS_PER_DB 1 #define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB
#define TSDB_DEFAULT_STBS_PER_DB 0 #define TSDB_DEFAULT_BUFFER_PER_VNODE 96
#define TSDB_MIN_BUFFER_SIZE 1 #define TSDB_MIN_PAGES_PER_VNODE 64
#define TSDB_MAX_BUFFER_SIZE 16384 #define TSDB_MAX_PAGES_PER_VNODE 16384
#define TSDB_DEFAULT_BUFFER_SIZE 96 // 96MB for each vnode #define TSDB_DEFAULT_PAGES_PER_VNODE 256
#define TSDB_MIN_PAGE_SIZE 1 // 1KB #define TSDB_MIN_PAGESIZE_PER_VNODE 1 // unit KB
#define TSDB_MAX_PAGE_SIZE 16384 // 16MB #define TSDB_MAX_PAGESIZE_PER_VNODE 16384
#define TSDB_DEFAULT_PAGE_SIZE 4 #define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4
#define TSDB_MIN_TOTAL_PAGES 64 #define TSDB_MIN_DAYS_PER_FILE 60 // unit minute
#define TSDB_MAX_TOTAL_PAGES 16384 #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440)
#define TSDB_DEFAULT_TOTAL_PAGES 256 #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
#define TSDB_MIN_DURATION_PER_FILE 60 // unit minute #define TSDB_MIN_DURATION_PER_FILE 60 // unit minute
#define TSDB_MAX_DURATION_PER_FILE (3650 * 1440) #define TSDB_MAX_DURATION_PER_FILE (3650 * 1440)
#define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) #define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440)
#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. #define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years #define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10 #define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000 #define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100 #define TSDB_DEFAULT_MINROWS_FBLOCK 100
#define TSDB_MIN_MAXROWS_FBLOCK 200 #define TSDB_MIN_MAXROWS_FBLOCK 200
#define TSDB_MAX_MAXROWS_FBLOCK 10000 #define TSDB_MAX_MAXROWS_FBLOCK 10000
#define TSDB_DEFAULT_MAXROWS_FBLOCK 4096 #define TSDB_DEFAULT_MAXROWS_FBLOCK 4096
#define TSDB_MIN_COMMIT_TIME 30 #define TSDB_MIN_FSYNC_PERIOD 0
#define TSDB_MAX_COMMIT_TIME 40960 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond
#define TSDB_DEFAULT_COMMIT_TIME 3600 #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second
#define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MIN_WAL_LEVEL 1
#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_MAX_WAL_LEVEL 2
#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second #define TSDB_DEFAULT_WAL_LEVEL 1
#define TSDB_MIN_DB_TTL 1 #define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_DEFAULT_DB_TTL 1 #define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO
#define TSDB_MIN_WAL_LEVEL 1 #define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI
#define TSDB_MAX_WAL_LEVEL 2 #define TSDB_MIN_COMP_LEVEL 0
#define TSDB_DEFAULT_WAL_LEVEL 1 #define TSDB_MAX_COMP_LEVEL 2
#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI #define TSDB_DEFAULT_COMP_LEVEL 2
#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO #define TSDB_MIN_DB_REPLICA 1
#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI #define TSDB_MAX_DB_REPLICA 3
#define TSDB_MIN_COMP_LEVEL 0 #define TSDB_DEFAULT_DB_REPLICA 1
#define TSDB_MAX_COMP_LEVEL 2 #define TSDB_DB_STRICT_OFF 0
#define TSDB_DEFAULT_COMP_LEVEL 2 #define TSDB_DB_STRICT_ON 1
#define TSDB_MIN_DB_REPLICA 1 #define TSDB_DEFAULT_DB_STRICT 0
#define TSDB_MAX_DB_REPLICA 3 #define TSDB_MIN_DB_CACHE_LAST_ROW 0
#define TSDB_DEFAULT_DB_REPLICA 1 #define TSDB_MAX_DB_CACHE_LAST_ROW 3
#define TSDB_DB_STRICT_OFF 0 #define TSDB_DEFAULT_CACHE_LAST_ROW 0
#define TSDB_DB_STRICT_ON 1 #define TSDB_DB_STREAM_MODE_OFF 0
#define TSDB_DEFAULT_DB_STRICT 0 #define TSDB_DB_STREAM_MODE_ON 1
#define TSDB_MIN_DB_UPDATE 0 #define TSDB_DEFAULT_DB_STREAM_MODE 0
#define TSDB_MAX_DB_UPDATE 2 #define TSDB_DB_SINGLE_STABLE_ON 0
#define TSDB_DEFAULT_DB_UPDATE 0 #define TSDB_DB_SINGLE_STABLE_OFF 1
#define TSDB_MIN_DB_CACHE_LAST_ROW 0 #define TSDB_DEFAULT_DB_SINGLE_STABLE 0
#define TSDB_MAX_DB_CACHE_LAST_ROW 3
#define TSDB_DEFAULT_CACHE_LAST_ROW 0
#define TSDB_DB_STREAM_MODE_OFF 0
#define TSDB_DB_STREAM_MODE_ON 1
#define TSDB_DEFAULT_DB_STREAM_MODE 0
#define TSDB_DB_SINGLE_STABLE_ON 0
#define TSDB_DB_SINGLE_STABLE_OFF 1
#define TSDB_DEFAULT_DB_SINGLE_STABLE 0
#define TSDB_MIN_DB_FILE_FACTOR 0 #define TSDB_MIN_ROLLUP_FILE_FACTOR 0
#define TSDB_MAX_DB_FILE_FACTOR 1 #define TSDB_MAX_ROLLUP_FILE_FACTOR 1
#define TSDB_DEFAULT_DB_FILE_FACTOR 0.1 #define TSDB_DEFAULT_ROLLUP_FILE_FACTOR 0.1
#define TSDB_MIN_DB_DELAY 1 #define TSDB_MIN_ROLLUP_DELAY 1
#define TSDB_MAX_DB_DELAY 10 #define TSDB_MAX_ROLLUP_DELAY 10
#define TSDB_DEFAULT_DB_DELAY 2 #define TSDB_DEFAULT_ROLLUP_DELAY 2
#define TSDB_MIN_EXPLAIN_RATIO 0 #define TSDB_MIN_TABLE_TTL 0
#define TSDB_MAX_EXPLAIN_RATIO 1 #define TSDB_DEFAULT_TABLE_TTL 0
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_MIN_EXPLAIN_RATIO 0
#define TSDB_MAX_EXPLAIN_RATIO 1
#define TSDB_DEFAULT_EXPLAIN_RATIO 0.001
#define TSDB_MAX_JOIN_TABLE_NUM 10 #define TSDB_MAX_JOIN_TABLE_NUM 10
#define TSDB_MAX_UNION_CLAUSE 5 #define TSDB_MAX_UNION_CLAUSE 5

60
include/util/tencode.h
View File

@ -17,7 +17,8 @@
#define _TD_UTIL_ENCODE_H_ #define _TD_UTIL_ENCODE_H_
#include "tcoding.h" #include "tcoding.h"
#include "tfreelist.h" #include "tlist.h"
// #include "tfreelist.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -62,10 +63,14 @@ struct SCoderNode {
CODER_NODE_FIELDS CODER_NODE_FIELDS
}; };
typedef struct SCoderMem {
struct SCoderMem* next;
} SCoderMem;
typedef struct { typedef struct {
td_coder_t type; td_coder_t type;
td_endian_t endian; td_endian_t endian;
SFreeList fl; SCoderMem* mList;
CODER_NODE_FIELDS CODER_NODE_FIELDS
TD_SLIST(SCoderNode) stack; TD_SLIST(SCoderNode) stack;
} SCoder; } SCoder;
@ -74,7 +79,52 @@ typedef struct {
#define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos) #define TD_CODER_CURRENT(CODER) ((CODER)->data + (CODER)->pos)
#define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE)) #define TD_CODER_MOVE_POS(CODER, MOVE) ((CODER)->pos += (MOVE))
#define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE)) #define TD_CODER_CHECK_CAPACITY_FAILED(CODER, EXPSIZE) (((CODER)->size - (CODER)->pos) < (EXPSIZE))
#define TCODER_MALLOC(PTR, TYPE, SIZE, CODER) TFL_MALLOC(PTR, TYPE, SIZE, &((CODER)->fl)) // #define TCODER_MALLOC(PCODER, SIZE) \
// ({ \
// void* ptr = NULL; \
// SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(*pMem) + (SIZE)); \
// if (pMem) { \
// pMem->next = (PCODER)->mList; \
// (PCODER)->mList = pMem; \
// ptr = (void*)&pMem[1]; \
// } \
// ptr; \
// })
static FORCE_INLINE void* tCoderMalloc(SCoder* pCoder, int32_t size) {
void* ptr = NULL;
SCoderMem* pMem = (SCoderMem*)taosMemoryMalloc(sizeof(SCoderMem*) + size);
if (pMem) {
pMem->next = pCoder->mList;
pCoder->mList = pMem;
ptr = (void*)&pMem[1];
}
return ptr;
}
#define tEncodeSize(E, S, SIZE, RET) \
do{ \
SCoder coder = {0}; \
tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
if ((E)(&coder, S) == 0) { \
SIZE = coder.pos; \
} else { \
RET = -1; \
} \
tCoderClear(&coder); \
}while(0)
// #define tEncodeSize(E, S, SIZE) \
// ({ \
// SCoder coder = {0}; \
// int ret = 0; \
// tCoderInit(&coder, TD_LITTLE_ENDIAN, NULL, 0, TD_ENCODER); \
// if ((E)(&coder, S) == 0) { \
// SIZE = coder.pos; \
// } else { \
// ret = -1; \
// } \
// tCoderClear(&coder); \
// ret; \
// })
void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type); void tCoderInit(SCoder* pCoder, td_endian_t endian, uint8_t* data, int32_t size, td_coder_t type);
void tCoderClear(SCoder* pCoder); void tCoderClear(SCoder* pCoder);
@ -377,7 +427,9 @@ static FORCE_INLINE int32_t tDecodeBinary(SCoder* pDecoder, const void** val, ui
if (tDecodeU64v(pDecoder, len) < 0) return -1; if (tDecodeU64v(pDecoder, len) < 0) return -1;
if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1; if (TD_CODER_CHECK_CAPACITY_FAILED(pDecoder, *len)) return -1;
*val = (void*)TD_CODER_CURRENT(pDecoder); if (val) {
*val = (void*)TD_CODER_CURRENT(pDecoder);
}
TD_CODER_MOVE_POS(pDecoder, *len); TD_CODER_MOVE_POS(pDecoder, *len);
return 0; return 0;

3
include/util/tlog.h
View File

@ -59,6 +59,7 @@ extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag; extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag; extern int32_t tqDebugFlag;
extern int32_t fsDebugFlag; extern int32_t fsDebugFlag;
extern int32_t metaDebugFlag;
extern int32_t fnDebugFlag; extern int32_t fnDebugFlag;
int32_t taosInitLog(const char *logName, int32_t maxFiles); int32_t taosInitLog(const char *logName, int32_t maxFiles);
@ -79,6 +80,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#endif #endif
; ;
// clang-format off
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uError(...) { if (uDebugFlag & DEBUG_ERROR) { taosPrintLog("UTL ERROR ", DEBUG_ERROR, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
#define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }} #define uWarn(...) { if (uDebugFlag & DEBUG_WARN) { taosPrintLog("UTL WARN ", DEBUG_WARN, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}
@ -88,6 +90,7 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); } #define pError(...) { taosPrintLog("APP ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }
#define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); } #define pPrint(...) { taosPrintLog("APP ", DEBUG_INFO, 255, __VA_ARGS__); }
// clang-format on
#ifdef __cplusplus #ifdef __cplusplus
} }

42
include/util/tscalablebf.h Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_SCALABLEBF_H_
#define _TD_UTIL_SCALABLEBF_H_
#include "tbloomfilter.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct SScalableBf {
SArray *bfArray; // array of bloom filters
uint32_t growth;
uint64_t numBits;
} SScalableBf;
SScalableBf *tScalableBfInit(uint64_t expectedEntries, double errorRate);
int32_t tScalableBfPut(SScalableBf *pSBf, const void *keyBuf, uint32_t len);
int32_t tScalableBfNoContain(const SScalableBf *pSBf, const void *keyBuf,
uint32_t len);
void tScalableBfDestroy(SScalableBf *pSBf);
void tScalableBfDump(const SScalableBf *pSBf);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_SCALABLEBF_H_*/

4
include/util/ttimer.h
View File

@ -41,10 +41,6 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *han
void taosTmrCleanUp(void *handle); void taosTmrCleanUp(void *handle);
int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms);
void taosUninitTimer();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

13
source/client/CMakeLists.txt
View File

@ -1,5 +1,9 @@
aux_source_directory(src CLIENT_SRC) aux_source_directory(src CLIENT_SRC)
add_library(taos SHARED ${CLIENT_SRC}) if(TD_WINDOWS)
add_library(taos SHARED ${CLIENT_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/taos.rc.in)
else()
add_library(taos SHARED ${CLIENT_SRC})
endif ()
target_include_directories( target_include_directories(
taos taos
PUBLIC "${TD_SOURCE_DIR}/include/client" PUBLIC "${TD_SOURCE_DIR}/include/client"
@ -10,6 +14,13 @@ target_link_libraries(
INTERFACE api INTERFACE api
PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom PRIVATE os util common transport nodes parser command planner catalog scheduler function qcom
) )
if(TD_WINDOWS)
set_target_properties(taos
PROPERTIES
LINK_FLAGS
/DEF:${CMAKE_CURRENT_SOURCE_DIR}/src/taos.def
)
endif ()
set_target_properties( set_target_properties(
taos taos

5
source/client/inc/clientInt.h
View File

@ -188,6 +188,7 @@ typedef struct SRequestSendRecvBody {
typedef struct { typedef struct {
int8_t resType; int8_t resType;
int32_t code;
char topic[TSDB_TOPIC_FNAME_LEN]; char topic[TSDB_TOPIC_FNAME_LEN];
int32_t vgId; int32_t vgId;
SSchemaWrapper schema; SSchemaWrapper schema;
@ -254,8 +255,6 @@ extern int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t
int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code); int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code);
SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj);
int taos_init();
void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo); void* createTscObj(const char* user, const char* auth, const char* db, SAppInstInfo* pAppInfo);
void destroyTscObj(void* pObj); void destroyTscObj(void* pObj);
STscObj* acquireTscObj(int64_t rid); STscObj* acquireTscObj(int64_t rid);
@ -310,6 +309,8 @@ int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* v
void hbMgrInitMqHbRspHandle(); void hbMgrInitMqHbRspHandle();
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery);
int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList);
int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList);
#ifdef __cplusplus #ifdef __cplusplus
} }

35
source/client/inc/clientStmt.h
View File

@ -34,8 +34,7 @@ typedef enum {
STMT_PREPARE, STMT_PREPARE,
STMT_SETTBNAME, STMT_SETTBNAME,
STMT_SETTAGS, STMT_SETTAGS,
STMT_FETCH_TAG_FIELDS, STMT_FETCH_FIELDS,
STMT_FETCH_COL_FIELDS,
STMT_BIND, STMT_BIND,
STMT_BIND_COL, STMT_BIND_COL,
STMT_ADD_BATCH, STMT_ADD_BATCH,
@ -47,6 +46,12 @@ typedef struct SStmtTableCache {
void* boundTags; void* boundTags;
} SStmtTableCache; } SStmtTableCache;
typedef struct SQueryFields {
TAOS_FIELD* fields;
TAOS_FIELD* userFields;
uint32_t numOfCols;
} SQueryFields;
typedef struct SStmtBindInfo { typedef struct SStmtBindInfo {
bool needParse; bool needParse;
uint64_t tbUid; uint64_t tbUid;
@ -67,14 +72,17 @@ typedef struct SStmtExecInfo {
} SStmtExecInfo; } SStmtExecInfo;
typedef struct SStmtSQLInfo { typedef struct SStmtSQLInfo {
STMT_TYPE type; STMT_TYPE type;
STMT_STATUS status; STMT_STATUS status;
bool autoCreate; bool autoCreate;
uint64_t runTimes; uint64_t runTimes;
SHashObj* pTableCache; //SHash<SStmtTableCache> SHashObj* pTableCache; //SHash<SStmtTableCache>
SQuery* pQuery; SQuery* pQuery;
char* sqlStr; char* sqlStr;
int32_t sqlLen; int32_t sqlLen;
SArray* nodeList;
SQueryPlan* pQueryPlan;
SQueryFields fields;
} SStmtSQLInfo; } SStmtSQLInfo;
typedef struct STscStmt { typedef struct STscStmt {
@ -87,6 +95,8 @@ typedef struct STscStmt {
SStmtBindInfo bInfo; SStmtBindInfo bInfo;
} STscStmt; } STscStmt;
#define STMT_STATUS_NE(S) (pStmt->sql.status != STMT_##S)
#define STMT_STATUS_EQ(S) (pStmt->sql.status == STMT_##S)
#define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define STMT_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define STMT_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)
@ -97,14 +107,15 @@ int stmtClose(TAOS_STMT *stmt);
int stmtExec(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt);
const char *stmtErrstr(TAOS_STMT *stmt); const char *stmtErrstr(TAOS_STMT *stmt);
int stmtAffectedRows(TAOS_STMT *stmt); int stmtAffectedRows(TAOS_STMT *stmt);
int stmtAffectedRowsOnce(TAOS_STMT *stmt);
int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length); int stmtPrepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
int stmtSetTbName(TAOS_STMT *stmt, const char *tbName); int stmtSetTbName(TAOS_STMT *stmt, const char *tbName);
int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags); int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags);
int stmtIsInsert(TAOS_STMT *stmt, int *insert); int stmtIsInsert(TAOS_STMT *stmt, int *insert);
int stmtGetParamNum(TAOS_STMT *stmt, int *nums); int stmtGetParamNum(TAOS_STMT *stmt, int *nums);
int stmtAddBatch(TAOS_STMT *stmt); int stmtAddBatch(TAOS_STMT *stmt);
TAOS_RES *stmtUseResult(TAOS_STMT *stmt); TAOS_RES *stmtUseResult(TAOS_STMT *stmt);
int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx); int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx);
#ifdef __cplusplus #ifdef __cplusplus

33
source/client/src/clientImpl.c
View File

@ -103,6 +103,7 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
if (port) { if (port) {
epSet.epSet.eps[0].port = port; epSet.epSet.eps[0].port = port;
epSet.epSet.eps[1].port = port;
} }
char* key = getClusterKey(user, secretEncrypt, ip, port); char* key = getClusterKey(user, secretEncrypt, ip, port);
@ -187,8 +188,8 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC
setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision); setResPrecision(&pRequest->body.resInfo, (*pQuery)->precision);
} }
TSWAP(pRequest->dbList, (*pQuery)->pDbList, SArray*); TSWAP(pRequest->dbList, (*pQuery)->pDbList);
TSWAP(pRequest->tableList, (*pQuery)->pTableList, SArray*); TSWAP(pRequest->tableList, (*pQuery)->pTableList);
} }
return code; return code;
@ -233,7 +234,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
.showRewrite = pQuery->showRewrite, .showRewrite = pQuery->showRewrite,
.pTransporter = pRequest->pTscObj->pAppInfo->pTransporter, .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
.pMsg = pRequest->msgBuf, .pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.placeholderNum = pQuery->placeholderNum};
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog); int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = qCreateQueryPlan(&cxt, pPlan, pNodeList); code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
@ -245,7 +247,12 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
ASSERT(pSchema != NULL && numOfCols > 0); ASSERT(pSchema != NULL && numOfCols > 0);
pResInfo->numOfCols = numOfCols; pResInfo->numOfCols = numOfCols;
// TODO handle memory leak if (pResInfo->fields != NULL) {
taosMemoryFree(pResInfo->fields);
}
if (pResInfo->userFields != NULL) {
taosMemoryFree(pResInfo->userFields);
}
pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
@ -305,6 +312,11 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
return pRequest->code; return pRequest->code;
} }
int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) {
*pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
return getPlan(pRequest, pQuery, &pRequest->body.pDag, *pNodeList);
}
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
switch (pQuery->execMode) { switch (pQuery->execMode) {
@ -347,8 +359,15 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
SQuery* pQuery = NULL; SQuery* pQuery = NULL;
int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest);
if (TSDB_CODE_SUCCESS == code) { if (code != TSDB_CODE_SUCCESS) {
code = parseSql(pRequest, false, &pQuery, NULL); terrno = code;
return NULL;
}
code = parseSql(pRequest, false, &pQuery, NULL);
if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code;
return pRequest;
} }
return launchQueryImpl(pRequest, pQuery, code, false); return launchQueryImpl(pRequest, pQuery, code, false);
@ -399,7 +418,7 @@ SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen) {
while (retryNum++ < REQUEST_MAX_TRY_TIMES) { while (retryNum++ < REQUEST_MAX_TRY_TIMES) {
pRequest = launchQuery(pTscObj, sql, sqlLen); pRequest = launchQuery(pTscObj, sql, sqlLen);
if (TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) { if (pRequest == NULL || TSDB_CODE_SUCCESS == pRequest->code || !NEED_CLIENT_HANDLE_ERROR(pRequest->code)) {
break; break;
} }

63
source/client/src/clientMain.c
View File

@ -110,17 +110,24 @@ int taos_errno(TAOS_RES *tres) {
return terrno; return terrno;
} }
if (TD_RES_TMQ(tres)) {
return 0;
}
return ((SRequestObj *)tres)->code; return ((SRequestObj *)tres)->code;
} }
const char *taos_errstr(TAOS_RES *res) { const char *taos_errstr(TAOS_RES *res) {
SRequestObj *pRequest = (SRequestObj *)res; if (res == NULL) {
if (pRequest == NULL) {
return (const char *)tstrerror(terrno); return (const char *)tstrerror(terrno);
} }
if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { if (TD_RES_TMQ(res)) {
return "success";
}
SRequestObj *pRequest = (SRequestObj *)res;
if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
return pRequest->msgBuf; return pRequest->msgBuf;
} else { } else {
return (const char *)tstrerror(pRequest->code); return (const char *)tstrerror(pRequest->code);
@ -131,7 +138,7 @@ void taos_free_result(TAOS_RES *res) {
if (NULL == res) { if (NULL == res) {
return; return;
} }
if (TD_RES_QUERY(res)) { if (TD_RES_QUERY(res)) {
SRequestObj *pRequest = (SRequestObj *)res; SRequestObj *pRequest = (SRequestObj *)res;
destroyRequest(pRequest); destroyRequest(pRequest);
@ -603,7 +610,7 @@ int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
return stmtPrepare(stmt, sql, length); return stmtPrepare(stmt, sql, length);
} }
int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND_v2 *tags) { int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags) {
if (stmt == NULL || name == NULL) { if (stmt == NULL || name == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
@ -632,11 +639,9 @@ int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name) {
return stmtSetTbName(stmt, name); return stmtSetTbName(stmt, name);
} }
int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name) { return taos_stmt_set_tbname(stmt, name); }
return taos_stmt_set_tbname(stmt, name);
}
int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
if (stmt == NULL || bind == NULL) { if (stmt == NULL || bind == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
@ -648,11 +653,11 @@ int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
return stmtBindBatch(stmt, bind, -1); return stmtBindBatch(stmt, bind, -1);
} }
int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) { int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
if (stmt == NULL || bind == NULL) { if (stmt == NULL || bind == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
@ -665,10 +670,18 @@ int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind) {
return terrno; return terrno;
} }
int32_t insert = 0;
stmtIsInsert(stmt, &insert);
if (0 == insert && bind->num > 1) {
tscError("only one row data allowed for query");
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return stmtBindBatch(stmt, bind, -1); return stmtBindBatch(stmt, bind, -1);
} }
int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int colIdx) { int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx) {
if (stmt == NULL || bind == NULL) { if (stmt == NULL || bind == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
@ -680,7 +693,15 @@ int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int c
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
return terrno; return terrno;
} }
int32_t insert = 0;
stmtIsInsert(stmt, &insert);
if (0 == insert && bind->num > 1) {
tscError("only one row data allowed for query");
terrno = TSDB_CODE_INVALID_PARA;
return terrno;
}
return stmtBindBatch(stmt, bind, colIdx); return stmtBindBatch(stmt, bind, colIdx);
} }
@ -734,9 +755,7 @@ TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt) {
return stmtUseResult(stmt); return stmtUseResult(stmt);
} }
char *taos_stmt_errstr(TAOS_STMT *stmt) { char *taos_stmt_errstr(TAOS_STMT *stmt) { return (char *)stmtErrstr(stmt); }
return (char *)stmtErrstr(stmt);
}
int taos_stmt_affected_rows(TAOS_STMT *stmt) { int taos_stmt_affected_rows(TAOS_STMT *stmt) {
if (stmt == NULL) { if (stmt == NULL) {
@ -748,6 +767,16 @@ int taos_stmt_affected_rows(TAOS_STMT *stmt) {
return stmtAffectedRows(stmt); return stmtAffectedRows(stmt);
} }
int taos_stmt_affected_rows_once(TAOS_STMT *stmt) {
if (stmt == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_INVALID_PARA;
return 0;
}
return stmtAffectedRowsOnce(stmt);
}
int taos_stmt_close(TAOS_STMT *stmt) { int taos_stmt_close(TAOS_STMT *stmt) {
if (stmt == NULL) { if (stmt == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);

211
source/client/src/clientStmt.c
View File

@ -5,14 +5,52 @@
#include "tdef.h" #include "tdef.h"
int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) {
int32_t code = 0;
switch (newStatus) { switch (newStatus) {
case STMT_PREPARE:
break;
case STMT_SETTBNAME: case STMT_SETTBNAME:
if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND) || STMT_STATUS_EQ(BIND_COL)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
break;
case STMT_SETTAGS:
if (STMT_STATUS_NE(SETTBNAME)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
break;
case STMT_FETCH_FIELDS:
if (STMT_STATUS_EQ(INIT)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
break;
case STMT_BIND:
if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND_COL)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
break;
case STMT_BIND_COL:
if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
break;
case STMT_ADD_BATCH:
if (STMT_STATUS_NE(BIND) && STMT_STATUS_NE(BIND_COL) && STMT_STATUS_NE(FETCH_FIELDS)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
break;
case STMT_EXECUTE:
if (STMT_STATUS_NE(ADD_BATCH) && STMT_STATUS_NE(FETCH_FIELDS)) {
code = TSDB_CODE_TSC_STMT_API_ERROR;
}
break; break;
default: default:
code = TSDB_CODE_TSC_APP_ERROR;
break; break;
} }
//STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); STMT_ERR_RET(code);
pStmt->sql.status = newStatus; pStmt->sql.status = newStatus;
@ -35,6 +73,22 @@ int32_t stmtGetTbName(TAOS_STMT *stmt, char **tbName) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t stmtBackupQueryFields(STscStmt* pStmt) {
SQueryFields *pFields = &pStmt->sql.fields;
int32_t size = pFields->numOfCols * sizeof(TAOS_FIELD);
pFields->numOfCols = pStmt->exec.pRequest->body.resInfo.numOfCols;
pFields->fields = taosMemoryMalloc(size);
pFields->userFields = taosMemoryMalloc(size);
if (NULL == pFields->fields || NULL == pFields->userFields) {
STMT_ERR_RET(TSDB_CODE_TSC_OUT_OF_MEMORY);
}
memcpy(pFields->fields, pStmt->exec.pRequest->body.resInfo.fields, size);
memcpy(pFields->userFields, pStmt->exec.pRequest->body.resInfo.userFields, size);
return TSDB_CODE_SUCCESS;
}
int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) { int32_t stmtSetBindInfo(TAOS_STMT* stmt, STableMeta* pTableMeta, void* tags) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
@ -69,15 +123,10 @@ int32_t stmtCacheBlock(STscStmt *pStmt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
uint64_t uid; uint64_t uid = pStmt->bInfo.tbUid;
if (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) { uint64_t tuid = (TSDB_CHILD_TABLE == pStmt->bInfo.tbType) ? pStmt->bInfo.tbSuid : uid;
uid = pStmt->bInfo.tbSuid;
} else {
ASSERT(TSDB_NORMAL_TABLE == pStmt->bInfo.tbType);
uid = pStmt->bInfo.tbUid;
}
if (taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid))) { if (taosHashGet(pStmt->sql.pTableCache, &tuid, sizeof(tuid))) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -91,7 +140,7 @@ int32_t stmtCacheBlock(STscStmt *pStmt) {
.boundTags = pStmt->bInfo.boundTags, .boundTags = pStmt->bInfo.boundTags,
}; };
if (taosHashPut(pStmt->sql.pTableCache, &uid, sizeof(uid), &cache, sizeof(cache))) { if (taosHashPut(pStmt->sql.pTableCache, &tuid, sizeof(tuid), &cache, sizeof(cache))) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -149,9 +198,11 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) { int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool freeRequest) {
taos_free_result(pStmt->exec.pRequest); if (STMT_TYPE_QUERY != pStmt->sql.type || freeRequest) {
pStmt->exec.pRequest = NULL; taos_free_result(pStmt->exec.pRequest);
pStmt->exec.pRequest = NULL;
}
void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL); void *pIter = taosHashIterate(pStmt->exec.pBlockHash, NULL);
while (pIter) { while (pIter) {
@ -186,7 +237,9 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable) {
int32_t stmtCleanSQLInfo(STscStmt* pStmt) { int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
taosMemoryFree(pStmt->sql.sqlStr); taosMemoryFree(pStmt->sql.sqlStr);
qDestroyQuery(pStmt->sql.pQuery); qDestroyQuery(pStmt->sql.pQuery);
qDestroyQueryPlan(pStmt->sql.pQueryPlan);
taosArrayDestroy(pStmt->sql.nodeList);
void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL); void *pIter = taosHashIterate(pStmt->sql.pTableCache, NULL);
while (pIter) { while (pIter) {
SStmtTableCache* pCache = (SStmtTableCache*)pIter; SStmtTableCache* pCache = (SStmtTableCache*)pIter;
@ -201,7 +254,7 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) {
memset(&pStmt->sql, 0, sizeof(pStmt->sql)); memset(&pStmt->sql, 0, sizeof(pStmt->sql));
STMT_ERR_RET(stmtCleanExecInfo(pStmt, false)); STMT_ERR_RET(stmtCleanExecInfo(pStmt, false, true));
STMT_ERR_RET(stmtCleanBindInfo(pStmt)); STMT_ERR_RET(stmtCleanBindInfo(pStmt));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -221,37 +274,42 @@ int32_t stmtGetFromCache(STscStmt* pStmt) {
STableMeta *pTableMeta = NULL; STableMeta *pTableMeta = NULL;
SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); SEpSet ep = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp);
STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta)); STMT_ERR_RET(catalogGetTableMeta(pStmt->pCatalog, pStmt->taos->pAppInfo->pTransporter, &ep, &pStmt->bInfo.sname, &pTableMeta));
uint64_t uid = pTableMeta->uid;
if (pTableMeta->uid == pStmt->bInfo.tbUid) { uint64_t suid = pTableMeta->suid;
int8_t tableType = pTableMeta->tableType;
taosMemoryFree(pTableMeta);
if (uid == pStmt->bInfo.tbUid) {
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (taosHashGet(pStmt->exec.pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid))) { if (taosHashGet(pStmt->exec.pBlockHash, &uid, sizeof(uid))) {
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
if (NULL == pCache) { if (NULL == pCache) {
tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", pTableMeta->uid); tscError("table uid %" PRIx64 "found in exec blockHash, but not in sql blockHash", uid);
STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR); STMT_ERR_RET(TSDB_CODE_TSC_APP_ERROR);
} }
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
pStmt->bInfo.tbUid = pTableMeta->uid; pStmt->bInfo.tbUid = uid;
pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbSuid = suid;
pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.tbType = tableType;
pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.boundTags = pCache->boundTags;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pTableMeta->uid, sizeof(pTableMeta->uid)); SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &uid, sizeof(uid));
if (pCache) { if (pCache) {
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
pStmt->bInfo.tbUid = pTableMeta->uid; pStmt->bInfo.tbUid = uid;
pStmt->bInfo.tbSuid = pTableMeta->suid; pStmt->bInfo.tbSuid = suid;
pStmt->bInfo.tbType = pTableMeta->tableType; pStmt->bInfo.tbType = tableType;
pStmt->bInfo.boundTags = pCache->boundTags; pStmt->bInfo.boundTags = pCache->boundTags;
STableDataBlocks* pNewBlock = NULL; STableDataBlocks* pNewBlock = NULL;
@ -333,6 +391,13 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME)); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTBNAME));
int32_t insert = 0;
stmtIsInsert(stmt, &insert);
if (0 == insert) {
tscError("set tb name not available for none insert statement");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
}
if (NULL == pStmt->exec.pRequest) { if (NULL == pStmt->exec.pRequest) {
STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
} }
@ -349,7 +414,7 @@ int stmtSetTbName(TAOS_STMT *stmt, const char *tbName) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) { int stmtSetTbTags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS)); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_SETTAGS));
@ -370,15 +435,7 @@ int stmtSetTbTags(TAOS_STMT *stmt, TAOS_BIND_v2 *tags) {
} }
int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { int32_t stmtFetchTagFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) {
STscStmt* pStmt = (STscStmt*)stmt;
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_TAG_FIELDS));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
}
if (STMT_TYPE_QUERY == pStmt->sql.type) { if (STMT_TYPE_QUERY == pStmt->sql.type) {
tscError("invalid operation to get query tag fileds"); tscError("invalid operation to get query tag fileds");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@ -395,15 +452,7 @@ int32_t stmtFetchTagFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fields) { int32_t stmtFetchColFields(STscStmt* pStmt, int32_t *fieldNum, TAOS_FIELD** fields) {
STscStmt* pStmt = (STscStmt*)stmt;
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_COL_FIELDS));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
}
if (STMT_TYPE_QUERY == pStmt->sql.type) { if (STMT_TYPE_QUERY == pStmt->sql.type) {
tscError("invalid operation to get query column fileds"); tscError("invalid operation to get query column fileds");
STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR);
@ -420,7 +469,7 @@ int32_t stmtFetchColFields(TAOS_STMT *stmt, int32_t *fieldNum, TAOS_FIELD** fiel
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) { int stmtBindBatch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int32_t colIdx) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND));
@ -429,6 +478,11 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
pStmt->bInfo.needParse = false; pStmt->bInfo.needParse = false;
} }
if (pStmt->exec.pRequest && STMT_TYPE_QUERY == pStmt->sql.type && pStmt->sql.runTimes) {
taos_free_result(pStmt->exec.pRequest);
pStmt->exec.pRequest = NULL;
}
if (NULL == pStmt->exec.pRequest) { if (NULL == pStmt->exec.pRequest) {
STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest)); STMT_ERR_RET(buildRequest(pStmt->taos, pStmt->sql.sqlStr, pStmt->sql.sqlLen, &pStmt->exec.pRequest));
} }
@ -437,6 +491,17 @@ int stmtBindBatch(TAOS_STMT *stmt, TAOS_BIND_v2 *bind, int32_t colIdx) {
STMT_ERR_RET(stmtParseSql(pStmt)); STMT_ERR_RET(stmtParseSql(pStmt));
} }
if (STMT_TYPE_QUERY == pStmt->sql.type) {
if (NULL == pStmt->sql.pQueryPlan) {
STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
pStmt->exec.pRequest->body.pDag = NULL;
STMT_ERR_RET(stmtBackupQueryFields(pStmt));
}
STMT_RET(qStmtBindParam(pStmt->sql.pQueryPlan, bind, colIdx, pStmt->exec.pRequest->requestId));
}
STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid)); STableDataBlocks **pDataBlock = (STableDataBlocks**)taosHashGet(pStmt->exec.pBlockHash, (const char*)&pStmt->bInfo.tbUid, sizeof(pStmt->bInfo.tbUid));
if (NULL == pDataBlock) { if (NULL == pDataBlock) {
tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid); tscError("table uid %" PRIx64 "not found in exec blockHash", pStmt->bInfo.tbUid);
@ -480,10 +545,13 @@ int stmtExec(TAOS_STMT *stmt) {
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE));
STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash)); if (STMT_TYPE_QUERY == pStmt->sql.type) {
scheduleQuery(pStmt->exec.pRequest, pStmt->sql.pQueryPlan, pStmt->sql.nodeList);
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true); } else {
STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->exec.pVgHash, pStmt->exec.pBlockHash));
launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, TSDB_CODE_SUCCESS, true);
}
STMT_ERR_JRET(pStmt->exec.pRequest->code); STMT_ERR_JRET(pStmt->exec.pRequest->code);
pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest); pStmt->exec.affectedRows = taos_affected_rows(pStmt->exec.pRequest);
@ -491,7 +559,7 @@ int stmtExec(TAOS_STMT *stmt) {
_return: _return:
stmtCleanExecInfo(pStmt, (code ? false : true)); stmtCleanExecInfo(pStmt, (code ? false : true), false);
++pStmt->sql.runTimes; ++pStmt->sql.runTimes;
@ -503,6 +571,8 @@ int stmtClose(TAOS_STMT *stmt) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
STMT_RET(stmtCleanSQLInfo(pStmt)); STMT_RET(stmtCleanSQLInfo(pStmt));
taosMemoryFree(stmt);
} }
const char *stmtErrstr(TAOS_STMT *stmt) { const char *stmtErrstr(TAOS_STMT *stmt) {
@ -523,6 +593,10 @@ int stmtAffectedRows(TAOS_STMT *stmt) {
return ((STscStmt*)stmt)->affectedRows; return ((STscStmt*)stmt)->affectedRows;
} }
int stmtAffectedRowsOnce(TAOS_STMT *stmt) {
return ((STscStmt*)stmt)->exec.affectedRows;
}
int stmtIsInsert(TAOS_STMT *stmt, int *insert) { int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
STscStmt* pStmt = (STscStmt*)stmt; STscStmt* pStmt = (STscStmt*)stmt;
@ -536,13 +610,38 @@ int stmtIsInsert(TAOS_STMT *stmt, int *insert) {
} }
int stmtGetParamNum(TAOS_STMT *stmt, int *nums) { int stmtGetParamNum(TAOS_STMT *stmt, int *nums) {
STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL)); STscStmt* pStmt = (STscStmt*)stmt;
STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS));
if (pStmt->bInfo.needParse) {
STMT_ERR_RET(stmtParseSql(pStmt));
}
if (STMT_TYPE_QUERY == pStmt->sql.type) {
if (NULL == pStmt->sql.pQueryPlan) {
STMT_ERR_RET(getQueryPlan(pStmt->exec.pRequest, pStmt->sql.pQuery, &pStmt->sql.nodeList));
pStmt->sql.pQueryPlan = pStmt->exec.pRequest->body.pDag;
pStmt->exec.pRequest->body.pDag = NULL;
}
*nums = taosArrayGetSize(pStmt->sql.pQueryPlan->pPlaceholderValues);
} else {
STMT_ERR_RET(stmtFetchColFields(stmt, nums, NULL));
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
TAOS_RES *stmtUseResult(TAOS_STMT *stmt) { TAOS_RES *stmtUseResult(TAOS_STMT *stmt) {
return NULL; STscStmt* pStmt = (STscStmt*)stmt;
if (STMT_TYPE_QUERY != pStmt->sql.type) {
tscError("useResult only for query statement");
return NULL;
}
return pStmt->exec.pRequest;
} }

81
source/client/src/taos.def Normal file
View File

@ -0,0 +1,81 @@
taos_cleanup
taos_options
taos_set_config
taos_init
taos_connect
taos_connect_l
taos_connect_auth
taos_close
taos_data_type
taos_stmt_init
taos_stmt_prepare
taos_stmt_set_tbname_tags
taos_stmt_set_tbname
taos_stmt_set_sub_tbname
taos_stmt_is_insert
taos_stmt_num_params
taos_stmt_get_param
taos_stmt_bind_param
taos_stmt_bind_param_batch
taos_stmt_bind_single_param_batch
taos_stmt_add_batch
taos_stmt_execute
taos_stmt_use_result
taos_stmt_close
taos_stmt_errstr
taos_stmt_affected_rows
taos_stmt_affected_rows_once
taos_query
taos_query_l
taos_fetch_row
taos_result_precision
taos_free_result
taos_field_count
taos_num_fields
taos_affected_rows
taos_fetch_fields
taos_select_db
taos_print_row
taos_stop_query
taos_is_null
taos_is_update_query
taos_fetch_block
taos_fetch_block_s
taos_fetch_raw_block
taos_get_column_data_offset
taos_validate_sql
taos_reset_current_db
taos_fetch_lengths
taos_result_block
taos_get_server_info
taos_get_client_info
taos_errstr
taos_errno
taos_query_a
taos_fetch_rows_a
taos_subscribe
taos_consume
taos_unsubscribe
taos_load_table_info
taos_schemaless_insert
tmq_list_new
tmq_list_append
tmq_list_destroy
tmq_list_get_size
tmq_list_to_c_array
tmq_consumer_new
tmq_err2str
tmq_subscribe
tmq_unsubscribe
tmq_subscription
tmq_consumer_poll
tmq_consumer_close
tmq_commit
tmq_conf_new
tmq_conf_set
tmq_conf_destroy
tmq_conf_set_offset_commit_cb
tmq_get_topic_name
tmq_get_vgroup_id
tmq_create_stream
taos_check_server_status

31
source/client/src/taos.rc.in Normal file
View File

@ -0,0 +1,31 @@
1 VERSIONINFO
FILEVERSION ${TD_VER_NUMBER}
PRODUCTVERSION ${TD_VER_NUMBER}
FILEFLAGSMASK 0x17L
#ifdef _DEBUG
FILEFLAGS 0x1L
#else
FILEFLAGS 0x0L
#endif
FILEOS 0x4L
FILETYPE 0x0L
FILESUBTYPE 0x0L
BEGIN
BLOCK "StringFileInfo"
BEGIN
BLOCK "040904b0"
BEGIN
VALUE "FileDescription", "Native C Driver for TDengine"
VALUE "FileVersion", "${TD_VER_NUMBER}"
VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})"
VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data"
VALUE "OriginalFilename", ""
VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})"
VALUE "ProductVersion", "${TD_VER_NUMBER}"
END
END
BLOCK "VarFileInfo"
BEGIN
VALUE "Translation", 0x409, 1200
END
END

563
source/client/src/tmq.c
View File

@ -16,13 +16,22 @@
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "parser.h" #include "parser.h"
#include "planner.h"
#include "tdatablock.h" #include "tdatablock.h"
#include "tdef.h" #include "tdef.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsgtype.h" #include "tmsgtype.h"
#include "tqueue.h" #include "tqueue.h"
#include "tref.h" #include "tref.h"
#include "ttimer.h"
int32_t tmqAskEp(tmq_t* tmq, bool async);
typedef struct {
int8_t inited;
tmr_h timer;
} SMqMgmt;
static SMqMgmt tmqMgmt = {0};
typedef struct { typedef struct {
int8_t tmqRspType; int8_t tmqRspType;
@ -30,9 +39,9 @@ typedef struct {
} SMqRspWrapper; } SMqRspWrapper;
typedef struct { typedef struct {
int8_t tmqRspType; int8_t tmqRspType;
int32_t epoch; int32_t epoch;
SMqCMGetSubEpRsp msg; SMqAskEpRsp msg;
} SMqAskEpRspWrapper; } SMqAskEpRspWrapper;
struct tmq_list_t { struct tmq_list_t {
@ -53,37 +62,49 @@ struct tmq_conf_t {
int8_t autoCommit; int8_t autoCommit;
int8_t resetOffset; int8_t resetOffset;
uint16_t port; uint16_t port;
uint16_t autoCommitInterval; int32_t autoCommitInterval;
char* ip; char* ip;
char* user; char* user;
char* pass; char* pass;
char* db; char* db;
tmq_commit_cb* commit_cb; tmq_commit_cb* commitCb;
}; };
struct tmq_t { struct tmq_t {
// conf // conf
char groupId[TSDB_CGROUP_LEN]; char groupId[TSDB_CGROUP_LEN];
char clientId[256]; char clientId[256];
int8_t autoCommit; int8_t autoCommit;
/*int8_t inWaiting;*/ int32_t autoCommitInterval;
int64_t consumerId;
int32_t epoch;
int32_t resetOffsetCfg; int32_t resetOffsetCfg;
int64_t status; int64_t consumerId;
STscObj* pTscObj;
tmq_commit_cb* commit_cb; tmq_commit_cb* commit_cb;
/*int32_t nextTopicIdx;*/
// status
int8_t status;
int32_t epoch;
#if 0
int8_t epStatus; int8_t epStatus;
int32_t epSkipCnt; int32_t epSkipCnt;
/*int32_t waitingRequest;*/ #endif
/*int32_t readyRequest;*/
SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of tmq_message_t
STaosQall* qall;
tsem_t rspSem;
// stat
int64_t pollCnt; int64_t pollCnt;
// timer
tmr_h hbTimer;
tmr_h reportTimer;
tmr_h commitTimer;
// connection
STscObj* pTscObj;
// container
SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of rsp
STaosQall* qall;
STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
// ctl
tsem_t rspSem;
}; };
enum { enum {
@ -94,6 +115,13 @@ enum {
enum { enum {
TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__INIT = 0,
TMQ_CONSUMER_STATUS__READY, TMQ_CONSUMER_STATUS__READY,
TMQ_CONSUMER_STATUS__NO_TOPIC,
};
enum {
TMQ_DELAYED_TASK__HB = 1,
TMQ_DELAYED_TASK__REPORT,
TMQ_DELAYED_TASK__COMMIT,
}; };
typedef struct { typedef struct {
@ -110,13 +138,11 @@ typedef struct {
typedef struct { typedef struct {
// subscribe info // subscribe info
int32_t sqlLen; char* topicName;
char* sql;
char* topicName; SArray* vgs; // SArray<SMqClientVg>
int64_t topicId;
SArray* vgs; // SArray<SMqClientVg>
int8_t isSchemaAdaptive; int8_t isSchemaAdaptive;
int32_t numOfFields;
SSchemaWrapper schema; SSchemaWrapper schema;
} SMqClientTopic; } SMqClientTopic;
@ -137,7 +163,7 @@ typedef struct {
typedef struct { typedef struct {
tmq_t* tmq; tmq_t* tmq;
int32_t code; int32_t code;
int32_t sync; int32_t async;
tsem_t rspSem; tsem_t rspSem;
} SMqAskEpCbParam; } SMqAskEpCbParam;
@ -148,7 +174,6 @@ typedef struct {
int32_t epoch; int32_t epoch;
int32_t vgId; int32_t vgId;
tsem_t rspSem; tsem_t rspSem;
int32_t sync;
} SMqPollCbParam; } SMqPollCbParam;
typedef struct { typedef struct {
@ -156,12 +181,12 @@ typedef struct {
int32_t async; int32_t async;
tsem_t rspSem; tsem_t rspSem;
tmq_resp_err_t rspErr; tmq_resp_err_t rspErr;
/*SMqClientVg* pVg;*/
} SMqCommitCbParam; } SMqCommitCbParam;
tmq_conf_t* tmq_conf_new() { tmq_conf_t* tmq_conf_new() {
tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t)); tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
conf->autoCommit = false; conf->autoCommit = false;
conf->autoCommitInterval = 5000;
conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST; conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
return conf; return conf;
} }
@ -251,13 +276,7 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
void tmq_list_destroy(tmq_list_t* list) { void tmq_list_destroy(tmq_list_t* list) {
SArray* container = &list->container; SArray* container = &list->container;
/*taosArrayDestroy(container);*/ taosArrayDestroyP(container, taosMemoryFree);
int32_t sz = taosArrayGetSize(container);
for (int32_t i = 0; i < sz; i++) {
char* str = taosArrayGetP(container, i);
taosMemoryFree(str);
}
taosArrayDestroy(container);
} }
int32_t tmq_list_get_size(const tmq_list_t* list) { int32_t tmq_list_get_size(const tmq_list_t* list) {
@ -274,6 +293,53 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
return sprintf(dst, "%s:%d", topicName, vg); return sprintf(dst, "%s:%d", topicName, vg);
} }
void tmqAssignDelayedHbTask(void* param, void* tmrId) {
tmq_t* tmq = (tmq_t*)param;
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
*pTaskType = TMQ_DELAYED_TASK__HB;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
void tmqAssignDelayedCommitTask(void* param, void* tmrId) {
tmq_t* tmq = (tmq_t*)param;
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
*pTaskType = TMQ_DELAYED_TASK__COMMIT;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
void tmqAssignDelayedReportTask(void* param, void* tmrId) {
tmq_t* tmq = (tmq_t*)param;
int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t));
*pTaskType = TMQ_DELAYED_TASK__REPORT;
taosWriteQitem(tmq->delayedTask, pTaskType);
tsem_post(&tmq->rspSem);
}
int32_t tmqHandleAllDelayedTask(tmq_t* tmq) {
STaosQall* qall = taosAllocateQall();
taosReadAllQitems(tmq->delayedTask, qall);
while (1) {
int8_t* pTaskType = NULL;
taosGetQitem(qall, (void**)&pTaskType);
if (pTaskType == NULL) break;
if (*pTaskType == TMQ_DELAYED_TASK__HB) {
tmqAskEp(tmq, true);
taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) {
tmq_commit(tmq, NULL, true);
taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer);
} else if (*pTaskType == TMQ_DELAYED_TASK__REPORT) {
} else {
ASSERT(0);
}
}
taosFreeQall(qall);
return 0;
}
void tmqClearUnhandleMsg(tmq_t* tmq) { void tmqClearUnhandleMsg(tmq_t* tmq) {
SMqRspWrapper* msg = NULL; SMqRspWrapper* msg = NULL;
while (1) { while (1) {
@ -298,6 +364,7 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code; pParam->rspErr = code;
tmq_t* tmq = pParam->tmq;
tsem_post(&pParam->rspSem); tsem_post(&pParam->rspSem);
return 0; return 0;
} }
@ -318,14 +385,16 @@ tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) {
} }
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i); SMqClientTopic* topic = taosArrayGetP(tmq->clientTopics, i);
tmq_list_append(*topics, strdup(topic->topicName)); tmq_list_append(*topics, topic->topicName);
} }
return TMQ_RESP_ERR__SUCCESS; return TMQ_RESP_ERR__SUCCESS;
} }
tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) { tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq) {
tmq_list_t* lst = tmq_list_new(); tmq_list_t* lst = tmq_list_new();
return tmq_subscribe(tmq, lst); tmq_resp_err_t rsp = tmq_subscribe(tmq, lst);
tmq_list_destroy(lst);
return rsp;
} }
#if 0 #if 0
@ -335,12 +404,9 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
return NULL; return NULL;
} }
pTmq->pTscObj = (STscObj*)conn; pTmq->pTscObj = (STscObj*)conn;
/*pTmq->inWaiting = 0;*/
pTmq->status = 0; pTmq->status = 0;
pTmq->pollCnt = 0; pTmq->pollCnt = 0;
pTmq->epoch = 0; pTmq->epoch = 0;
/*pTmq->waitingRequest = 0;*/
/*pTmq->readyRequest = 0;*/
pTmq->epStatus = 0; pTmq->epStatus = 0;
pTmq->epSkipCnt = 0; pTmq->epSkipCnt = 0;
// set conf // set conf
@ -367,46 +433,77 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
#endif #endif
tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
// init timer
int8_t inited = atomic_val_compare_exchange_8(&tmqMgmt.inited, 0, 1);
if (inited == 0) {
tmqMgmt.timer = taosTmrInit(1000, 100, 360000, "TMQ");
if (tmqMgmt.timer == NULL) {
atomic_store_8(&tmqMgmt.inited, 0);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
}
tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t));
if (pTmq == NULL) { if (pTmq == NULL) {
return NULL; return NULL;
} }
const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user; const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user;
const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass;
ASSERT(user); ASSERT(user);
ASSERT(pass); ASSERT(pass);
/*ASSERT(conf->db);*/
ASSERT(conf->groupId[0]); ASSERT(conf->groupId[0]);
pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
if (pTmq->pTscObj == NULL) return NULL; pTmq->mqueue = taosOpenQueue();
pTmq->qall = taosAllocateQall();
pTmq->delayedTask = taosOpenQueue();
pTmq->status = 0; if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL) {
goto FAIL;
}
// init status
pTmq->status = TMQ_CONSUMER_STATUS__INIT;
pTmq->pollCnt = 0; pTmq->pollCnt = 0;
pTmq->epoch = 0; pTmq->epoch = 0;
pTmq->epStatus = 0; /*pTmq->epStatus = 0;*/
pTmq->epSkipCnt = 0; /*pTmq->epSkipCnt = 0;*/
// set conf // set conf
strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId); strcpy(pTmq->groupId, conf->groupId);
pTmq->autoCommit = conf->autoCommit; pTmq->autoCommit = conf->autoCommit;
pTmq->commit_cb = conf->commit_cb; pTmq->autoCommitInterval = conf->autoCommitInterval;
pTmq->commit_cb = conf->commitCb;
pTmq->resetOffsetCfg = conf->resetOffset; pTmq->resetOffsetCfg = conf->resetOffset;
// assign consumerId
pTmq->consumerId = tGenIdPI64(); pTmq->consumerId = tGenIdPI64();
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
if (pTmq->clientTopics == NULL) { // init semaphore
taosMemoryFree(pTmq); if (tsem_init(&pTmq->rspSem, 0, 0) != 0) {
return NULL; goto FAIL;
} }
pTmq->mqueue = taosOpenQueue(); // init connection
pTmq->qall = taosAllocateQall(); pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ);
if (pTmq->pTscObj == NULL) {
tsem_init(&pTmq->rspSem, 0, 0); tsem_destroy(&pTmq->rspSem);
goto FAIL;
}
return pTmq; return pTmq;
FAIL:
if (pTmq->clientTopics) taosArrayDestroy(pTmq->clientTopics);
if (pTmq->mqueue) taosCloseQueue(pTmq->mqueue);
if (pTmq->delayedTask) taosCloseQueue(pTmq->delayedTask);
if (pTmq->qall) taosFreeQall(pTmq->qall);
taosMemoryFree(pTmq);
return NULL;
} }
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
@ -497,103 +594,104 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
return resp; return resp;
} }
tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
SRequestObj* pRequest = NULL; const SArray* container = &topic_list->container;
SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container);
int32_t sz = taosArrayGetSize(container); void* buf = NULL;
// destroy ex SCMSubscribeReq req = {0};
taosArrayDestroy(tmq->clientTopics); int32_t code = -1;
tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic));
SCMSubscribeReq req;
req.consumerId = tmq->consumerId; req.consumerId = tmq->consumerId;
strcpy(req.cgroup, tmq->groupId); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN);
req.topicNames = taosArrayInit(sz, sizeof(void*)); req.topicNames = taosArrayInit(sz, sizeof(void*));
if (req.topicNames == NULL) goto FAIL;
for (int i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
/*char* topicName = topic_list->elems[i];*/ char* topic = taosArrayGetP(container, i);
char* topicName = taosArrayGetP(container, i);
SName name = {0}; SName name = {0};
#if 0 tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic));
char* dbName = getDbOfConnection(tmq->pTscObj);
if (dbName == NULL) {
return TMQ_RESP_ERR__FAIL;
}
#endif
tNameSetDbName(&name, tmq->pTscObj->acctId, topicName, strlen(topicName));
#if 0
tNameFromString(&name, topicName, T_NAME_TABLE);
#endif
char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
if (topicFname == NULL) { if (topicFName == NULL) {
goto _return; goto FAIL;
} }
tNameExtractFullName(&name, topicFname); tNameExtractFullName(&name, topicFName);
tscDebug("subscribe topic: %s", topicFname);
SMqClientTopic topic = { tscDebug("subscribe topic: %s", topicFName);
.sql = NULL,
.sqlLen = 0, taosArrayPush(req.topicNames, &topicFName);
.topicId = 0,
.topicName = topicFname,
.vgs = NULL,
};
topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
taosArrayPush(tmq->clientTopics, &topic);
taosArrayPush(req.topicNames, &topicFname);
#if 0
taosMemoryFree(dbName);
#endif
} }
int tlen = tSerializeSCMSubscribeReq(NULL, &req); int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req);
void* buf = taosMemoryMalloc(tlen); buf = taosMemoryMalloc(tlen);
if (buf == NULL) { if (buf == NULL) goto FAIL;
goto _return;
}
void* abuf = buf; void* abuf = buf;
tSerializeSCMSubscribeReq(&abuf, &req); tSerializeSCMSubscribeReq(&abuf, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
if (pRequest == NULL) { if (sendInfo == NULL) goto FAIL;
tscError("failed to malloc request");
}
SMqSubscribeCbParam param = { SMqSubscribeCbParam param = {
.rspErr = TMQ_RESP_ERR__SUCCESS, .rspErr = TMQ_RESP_ERR__SUCCESS,
.tmq = tmq, .tmq = tmq,
}; };
tsem_init(&param.rspSem, 0, 0);
pRequest->body.requestMsg = (SDataBuf){ if (tsem_init(&param.rspSem, 0, 0) != 0) goto FAIL;
sendInfo->msgInfo = (SDataBuf){
.pData = buf, .pData = buf,
.len = tlen, .len = tlen,
.handle = NULL, .handle = NULL,
}; };
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->param = &param; sendInfo->param = &param;
sendInfo->fp = tmqSubscribeCb; sendInfo->fp = tmqSubscribeCb;
sendInfo->msgType = TDMT_MND_SUBSCRIBE;
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0; int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
// avoid double free if msg is sent
buf = NULL;
tsem_wait(&param.rspSem); tsem_wait(&param.rspSem);
tsem_destroy(&param.rspSem); tsem_destroy(&param.rspSem);
_return: code = param.rspErr;
/*if (sendInfo != NULL) {*/ if (code != 0) goto FAIL;
/*destroySendMsgInfo(sendInfo);*/
/*}*/
return param.rspErr; while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, false)) {
tscDebug("not ready, retry");
taosMsleep(500);
}
// init hb timer
tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer);
// init auto commit timer
if (tmq->autoCommit) {
tmq->commitTimer = taosTmrStart(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer);
}
code = 0;
FAIL:
if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree);
if (code != 0 && buf) {
taosMemoryFree(buf);
}
return code;
} }
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
//
conf->commitCb = cb;
}
TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) { TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
STscObj* pTscObj = (STscObj*)taos; STscObj* pTscObj = (STscObj*)taos;
@ -627,9 +725,6 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
int32_t code = 0; int32_t code = 0;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return); CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return);
// todo check for invalid sql statement and return with error code
CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
/*printf("%s\n", pStr);*/ /*printf("%s\n", pStr);*/
@ -653,7 +748,6 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
} }
tSerializeSCMCreateStreamReq(buf, tlen, &req); tSerializeSCMCreateStreamReq(buf, tlen, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest->body.requestMsg = (SDataBuf){ pRequest->body.requestMsg = (SDataBuf){
.pData = buf, .pData = buf,
@ -684,94 +778,6 @@ _return:
return pRequest; return pRequest;
} }
#if 0
TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos;
SRequestObj* pRequest = NULL;
SQuery* pQueryNode = NULL;
char* astStr = NULL;
terrno = TSDB_CODE_SUCCESS;
if (taos == NULL || topicName == NULL || sql == NULL) {
tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql);
terrno = TSDB_CODE_TSC_INVALID_INPUT;
goto _return;
}
if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) {
tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1);
terrno = TSDB_CODE_TSC_INVALID_INPUT;
goto _return;
}
if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) {
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
goto _return;
}
tscDebug("start to create topic: %s", topicName);
int32_t code = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return);
// todo check for invalid sql statement and return with error code
CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
/*printf("%s\n", pStr);*/
SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T};
strcpy(name.dbname, pRequest->pDb);
strcpy(name.tname, topicName);
SCMCreateTopicReq req = {
.igExists = 1,
.ast = astStr,
.sql = (char*)sql,
};
tNameExtractFullName(&name, req.name);
int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req);
void* buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
goto _return;
}
tSerializeSCMCreateTopicReq(buf, tlen, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
.len = tlen,
.handle = NULL,
};
pRequest->type = TDMT_MND_CREATE_TOPIC;
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
_return:
taosMemoryFreeClear(astStr);
qDestroyQuery(pQueryNode);
/*if (sendInfo != NULL) {*/
/*destroySendMsgInfo(sendInfo);*/
/*}*/
if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
pRequest->code = terrno;
}
return pRequest;
}
#endif
#if 0 #if 0
int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return 0; if (tmq_message == NULL) return 0;
@ -796,7 +802,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
// do not write into queue since updating epoch reset // do not write into queue since updating epoch reset
tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch, tscWarn("msg discard from vg %d since from earlier epoch, rsp epoch %d, current epoch %d", pParam->vgId, msgEpoch,
tmqEpoch); tmqEpoch);
/*tsem_post(&tmq->rspSem);*/ tsem_post(&tmq->rspSem);
return 0; return 0;
} }
@ -804,25 +810,6 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch); tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", pParam->vgId, msgEpoch, tmqEpoch);
} }
#if 0
if (pParam->sync == 1) {
/**pParam->msg = taosMemoryMalloc(sizeof(tmq_message_t));*/
*pParam->msg = taosAllocateQitem(sizeof(tmq_message_t));
if (*pParam->msg) {
memcpy(*pParam->msg, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &((*pParam->msg)->consumeRsp));
if ((*pParam->msg)->consumeRsp.numOfTopics != 0) {
pVg->currentOffset = (*pParam->msg)->consumeRsp.rspOffset;
}
taosWriteQitem(tmq->mqueue, *pParam->msg);
tsem_post(&pParam->rspSem);
return 0;
}
tsem_post(&pParam->rspSem);
return -1;
}
#endif
SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper)); SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper));
if (pRspWrapper == NULL) { if (pRspWrapper == NULL) {
tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch); tscWarn("msg discard from vg %d, epoch %d since out of memory", pParam->vgId, pParam->epoch);
@ -841,18 +828,18 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset);
taosWriteQitem(tmq->mqueue, pRspWrapper); taosWriteQitem(tmq->mqueue, pRspWrapper);
/*tsem_post(&tmq->rspSem);*/ tsem_post(&tmq->rspSem);
return 0; return 0;
CREATE_MSG_FAIL: CREATE_MSG_FAIL:
if (pParam->epoch == tmq->epoch) { if (pParam->epoch == tmq->epoch) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
} }
/*tsem_post(&tmq->rspSem);*/ tsem_post(&tmq->rspSem);
return -1; return -1;
} }
bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) { bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) {
/*printf("call update ep %d\n", epoch);*/ /*printf("call update ep %d\n", epoch);*/
bool set = false; bool set = false;
int32_t topicNumGet = taosArrayGetSize(pRsp->topics); int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
@ -925,6 +912,12 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqCMGetSubEpRsp* pRsp) {
if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics); if (tmq->clientTopics) taosArrayDestroy(tmq->clientTopics);
taosHashCleanup(pHash); taosHashCleanup(pHash);
tmq->clientTopics = newTopics; tmq->clientTopics = newTopics;
if (taosArrayGetSize(tmq->clientTopics) == 0)
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__NO_TOPIC);
else
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
atomic_store_32(&tmq->epoch, epoch); atomic_store_32(&tmq->epoch, epoch);
return set; return set;
} }
@ -934,7 +927,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
tmq_t* tmq = pParam->tmq; tmq_t* tmq = pParam->tmq;
pParam->code = code; pParam->code = code;
if (code != 0) { if (code != 0) {
tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->sync); tscError("consumer %ld get topic endpoint error, not ready, wait:%d", tmq->consumerId, pParam->async);
goto END; goto END;
} }
@ -948,15 +941,13 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
goto END; goto END;
} }
if (pParam->sync) { if (!pParam->async) {
SMqCMGetSubEpRsp rsp; SMqAskEpRsp rsp;
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp);
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
/*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
if (tmqUpdateEp(tmq, head->epoch, &rsp)) { tmqUpdateEp(tmq, head->epoch, &rsp);
atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY); tDeleteSMqAskEpRsp(&rsp);
}
tDeleteSMqCMGetSubEpRsp(&rsp);
} else { } else {
SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper)); SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper));
if (pWrapper == NULL) { if (pWrapper == NULL) {
@ -967,23 +958,24 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP; pWrapper->tmqRspType = TMQ_MSG_TYPE__EP_RSP;
pWrapper->epoch = head->epoch; pWrapper->epoch = head->epoch;
memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); memcpy(&pWrapper->msg, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg); tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pWrapper->msg);
taosWriteQitem(tmq->mqueue, pWrapper); taosWriteQitem(tmq->mqueue, pWrapper);
/*tsem_post(&tmq->rspSem);*/ tsem_post(&tmq->rspSem);
taosMemoryFree(pParam); taosMemoryFree(pParam);
} }
END: END:
atomic_store_8(&tmq->epStatus, 0); /*atomic_store_8(&tmq->epStatus, 0);*/
if (pParam->sync) { if (!pParam->async) {
tsem_post(&pParam->rspSem); tsem_post(&pParam->rspSem);
} }
return code; return code;
} }
int32_t tmqAskEp(tmq_t* tmq, bool sync) { int32_t tmqAskEp(tmq_t* tmq, bool async) {
int32_t code = 0; int32_t code = 0;
#if 0
int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1); int8_t epStatus = atomic_val_compare_exchange_8(&tmq->epStatus, 0, 1);
if (epStatus == 1) { if (epStatus == 1) {
int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1); int32_t epSkipCnt = atomic_add_fetch_32(&tmq->epSkipCnt, 1);
@ -991,11 +983,12 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
if (epSkipCnt < 5000) return 0; if (epSkipCnt < 5000) return 0;
} }
atomic_store_32(&tmq->epSkipCnt, 0); atomic_store_32(&tmq->epSkipCnt, 0);
int32_t tlen = sizeof(SMqCMGetSubEpReq); #endif
SMqCMGetSubEpReq* req = taosMemoryMalloc(tlen); int32_t tlen = sizeof(SMqAskEpReq);
SMqAskEpReq* req = taosMemoryMalloc(tlen);
if (req == NULL) { if (req == NULL) {
tscError("failed to malloc get subscribe ep buf"); tscError("failed to malloc get subscribe ep buf");
atomic_store_8(&tmq->epStatus, 0); /*atomic_store_8(&tmq->epStatus, 0);*/
return -1; return -1;
} }
req->consumerId = htobe64(tmq->consumerId); req->consumerId = htobe64(tmq->consumerId);
@ -1006,11 +999,11 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
if (pParam == NULL) { if (pParam == NULL) {
tscError("failed to malloc subscribe param"); tscError("failed to malloc subscribe param");
taosMemoryFree(req); taosMemoryFree(req);
atomic_store_8(&tmq->epStatus, 0); /*atomic_store_8(&tmq->epStatus, 0);*/
return -1; return -1;
} }
pParam->tmq = tmq; pParam->tmq = tmq;
pParam->sync = sync; pParam->async = async;
tsem_init(&pParam->rspSem, 0, 0); tsem_init(&pParam->rspSem, 0, 0);
SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
@ -1018,7 +1011,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
tsem_destroy(&pParam->rspSem); tsem_destroy(&pParam->rspSem);
taosMemoryFree(pParam); taosMemoryFree(pParam);
taosMemoryFree(req); taosMemoryFree(req);
atomic_store_8(&tmq->epStatus, 0); /*atomic_store_8(&tmq->epStatus, 0);*/
return -1; return -1;
} }
@ -1032,7 +1025,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
sendInfo->requestObjRefId = 0; sendInfo->requestObjRefId = 0;
sendInfo->param = pParam; sendInfo->param = pParam;
sendInfo->fp = tmqAskEpCb; sendInfo->fp = tmqAskEpCb;
sendInfo->msgType = TDMT_MND_GET_SUB_EP; sendInfo->msgType = TDMT_MND_MQ_ASK_EP;
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
@ -1041,7 +1034,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool sync) {
int64_t transporterId = 0; int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
if (sync) { if (!async) {
tsem_wait(&pParam->rspSem); tsem_wait(&pParam->rspSem);
code = pParam->code; code = pParam->code;
taosMemoryFree(pParam); taosMemoryFree(pParam);
@ -1072,7 +1065,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
return TMQ_RESP_ERR__FAIL; return TMQ_RESP_ERR__FAIL;
} }
SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) { SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int64_t reqOffset; int64_t reqOffset;
if (pVg->currentOffset >= 0) { if (pVg->currentOffset >= 0) {
reqOffset = pVg->currentOffset; reqOffset = pVg->currentOffset;
@ -1084,7 +1077,7 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient
reqOffset = tmq->resetOffsetCfg; reqOffset = tmq->resetOffsetCfg;
} }
SMqPollReqV2* pReq = taosMemoryMalloc(sizeof(SMqPollReqV2)); SMqPollReq* pReq = taosMemoryMalloc(sizeof(SMqPollReq));
if (pReq == NULL) { if (pReq == NULL) {
return NULL; return NULL;
} }
@ -1097,14 +1090,14 @@ SMqPollReqV2* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClient
pReq->subKey[tlen] = TMQ_SEPARATOR; pReq->subKey[tlen] = TMQ_SEPARATOR;
strcpy(pReq->subKey + tlen + 1, pTopic->topicName); strcpy(pReq->subKey + tlen + 1, pTopic->topicName);
pReq->blockingTime = blockingTime; pReq->waitTime = waitTime;
pReq->consumerId = tmq->consumerId; pReq->consumerId = tmq->consumerId;
pReq->epoch = tmq->epoch; pReq->epoch = tmq->epoch;
pReq->currentOffset = reqOffset; pReq->currentOffset = reqOffset;
pReq->reqId = generateRequestId(); pReq->reqId = generateRequestId();
pReq->head.vgId = htonl(pVg->vgId); pReq->head.vgId = htonl(pVg->vgId);
pReq->head.contLen = htonl(sizeof(SMqPollReqV2)); pReq->head.contLen = htonl(sizeof(SMqPollReq));
return pReq; return pReq;
} }
@ -1126,7 +1119,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
return pRspObj; return pRspObj;
} }
int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { int32_t tmqPollImpl(tmq_t* tmq, int64_t waitTime) {
/*printf("call poll\n");*/ /*printf("call poll\n");*/
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
@ -1147,17 +1140,17 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
#endif #endif
} }
atomic_store_32(&pVg->vgSkipCnt, 0); atomic_store_32(&pVg->vgSkipCnt, 0);
SMqPollReqV2* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg); SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, waitTime, pTopic, pVg);
if (pReq == NULL) { if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
/*tsem_post(&tmq->rspSem);*/ tsem_post(&tmq->rspSem);
return -1; return -1;
} }
SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam));
if (pParam == NULL) { if (pParam == NULL) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
/*tsem_post(&tmq->rspSem);*/ tsem_post(&tmq->rspSem);
return -1; return -1;
} }
pParam->tmq = tmq; pParam->tmq = tmq;
@ -1165,20 +1158,19 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
pParam->pTopic = pTopic; pParam->pTopic = pTopic;
pParam->vgId = pVg->vgId; pParam->vgId = pVg->vgId;
pParam->epoch = tmq->epoch; pParam->epoch = tmq->epoch;
pParam->sync = 0;
SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo)); SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
if (sendInfo == NULL) { if (sendInfo == NULL) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
taosMemoryFree(pParam); taosMemoryFree(pParam);
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
/*tsem_post(&tmq->rspSem);*/ tsem_post(&tmq->rspSem);
return -1; return -1;
} }
sendInfo->msgInfo = (SDataBuf){ sendInfo->msgInfo = (SDataBuf){
.pData = pReq, .pData = pReq,
.len = sizeof(SMqPollReqV2), .len = sizeof(SMqPollReq),
.handle = NULL, .handle = NULL,
}; };
sendInfo->requestId = pReq->reqId; sendInfo->requestId = pReq->reqId;
@ -1189,7 +1181,6 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
int64_t transporterId = 0; int64_t transporterId = 0;
/*printf("send poll\n");*/ /*printf("send poll\n");*/
/*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/
tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
/*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
@ -1206,7 +1197,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
/*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/
if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) {
SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper;
SMqCMGetSubEpRsp* rspMsg = &pEpRspWrapper->msg; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg;
tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg);
/*tmqClearUnhandleMsg(tmq);*/ /*tmqClearUnhandleMsg(tmq);*/
*pReset = true; *pReset = true;
@ -1219,7 +1210,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset)
return 0; return 0;
} }
SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t waitTime, bool pollIfReset) {
while (1) { while (1) {
SMqRspWrapper* rspWrapper = NULL; SMqRspWrapper* rspWrapper = NULL;
taosGetQitem(tmq->qall, (void**)&rspWrapper); taosGetQitem(tmq->qall, (void**)&rspWrapper);
@ -1258,45 +1249,45 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
taosFreeQitem(rspWrapper); taosFreeQitem(rspWrapper);
if (pollIfReset && reset) { if (pollIfReset && reset) {
tscDebug("consumer %ld reset and repoll", tmq->consumerId); tscDebug("consumer %ld reset and repoll", tmq->consumerId);
tmqPollImpl(tmq, blockingTime); tmqPollImpl(tmq, waitTime);
} }
} }
} }
} }
TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t wait_time) {
SMqRspObj* rspObj; SMqRspObj* rspObj;
int64_t startTime = taosGetTimestampMs(); int64_t startTime = taosGetTimestampMs();
// TODO: put into another thread or delayed queue rspObj = tmqHandleAllRsp(tmq, wait_time, false);
int64_t status = atomic_load_64(&tmq->status);
while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) {
tscDebug("not ready, retry\n");
taosSsleep(1);
}
rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspObj) { if (rspObj) {
return (TAOS_RES*)rspObj; return (TAOS_RES*)rspObj;
} }
// in no topic status also need process delayed task
if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) {
return NULL;
}
while (1) { while (1) {
/*printf("cycle\n");*/ tmqHandleAllDelayedTask(tmq);
tmqAskEp(tmq, false); tmqPollImpl(tmq, wait_time);
tmqPollImpl(tmq, blocking_time);
/*tsem_wait(&tmq->rspSem);*/ rspObj = tmqHandleAllRsp(tmq, wait_time, false);
rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspObj) { if (rspObj) {
return (TAOS_RES*)rspObj; return (TAOS_RES*)rspObj;
} }
if (blocking_time != 0) { if (wait_time != 0) {
int64_t endTime = taosGetTimestampMs(); int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) { int64_t leftTime = endTime - startTime;
if (leftTime > wait_time) {
tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch); tscDebug("consumer %ld (epoch %d) timeout, no rsp", tmq->consumerId, tmq->epoch);
return NULL; return NULL;
} }
tsem_timewait(&tmq->rspSem, leftTime * 1000);
} else {
// use tsem_timewait instead of tsem_wait to avoid unexpected stuck
tsem_timewait(&tmq->rspSem, 500 * 1000);
} }
} }
} }

33
source/common/src/tdatablock.c
View File

@ -79,7 +79,11 @@ int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRo
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
return pColumnInfoData->varmeta.length; return pColumnInfoData->varmeta.length;
} else { } else {
return pColumnInfoData->info.bytes * numOfRows; if (pColumnInfoData->info.type == TSDB_DATA_TYPE_NULL) {
return 0;
} else {
return pColumnInfoData->info.bytes * numOfRows;
}
} }
} }
@ -113,22 +117,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con
int32_t type = pColumnInfoData->info.type; int32_t type = pColumnInfoData->info.type;
if (IS_VAR_DATA_TYPE(type)) { if (IS_VAR_DATA_TYPE(type)) {
int32_t dataLen = varDataTLen(pData); int32_t dataLen = varDataTLen(pData);
if(type == TSDB_DATA_TYPE_JSON) { if (type == TSDB_DATA_TYPE_JSON) {
if(*pData == TSDB_DATA_TYPE_NULL) { if (*pData == TSDB_DATA_TYPE_NULL) {
dataLen = 0; dataLen = 0;
}else if(*pData == TSDB_DATA_TYPE_NCHAR) { } else if (*pData == TSDB_DATA_TYPE_NCHAR) {
dataLen = varDataTLen(pData+CHAR_BYTES); dataLen = varDataTLen(pData + CHAR_BYTES);
}else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) {
dataLen = LONG_BYTES; dataLen = LONG_BYTES;
}else if(*pData == TSDB_DATA_TYPE_BOOL) { } else if (*pData == TSDB_DATA_TYPE_BOOL) {
dataLen = CHAR_BYTES; dataLen = CHAR_BYTES;
} }
dataLen += CHAR_BYTES; dataLen += CHAR_BYTES;
} }
SVarColAttr* pAttr = &pColumnInfoData->varmeta; SVarColAttr* pAttr = &pColumnInfoData->varmeta;
if (pAttr->allocLen < pAttr->length + dataLen) { if (pAttr->allocLen < pAttr->length + dataLen) {
uint32_t newSize = pAttr->allocLen; uint32_t newSize = pAttr->allocLen;
if (newSize == 0) { if (newSize <= 1) {
newSize = 8; newSize = 8;
} }
@ -220,12 +225,16 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
// Handle the bitmap // Handle the bitmap
char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2));
if (p == NULL) { if (p == NULL) {
// TODO return TSDB_CODE_OUT_OF_MEMORY;
} }
pColumnInfoData->varmeta.offset = (int32_t*)p; pColumnInfoData->varmeta.offset = (int32_t*)p;
for (int32_t i = 0; i < numOfRow2; ++i) { for (int32_t i = 0; i < numOfRow2; ++i) {
pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length; if (pSource->varmeta.offset[i] == -1) {
pColumnInfoData->varmeta.offset[i + numOfRow1] = -1;
} else {
pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length;
}
} }
// copy data // copy data
@ -234,7 +243,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co
if (pColumnInfoData->varmeta.allocLen < len + oldLen) { if (pColumnInfoData->varmeta.allocLen < len + oldLen) {
char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, len + oldLen);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_VND_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pColumnInfoData->pData = tmp; pColumnInfoData->pData = tmp;
@ -307,6 +316,8 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows);
} }
pColumnInfoData->hasNull = pSource->hasNull;
pColumnInfoData->info = pSource->info;
return 0; return 0;
} }

503
source/common/src/tmsg.c
View File

@ -93,7 +93,7 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
return row; return row;
} }
} }
#if 0
// TODO: KEEP one suite of iterator API finally. // TODO: KEEP one suite of iterator API finally.
// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts // 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts
// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later // 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later
@ -173,7 +173,7 @@ STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) {
return row; return row;
} }
} }
#endif
int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) { int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) {
if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1;
if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1;
@ -490,205 +490,6 @@ int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchR
return 0; return 0;
} }
int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedU32(buf, pReq->ttl);
tlen += taosEncodeFixedU32(buf, pReq->keep);
tlen += taosEncodeFixedU8(buf, pReq->info);
switch (pReq->type) {
case TD_SUPER_TABLE:
tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nCols);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->stbCfg.pSchema[i].name);
}
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.nTagCols);
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pTagSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->stbCfg.pTagSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->stbCfg.pTagSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->stbCfg.pTagSchema[i].name);
}
if (pReq->rollup && pReq->stbCfg.pRSmaParam) {
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
tlen += taosEncodeFixedI32(buf, param->delay);
tlen += taosEncodeFixedI8(buf, param->nFuncIds);
for (int8_t i = 0; i < param->nFuncIds; ++i) {
tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
}
tlen += taosEncodeFixedI32(buf, param->qmsg1Len);
if (param->qmsg1Len > 0) {
tlen += taosEncodeString(buf, param->qmsg1);
}
tlen += taosEncodeFixedI32(buf, param->qmsg2Len);
if (param->qmsg2Len > 0) {
tlen += taosEncodeString(buf, param->qmsg2);
}
}
break;
case TD_CHILD_TABLE:
tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid);
tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag);
break;
case TD_NORMAL_TABLE:
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nCols);
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.nBSmaCols);
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].type);
tlen += taosEncodeFixedI8(buf, pReq->ntbCfg.pSchema[i].flags);
tlen += taosEncodeFixedI16(buf, pReq->ntbCfg.pSchema[i].colId);
tlen += taosEncodeFixedI32(buf, pReq->ntbCfg.pSchema[i].bytes);
tlen += taosEncodeString(buf, pReq->ntbCfg.pSchema[i].name);
}
if (pReq->rollup && pReq->ntbCfg.pRSmaParam) {
SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
tlen += taosEncodeBinary(buf, (const void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
tlen += taosEncodeFixedI32(buf, param->delay);
tlen += taosEncodeFixedI8(buf, param->nFuncIds);
for (int8_t i = 0; i < param->nFuncIds; ++i) {
tlen += taosEncodeFixedI32(buf, param->pFuncIds[i]);
}
}
break;
default:
ASSERT(0);
}
return tlen;
}
void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) {
buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeFixedU32(buf, &(pReq->ttl));
buf = taosDecodeFixedU32(buf, &(pReq->keep));
buf = taosDecodeFixedU8(buf, &(pReq->info));
switch (pReq->type) {
case TD_SUPER_TABLE:
buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols));
pReq->stbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].flags));
buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId));
buf = taosDecodeFixedI32(buf, &(pReq->stbCfg.pSchema[i].bytes));
buf = taosDecodeStringTo(buf, pReq->stbCfg.pSchema[i].name);
}
buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols);
pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) {
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type));
buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].flags));
buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->stbCfg.pTagSchema[i].name);
}
if (pReq->rollup) {
pReq->stbCfg.pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam));
SRSmaParam *param = pReq->stbCfg.pRSmaParam;
buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
buf = taosDecodeFixedI32(buf, &param->delay);
buf = taosDecodeFixedI8(buf, &param->nFuncIds);
if (param->nFuncIds > 0) {
param->pFuncIds = (func_id_t *)taosMemoryCalloc(param->nFuncIds, sizeof(func_id_t));
for (int8_t i = 0; i < param->nFuncIds; ++i) {
buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
}
}
buf = taosDecodeFixedI32(buf, &param->qmsg1Len);
if (param->qmsg1Len > 0) {
buf = taosDecodeString(buf, &param->qmsg1);
}
buf = taosDecodeFixedI32(buf, &param->qmsg2Len);
if (param->qmsg2Len > 0) {
buf = taosDecodeString(buf, &param->qmsg2);
}
} else {
pReq->stbCfg.pRSmaParam = NULL;
}
break;
case TD_CHILD_TABLE:
buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid);
buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag);
break;
case TD_NORMAL_TABLE:
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols);
buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols));
pReq->ntbCfg.pSchema = (SSchema *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchema));
for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) {
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type);
buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].flags);
buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId);
buf = taosDecodeFixedI32(buf, &pReq->ntbCfg.pSchema[i].bytes);
buf = taosDecodeStringTo(buf, pReq->ntbCfg.pSchema[i].name);
}
if (pReq->rollup) {
pReq->ntbCfg.pRSmaParam = (SRSmaParam *)taosMemoryMalloc(sizeof(SRSmaParam));
SRSmaParam *param = pReq->ntbCfg.pRSmaParam;
buf = taosDecodeBinaryTo(buf, (void *)&param->xFilesFactor, sizeof(param->xFilesFactor));
buf = taosDecodeFixedI32(buf, &param->delay);
buf = taosDecodeFixedI8(buf, &param->nFuncIds);
if (param->nFuncIds > 0) {
param->pFuncIds = (func_id_t *)taosMemoryMalloc(param->nFuncIds * sizeof(func_id_t));
for (int8_t i = 0; i < param->nFuncIds; ++i) {
buf = taosDecodeFixedI32(buf, param->pFuncIds + i);
}
} else {
param->pFuncIds = NULL;
}
} else {
pReq->ntbCfg.pRSmaParam = NULL;
}
break;
default:
ASSERT(0);
}
return buf;
}
int32_t tSerializeSVCreateTbBatchReq(void **buf, SVCreateTbBatchReq *pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->ver);
tlen += taosEncodeFixedU32(buf, taosArrayGetSize(pReq->pArray));
for (size_t i = 0; i < taosArrayGetSize(pReq->pArray); i++) {
SVCreateTbReq *pCreateTbReq = taosArrayGet(pReq->pArray, i);
tlen += tSerializeSVCreateTbReq(buf, pCreateTbReq);
}
return tlen;
}
void *tDeserializeSVCreateTbBatchReq(void *buf, SVCreateTbBatchReq *pReq) {
uint32_t nsize = 0;
buf = taosDecodeFixedI64(buf, &pReq->ver);
buf = taosDecodeFixedU32(buf, &nsize);
pReq->pArray = taosArrayInit(nsize, sizeof(SVCreateTbReq));
for (size_t i = 0; i < nsize; i++) {
SVCreateTbReq req = {0};
buf = tDeserializeSVCreateTbReq(buf, &req);
taosArrayPush(pReq->pArray, &req);
}
return buf;
}
int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) { int32_t tSerializeSVDropTbReq(void **buf, SVDropTbReq *pReq) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->ver); tlen += taosEncodeFixedI64(buf, pReq->ver);
@ -712,12 +513,10 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1; if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1;
if (tEncodeI32(&encoder, pReq->aggregationMethod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->delay) < 0) return -1; if (tEncodeI32(&encoder, pReq->delay) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfSmas) < 0) return -1;
if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1; if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1; if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1; if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1;
@ -727,6 +526,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeI8(&encoder, pField->type) < 0) return -1; if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
} }
for (int32_t i = 0; i < pReq->numOfTags; ++i) { for (int32_t i = 0; i < pReq->numOfTags; ++i) {
@ -734,13 +534,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeI8(&encoder, pField->type) < 0) return -1; if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1; if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
} if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
SField *pField = taosArrayGet(pReq->pSmas, i);
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
} }
if (pReq->commentLen > 0) { if (pReq->commentLen > 0) {
@ -767,20 +561,17 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1; if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->aggregationMethod) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1; if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfSmas) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1; if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1;
pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
pReq->pSmas = taosArrayInit(pReq->numOfSmas, sizeof(SField)); if (pReq->pColumns == NULL || pReq->pTags == NULL) {
if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pSmas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -790,6 +581,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeI8(&decoder, &field.type) < 0) return -1; if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
if (taosArrayPush(pReq->pColumns, &field) == NULL) { if (taosArrayPush(pReq->pColumns, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
@ -801,23 +593,13 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeI8(&decoder, &field.type) < 0) return -1; if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
if (taosArrayPush(pReq->pTags, &field) == NULL) { if (taosArrayPush(pReq->pTags, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
} }
for (int32_t i = 0; i < pReq->numOfSmas; ++i) {
SField field = {0};
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
if (taosArrayPush(pReq->pSmas, &field) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
if (pReq->commentLen > 0) { if (pReq->commentLen > 0) {
pReq->comment = taosMemoryMalloc(pReq->commentLen); pReq->comment = taosMemoryMalloc(pReq->commentLen);
if (pReq->comment == NULL) return -1; if (pReq->comment == NULL) return -1;
@ -845,13 +627,11 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { void tFreeSMCreateStbReq(SMCreateStbReq *pReq) {
taosArrayDestroy(pReq->pColumns); taosArrayDestroy(pReq->pColumns);
taosArrayDestroy(pReq->pTags); taosArrayDestroy(pReq->pTags);
taosArrayDestroy(pReq->pSmas);
taosMemoryFreeClear(pReq->comment); taosMemoryFreeClear(pReq->comment);
taosMemoryFreeClear(pReq->pAst1); taosMemoryFreeClear(pReq->pAst1);
taosMemoryFreeClear(pReq->pAst2); taosMemoryFreeClear(pReq->pAst2);
pReq->pColumns = NULL; pReq->pColumns = NULL;
pReq->pTags = NULL; pReq->pTags = NULL;
pReq->pSmas = NULL;
} }
int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
@ -1522,6 +1302,11 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *
return 0; return 0;
} }
void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) {
taosHashCleanup(pRsp->readDbs);
taosHashCleanup(pRsp->writeDbs);
}
int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { int32_t tSerializeSCreateDropMQSBNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) {
SCoder encoder = {0}; SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
@ -3454,7 +3239,7 @@ int32_t tEncodeSMqCMCommitOffsetReq(SCoder *encoder, const SMqCMCommitOffsetReq
int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) { int32_t tDecodeSMqCMCommitOffsetReq(SCoder *decoder, SMqCMCommitOffsetReq *pReq) {
if (tStartDecode(decoder) < 0) return -1; if (tStartDecode(decoder) < 0) return -1;
if (tDecodeI32(decoder, &pReq->num) < 0) return -1; if (tDecodeI32(decoder, &pReq->num) < 0) return -1;
TCODER_MALLOC(pReq->offsets, SMqOffset *, pReq->num * sizeof(SMqOffset), decoder); pReq->offsets = (SMqOffset *)tCoderMalloc(decoder, sizeof(SMqOffset) * pReq->num);
if (pReq->offsets == NULL) return -1; if (pReq->offsets == NULL) return -1;
for (int32_t i = 0; i < pReq->num; i++) { for (int32_t i = 0; i < pReq->num; i++) {
tDecodeSMqOffset(decoder, &pReq->offsets[i]); tDecodeSMqOffset(decoder, &pReq->offsets[i]);
@ -3670,48 +3455,79 @@ int32_t tDeserializeSQueryTableRsp(void *buf, int32_t bufLen, SQueryTableRsp *pR
} }
int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { int32_t tSerializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
SCoder encoder = {0}; // SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); // tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1; // if (tStartEncode(&encoder) < 0) return -1;
if (pRsp->rspList) { // if (pRsp->rspList) {
int32_t num = taosArrayGetSize(pRsp->rspList); // int32_t num = taosArrayGetSize(pRsp->rspList);
if (tEncodeI32(&encoder, num) < 0) return -1; // if (tEncodeI32(&encoder, num) < 0) return -1;
for (int32_t i = 0; i < num; ++i) { // for (int32_t i = 0; i < num; ++i) {
SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i); // SVCreateTbRsp *rsp = taosArrayGet(pRsp->rspList, i);
if (tEncodeI32(&encoder, rsp->code) < 0) return -1; // if (tEncodeI32(&encoder, rsp->code) < 0) return -1;
} // }
} else { // } else {
if (tEncodeI32(&encoder, 0) < 0) return -1; // if (tEncodeI32(&encoder, 0) < 0) return -1;
} // }
tEndEncode(&encoder); // tEndEncode(&encoder);
int32_t tlen = encoder.pos; // int32_t tlen = encoder.pos;
tCoderClear(&encoder); // tCoderClear(&encoder);
return tlen; // reture tlen;
return 0;
} }
int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) { int32_t tDeserializeSVCreateTbBatchRsp(void *buf, int32_t bufLen, SVCreateTbBatchRsp *pRsp) {
SCoder decoder = {0}; // SCoder decoder = {0};
int32_t num = 0; // int32_t num = 0;
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); // tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1; // if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeI32(&decoder, &num) < 0) return -1; // if (tDecodeI32(&decoder, &num) < 0) return -1;
if (num > 0) { // if (num > 0) {
pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp)); // pRsp->rspList = taosArrayInit(num, sizeof(SVCreateTbRsp));
if (NULL == pRsp->rspList) return -1; // if (NULL == pRsp->rspList) return -1;
for (int32_t i = 0; i < num; ++i) { // for (int32_t i = 0; i < num; ++i) {
SVCreateTbRsp rsp = {0}; // SVCreateTbRsp rsp = {0};
if (tDecodeI32(&decoder, &rsp.code) < 0) return -1; // if (tDecodeI32(&decoder, &rsp.code) < 0) return -1;
if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1; // if (NULL == taosArrayPush(pRsp->rspList, &rsp)) return -1;
} // }
} else { // } else {
pRsp->rspList = NULL; // pRsp->rspList = NULL;
// }
// tEndDecode(&decoder);
// tCoderClear(&decoder);
return 0;
}
int tEncodeSVCreateTbBatchRsp(SCoder *pCoder, const SVCreateTbBatchRsp *pRsp) {
int32_t nRsps = taosArrayGetSize(pRsp->pArray);
SVCreateTbRsp *pCreateRsp;
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI32v(pCoder, nRsps) < 0) return -1;
for (int32_t i = 0; i < nRsps; i++) {
pCreateRsp = taosArrayGet(pRsp->pArray, i);
if (tEncodeSVCreateTbRsp(pCoder, pCreateRsp) < 0) return -1;
} }
tEndDecode(&decoder);
tCoderClear(&decoder); tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbBatchRsp(SCoder *pCoder, SVCreateTbBatchRsp *pRsp) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI32v(pCoder, &pRsp->nRsps) < 0) return -1;
pRsp->pRsps = (SVCreateTbRsp *)tCoderMalloc(pCoder, sizeof(*pRsp->pRsps) * pRsp->nRsps);
for (int32_t i = 0; i < pRsp->nRsps; i++) {
if (tDecodeSVCreateTbRsp(pCoder, pRsp->pRsps + i) < 0) return -1;
}
tEndDecode(pCoder);
return 0; return 0;
} }
@ -3813,6 +3629,75 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
taosMemoryFreeClear(pReq->ast); taosMemoryFreeClear(pReq->ast);
} }
int32_t tEncodeSRSmaParam(SCoder *pCoder, const SRSmaParam *pRSmaParam) {
if (tEncodeFloat(pCoder, pRSmaParam->xFilesFactor) < 0) return -1;
if (tEncodeI32v(pCoder, pRSmaParam->delay) < 0) return -1;
if (tEncodeI32v(pCoder, pRSmaParam->qmsg1Len) < 0) return -1;
if (tEncodeI32v(pCoder, pRSmaParam->qmsg2Len) < 0) return -1;
if (pRSmaParam->qmsg1Len > 0) {
if (tEncodeBinary(pCoder, pRSmaParam->qmsg1, (uint64_t)pRSmaParam->qmsg1Len) < 0) // qmsg1Len contains len of '\0'
return -1;
}
if (pRSmaParam->qmsg2Len > 0) {
if (tEncodeBinary(pCoder, pRSmaParam->qmsg2, (uint64_t)pRSmaParam->qmsg2Len) < 0) // qmsg2Len contains len of '\0'
return -1;
}
return 0;
}
int32_t tDecodeSRSmaParam(SCoder *pCoder, SRSmaParam *pRSmaParam) {
if (tDecodeFloat(pCoder, &pRSmaParam->xFilesFactor) < 0) return -1;
if (tDecodeI32v(pCoder, &pRSmaParam->delay) < 0) return -1;
if (tDecodeI32v(pCoder, &pRSmaParam->qmsg1Len) < 0) return -1;
if (tDecodeI32v(pCoder, &pRSmaParam->qmsg2Len) < 0) return -1;
if (pRSmaParam->qmsg1Len > 0) {
uint64_t len;
if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg1, &len) < 0) return -1; // qmsg1Len contains len of '\0'
} else {
pRSmaParam->qmsg1 = NULL;
}
if (pRSmaParam->qmsg2Len > 0) {
uint64_t len;
if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg2, &len) < 0) return -1; // qmsg2Len contains len of '\0'
} else {
pRSmaParam->qmsg2 = NULL;
}
return 0;
}
int tEncodeSVCreateStbReq(SCoder *pCoder, const SVCreateStbReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
if (tEncodeI64(pCoder, pReq->suid) < 0) return -1;
if (tEncodeI8(pCoder, pReq->rollup) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
if (tEncodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
if (pReq->rollup) {
if (tEncodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
}
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateStbReq(SCoder *pCoder, SVCreateStbReq *pReq) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1;
if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schema) < 0) return -1;
if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1;
if (pReq->rollup) {
if (tDecodeSRSmaParam(pCoder, &pReq->pRSmaParam) < 0) return -1;
}
tEndDecode(pCoder);
return 0;
}
STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
STSchemaBuilder schemaBuilder = {0}; STSchemaBuilder schemaBuilder = {0};
if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) { if (tdInitTSchemaBuilder(&schemaBuilder, 0) < 0) {
@ -3836,3 +3721,97 @@ STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) {
tdDestroyTSchemaBuilder(&schemaBuilder); tdDestroyTSchemaBuilder(&schemaBuilder);
return pNSchema; return pNSchema;
} }
int tEncodeSVCreateTbReq(SCoder *pCoder, const SVCreateTbReq *pReq) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI64(pCoder, pReq->uid) < 0) return -1;
if (tEncodeI64(pCoder, pReq->ctime) < 0) return -1;
if (tEncodeCStr(pCoder, pReq->name) < 0) return -1;
if (tEncodeI32(pCoder, pReq->ttl) < 0) return -1;
if (tEncodeI8(pCoder, pReq->type) < 0) return -1;
if (pReq->type == TSDB_CHILD_TABLE) {
if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1;
if (tEncodeBinary(pCoder, pReq->ctb.pTag, kvRowLen(pReq->ctb.pTag)) < 0) return -1;
} else if (pReq->type == TSDB_NORMAL_TABLE) {
if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1;
} else {
ASSERT(0);
}
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbReq(SCoder *pCoder, SVCreateTbReq *pReq) {
uint64_t len;
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->uid) < 0) return -1;
if (tDecodeI64(pCoder, &pReq->ctime) < 0) return -1;
if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1;
if (tDecodeI32(pCoder, &pReq->ttl) < 0) return -1;
if (tDecodeI8(pCoder, &pReq->type) < 0) return -1;
if (pReq->type == TSDB_CHILD_TABLE) {
if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1;
if (tDecodeBinary(pCoder, &pReq->ctb.pTag, &len) < 0) return -1;
} else if (pReq->type == TSDB_NORMAL_TABLE) {
if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schema) < 0) return -1;
} else {
ASSERT(0);
}
tEndDecode(pCoder);
return 0;
}
int tEncodeSVCreateTbBatchReq(SCoder *pCoder, const SVCreateTbBatchReq *pReq) {
int32_t nReq = taosArrayGetSize(pReq->pArray);
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI32v(pCoder, nReq) < 0) return -1;
for (int iReq = 0; iReq < nReq; iReq++) {
if (tEncodeSVCreateTbReq(pCoder, (SVCreateTbReq *)taosArrayGet(pReq->pArray, iReq)) < 0) return -1;
}
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbBatchReq(SCoder *pCoder, SVCreateTbBatchReq *pReq) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI32v(pCoder, &pReq->nReqs) < 0) return -1;
pReq->pReqs = (SVCreateTbReq *)tCoderMalloc(pCoder, sizeof(SVCreateTbReq) * pReq->nReqs);
if (pReq->pReqs == NULL) return -1;
for (int iReq = 0; iReq < pReq->nReqs; iReq++) {
if (tDecodeSVCreateTbReq(pCoder, pReq->pReqs + iReq) < 0) return -1;
}
tEndDecode(pCoder);
return 0;
}
int tEncodeSVCreateTbRsp(SCoder *pCoder, const SVCreateTbRsp *pRsp) {
if (tStartEncode(pCoder) < 0) return -1;
if (tEncodeI32(pCoder, pRsp->code) < 0) return -1;
tEndEncode(pCoder);
return 0;
}
int tDecodeSVCreateTbRsp(SCoder *pCoder, SVCreateTbRsp *pRsp) {
if (tStartDecode(pCoder) < 0) return -1;
if (tDecodeI32(pCoder, &pRsp->code) < 0) return -1;
tEndDecode(pCoder);
return 0;
}

2
source/common/src/trow.c
View File

@ -220,7 +220,7 @@ static uint8_t tdGetMergedBitmapByte(uint8_t byte) {
} }
/** /**
* @brief Merge bitmap from 2 bits to 1 bits, and the memory buffer should be guaranteed by the invoker. * @brief Merge bitmap from 2 bits to 1 bit, and the memory buffer should be guaranteed by the invoker.
* *
* @param srcBitmap * @param srcBitmap
* @param nBits * @param nBits

12
source/common/src/ttypes.c
View File

@ -651,35 +651,35 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void *buf
switch (type) { switch (type) {
case TSDB_DATA_TYPE_INT: case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT: { case TSDB_DATA_TYPE_UINT: {
TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); TSWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight));
break; break;
} }
case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_UBIGINT: case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_TIMESTAMP: { case TSDB_DATA_TYPE_TIMESTAMP: {
TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); TSWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight));
break; break;
} }
case TSDB_DATA_TYPE_DOUBLE: { case TSDB_DATA_TYPE_DOUBLE: {
TSWAP(*(double *)(pLeft), *(double *)(pRight), double); TSWAP(*(double *)(pLeft), *(double *)(pRight));
break; break;
} }
case TSDB_DATA_TYPE_SMALLINT: case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_USMALLINT: { case TSDB_DATA_TYPE_USMALLINT: {
TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); TSWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight));
break; break;
} }
case TSDB_DATA_TYPE_FLOAT: { case TSDB_DATA_TYPE_FLOAT: {
TSWAP(*(float *)(pLeft), *(float *)(pRight), float); TSWAP(*(float *)(pLeft), *(float *)(pRight));
break; break;
} }
case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_UTINYINT: { case TSDB_DATA_TYPE_UTINYINT: {
TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); TSWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight));
break; break;
} }

6
source/common/test/tschemaTest.cpp
View File

@ -1,6 +0,0 @@
#include <gtest/gtest.h>
#include "tschema.h"
TEST(td_schema_test, build_schema_test) {
}

38
source/dnode/mgmt/exe/dmMain.c
View File

@ -17,11 +17,21 @@
#include "dmImp.h" #include "dmImp.h"
#include "tconfig.h" #include "tconfig.h"
#define DM_APOLLO_URL "The apollo string to use when configuring the server, such as: -a 'jsonFile:./tests/cfg.json', cfg.json text can be '{\"fqdn\":\"td1\"}'."
#define DM_CFG_DIR "Configuration directory."
#define DM_DMP_CFG "Dump configuration."
#define DM_ENV_CMD "The env cmd variable string to use when configuring the server, such as: -e 'TAOS_FQDN=td1'."
#define DM_ENV_FILE "The env variable file path to use when configuring the server, default is './.env', .env text can be 'TAOS_FQDN=td1'."
#define DM_NODE_TYPE "Startup type of the node, default is 0."
#define DM_MACHINE_CODE "Get machine code."
#define DM_VERSION "Print program version."
#define DM_EMAIL "<support@taosdata.com>"
static struct { static struct {
bool dumpConfig; bool dumpConfig;
bool generateGrant; bool generateGrant;
bool printAuth; bool printAuth;
bool printVersion; bool printVersion;
bool printHelp;
char envFile[PATH_MAX]; char envFile[PATH_MAX];
char apolloUrl[PATH_MAX]; char apolloUrl[PATH_MAX];
const char **envCmd; const char **envCmd;
@ -58,8 +68,9 @@ static void dmSetSignalHandle() {
static int32_t dmParseArgs(int32_t argc, char const *argv[]) { static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
int32_t cmdEnvIndex = 0; int32_t cmdEnvIndex = 0;
global.envCmd = taosMemoryMalloc(argc-1); if (argc < 2) return 0;
memset(global.envCmd, 0, argc-1); global.envCmd = taosMemoryMalloc((argc-1)*sizeof(char*));
memset(global.envCmd, 0, (argc-1)*sizeof(char*));
for (int32_t i = 1; i < argc; ++i) { for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) { if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) { if (i < argc - 1) {
@ -91,6 +102,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
} else if (strcmp(argv[i], "-e") == 0) { } else if (strcmp(argv[i], "-e") == 0) {
global.envCmd[cmdEnvIndex] = argv[++i]; global.envCmd[cmdEnvIndex] = argv[++i];
cmdEnvIndex++; cmdEnvIndex++;
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?")) {
global.printHelp = true;
} else { } else {
} }
} }
@ -111,6 +124,21 @@ static void dmPrintVersion() {
printf("buildInfo: %s\n", buildinfo); printf("buildInfo: %s\n", buildinfo);
} }
static void dmPrintHelp() {
char indent[] = " ";
printf("Usage: taosd [OPTION...] \n\n");
printf("%s%s%s%s\n", indent, "-a,", indent, DM_APOLLO_URL);
printf("%s%s%s%s\n", indent, "-c,", indent, DM_CFG_DIR);
printf("%s%s%s%s\n", indent, "-C,", indent, DM_DMP_CFG);
printf("%s%s%s%s\n", indent, "-e,", indent, DM_ENV_CMD);
printf("%s%s%s%s\n", indent, "-E,", indent, DM_ENV_FILE);
printf("%s%s%s%s\n", indent, "-n,", indent, DM_NODE_TYPE);
printf("%s%s%s%s\n", indent, "-k,", indent, DM_MACHINE_CODE);
printf("%s%s%s%s\n", indent, "-V,", indent, DM_VERSION);
printf("\n\nReport bugs to %s.\n", DM_EMAIL);
}
static void dmDumpCfg() { static void dmDumpCfg() {
SConfig *pCfg = taosGetCfg(); SConfig *pCfg = taosGetCfg();
cfgDumpCfg(pCfg, 0, true); cfgDumpCfg(pCfg, 0, true);
@ -197,6 +225,12 @@ int main(int argc, char const *argv[]) {
return 0; return 0;
} }
if (global.printHelp) {
dmPrintHelp();
taosCleanupArgs();
return 0;
}
if (global.printVersion) { if (global.printVersion) {
dmPrintVersion(); dmPrintVersion();
taosCleanupArgs(); taosCleanupArgs();

20
source/dnode/mgmt/implement/src/dmEps.c
View File

@ -120,7 +120,7 @@ int32_t dmReadEps(SDnode *pDnode) {
goto PRASE_DNODE_OVER; goto PRASE_DNODE_OVER;
} }
dnodeEp.id = dnodeId->valueint; dnodeEp.id = did->valueint;
cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn"); cJSON *dnodeFqdn = cJSON_GetObjectItem(node, "fqdn");
if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) { if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {
@ -156,11 +156,6 @@ PRASE_DNODE_OVER:
if (root != NULL) cJSON_Delete(root); if (root != NULL) cJSON_Delete(root);
if (pFile != NULL) taosCloseFile(&pFile); if (pFile != NULL) taosCloseFile(&pFile);
if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) {
dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file);
return -1;
}
if (taosArrayGetSize(pDnode->data.dnodeEps) == 0) { if (taosArrayGetSize(pDnode->data.dnodeEps) == 0) {
SDnodeEp dnodeEp = {0}; SDnodeEp dnodeEp = {0};
dnodeEp.isMnode = 1; dnodeEp.isMnode = 1;
@ -170,6 +165,11 @@ PRASE_DNODE_OVER:
dmResetEps(pDnode, pDnode->data.dnodeEps); dmResetEps(pDnode, pDnode->data.dnodeEps);
if (dmIsEpChanged(pDnode, pDnode->data.dnodeId, pDnode->data.localEp)) {
dError("localEp %s different with %s and need reconfigured", pDnode->data.localEp, file);
return -1;
}
terrno = code; terrno = code;
return code; return code;
} }
@ -291,13 +291,17 @@ static void dmPrintEps(SDnode *pDnode) {
static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep) { static bool dmIsEpChanged(SDnode *pDnode, int32_t dnodeId, const char *ep) {
bool changed = false; bool changed = false;
if (dnodeId == 0) return changed;
taosRLockLatch(&pDnode->data.latch); taosRLockLatch(&pDnode->data.latch);
SDnodeEp *pDnodeEp = taosHashGet(pDnode->data.dnodeHash, &dnodeId, sizeof(int32_t)); SDnodeEp *pDnodeEp = taosHashGet(pDnode->data.dnodeHash, &dnodeId, sizeof(int32_t));
if (pDnodeEp != NULL) { if (pDnodeEp != NULL) {
char epstr[TSDB_EP_LEN + 1]; char epstr[TSDB_EP_LEN + 1] = {0};
snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port); snprintf(epstr, TSDB_EP_LEN, "%s:%u", pDnodeEp->ep.fqdn, pDnodeEp->ep.port);
changed = strcmp(ep, epstr) != 0; changed = (strcmp(ep, epstr) != 0);
if (changed) {
dError("dnode:%d, localEp %s different from %s", dnodeId, ep, epstr);
}
} }
taosRUnLockLatch(&pDnode->data.latch); taosRUnLockLatch(&pDnode->data.latch);

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -211,7 +211,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, mmProcessReadMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_MQ_ASK_EP, mmProcessReadMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_STREAM, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

21
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -107,15 +107,12 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->vgId = pCreate->vgId; pCfg->vgId = pCreate->vgId;
strcpy(pCfg->dbname, pCreate->db); strcpy(pCfg->dbname, pCreate->db);
pCfg->wsize = 16 * 1024 * 1024;
pCfg->ssize = 1024;
pCfg->lsize = 1024 * 1024;
pCfg->streamMode = pCreate->streamMode; pCfg->streamMode = pCreate->streamMode;
pCfg->isWeak = true; pCfg->isWeak = true;
pCfg->tsdbCfg.keep2 = pCreate->durationToKeep0; pCfg->tsdbCfg.days = 10;
pCfg->tsdbCfg.keep0 = pCreate->durationToKeep2; pCfg->tsdbCfg.keep2 = 3650;
pCfg->tsdbCfg.keep1 = pCreate->durationToKeep0; pCfg->tsdbCfg.keep0 = 3650;
pCfg->tsdbCfg.lruCacheSize = 16; pCfg->tsdbCfg.keep1 = 3650;
pCfg->tsdbCfg.retentions = pCreate->pRetensions; pCfg->tsdbCfg.retentions = pCreate->pRetensions;
pCfg->walCfg.vgId = pCreate->vgId; pCfg->walCfg.vgId = pCreate->vgId;
pCfg->hashBegin = pCreate->hashBegin; pCfg->hashBegin = pCreate->hashBegin;
@ -202,6 +199,16 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
return code; return code;
} }
code = vnodeStart(pImpl);
if (code != 0) {
tFreeSCreateVnodeReq(&createReq);
dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr());
vnodeClose(pImpl);
vnodeDestroy(path, pMgmt->pTfs);
terrno = code;
return code;
}
code = vmWriteVnodesToFile(pMgmt); code = vmWriteVnodesToFile(pMgmt);
if (code != 0) { if (code != 0) {
tFreeSCreateVnodeReq(&createReq); tFreeSCreateVnodeReq(&createReq);

51
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -74,12 +74,6 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
return -1; return -1;
} }
// sync integration
vnodeSyncSetQ(pImpl, NULL);
vnodeSyncSetRpc(pImpl, NULL);
int32_t ret = vnodeSyncStart(pImpl);
assert(ret == 0);
taosWLockLatch(&pMgmt->latch); taosWLockLatch(&pMgmt->latch);
int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
taosWUnLockLatch(&pMgmt->latch); taosWUnLockLatch(&pMgmt->latch);
@ -153,6 +147,7 @@ static void *vmOpenVnodeFunc(void *param) {
pThread->failed++; pThread->failed++;
} else { } else {
vmOpenVnode(pMgmt, pCfg, pImpl); vmOpenVnode(pMgmt, pCfg, pImpl);
//vnodeStart(pImpl);
dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->opened++; pThread->opened++;
} }
@ -364,10 +359,52 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) {
return 0; return 0;
} }
static int32_t vmStart(SMgmtWrapper *pWrapper) {
dDebug("vnode-mgmt start to run");
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
taosRLockLatch(&pMgmt->latch);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode;
vnodeStart(pVnode->pImpl);
pIter = taosHashIterate(pMgmt->hash, pIter);
}
taosRUnLockLatch(&pMgmt->latch);
return 0;
}
static void vmStop(SMgmtWrapper *pWrapper) {
#if 0
dDebug("vnode-mgmt start to stop");
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
taosRLockLatch(&pMgmt->latch);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode;
vnodeStop(pVnode->pImpl);
pIter = taosHashIterate(pMgmt->hash, pIter);
}
taosRUnLockLatch(&pMgmt->latch);
#endif
}
void vmSetMgmtFp(SMgmtWrapper *pWrapper) { void vmSetMgmtFp(SMgmtWrapper *pWrapper) {
SMgmtFp mgmtFp = {0}; SMgmtFp mgmtFp = {0};
mgmtFp.openFp = vmInit; mgmtFp.openFp = vmInit;
mgmtFp.closeFp = vmCleanup; mgmtFp.closeFp = vmCleanup;
mgmtFp.startFp = vmStart;
mgmtFp.stopFp = vmStop;
mgmtFp.requiredFp = vmRequire; mgmtFp.requiredFp = vmRequire;
vmInitMsgHandle(pWrapper); vmInitMsgHandle(pWrapper);
@ -396,4 +433,4 @@ void vmGetVnodeLoads(SMgmtWrapper *pWrapper, SMonVloadInfo *pInfo) {
} }
taosRUnLockLatch(&pMgmt->latch); taosRUnLockLatch(&pMgmt->latch);
} }

2
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -23,10 +23,8 @@ extern "C" {
#endif #endif
enum { enum {
// MQ_CONSUMER_STATUS__INIT = 1,
MQ_CONSUMER_STATUS__MODIFY = 1, MQ_CONSUMER_STATUS__MODIFY = 1,
MQ_CONSUMER_STATUS__MODIFY_IN_REB, MQ_CONSUMER_STATUS__MODIFY_IN_REB,
// MQ_CONSUMER_STATUS__IDLE,
MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__READY,
MQ_CONSUMER_STATUS__LOST, MQ_CONSUMER_STATUS__LOST,
MQ_CONSUMER_STATUS__LOST_IN_REB, MQ_CONSUMER_STATUS__LOST_IN_REB,

50
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -89,6 +89,7 @@ typedef enum {
TRN_TYPE_DROP_STREAM = 1020, TRN_TYPE_DROP_STREAM = 1020,
TRN_TYPE_ALTER_STREAM = 1021, TRN_TYPE_ALTER_STREAM = 1021,
TRN_TYPE_CONSUMER_LOST = 1022, TRN_TYPE_CONSUMER_LOST = 1022,
TRN_TYPE_CONSUMER_RECOVER = 1023,
TRN_TYPE_BASIC_SCOPE_END, TRN_TYPE_BASIC_SCOPE_END,
TRN_TYPE_GLOBAL_SCOPE = 2000, TRN_TYPE_GLOBAL_SCOPE = 2000,
TRN_TYPE_CREATE_DNODE = 2001, TRN_TYPE_CREATE_DNODE = 2001,
@ -148,6 +149,10 @@ typedef struct {
int64_t dbUid; int64_t dbUid;
char dbname[TSDB_DB_FNAME_LEN]; char dbname[TSDB_DB_FNAME_LEN];
char lastError[TSDB_TRANS_ERROR_LEN]; char lastError[TSDB_TRANS_ERROR_LEN];
int32_t startFunc;
int32_t stopFunc;
int32_t paramLen;
void* param;
} STrans; } STrans;
typedef struct { typedef struct {
@ -348,18 +353,15 @@ typedef struct {
int32_t version; int32_t version;
int32_t nextColId; int32_t nextColId;
float xFilesFactor; float xFilesFactor;
int32_t aggregationMethod;
int32_t delay; int32_t delay;
int32_t ttl; int32_t ttl;
int32_t numOfColumns; int32_t numOfColumns;
int32_t numOfTags; int32_t numOfTags;
int32_t numOfSmas;
int32_t commentLen; int32_t commentLen;
int32_t ast1Len; int32_t ast1Len;
int32_t ast2Len; int32_t ast2Len;
SSchema* pColumns; SSchema* pColumns;
SSchema* pTags; SSchema* pTags;
SSchema* pSmas;
char* comment; char* comment;
char* pAst1; char* pAst1;
char* pAst2; char* pAst2;
@ -435,14 +437,12 @@ static FORCE_INLINE void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset)
} }
typedef struct { typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
int64_t createTime; int64_t createTime;
int64_t updateTime; int64_t updateTime;
int64_t uid; int64_t uid;
// TODO: use subDbUid
int64_t dbUid; int64_t dbUid;
int64_t subDbUid;
int32_t version; int32_t version;
int8_t subType; // db or table int8_t subType; // db or table
int8_t withTbName; int8_t withTbName;
@ -462,12 +462,14 @@ enum {
CONSUMER_UPDATE__ADD, CONSUMER_UPDATE__ADD,
CONSUMER_UPDATE__REMOVE, CONSUMER_UPDATE__REMOVE,
CONSUMER_UPDATE__LOST, CONSUMER_UPDATE__LOST,
CONSUMER_UPDATE__RECOVER,
CONSUMER_UPDATE__MODIFY, CONSUMER_UPDATE__MODIFY,
}; };
typedef struct { typedef struct {
int64_t consumerId; int64_t consumerId;
char cgroup[TSDB_CGROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
char appId[TSDB_CGROUP_LEN];
int8_t updateType; // used only for update int8_t updateType; // used only for update
int32_t epoch; int32_t epoch;
int32_t status; int32_t status;
@ -478,6 +480,17 @@ typedef struct {
SArray* currentTopics; // SArray<char*> SArray* currentTopics; // SArray<char*>
SArray* rebNewTopics; // SArray<char*> SArray* rebNewTopics; // SArray<char*>
SArray* rebRemovedTopics; // SArray<char*> SArray* rebRemovedTopics; // SArray<char*>
// subscribed by user
SArray* assignedTopics; // SArray<char*>
// data for display
int32_t pid;
SEpSet ep;
int64_t upTime;
int64_t subscribeTime;
int64_t rebalanceTime;
} SMqConsumerObj; } SMqConsumerObj;
SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]); SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);
@ -499,12 +512,12 @@ void* tDecodeSMqVgEp(const void* buf, SMqVgEp* pVgEp);
typedef struct { typedef struct {
int64_t consumerId; // -1 for unassigned int64_t consumerId; // -1 for unassigned
SArray* vgs; // SArray<SMqVgEp*> SArray* vgs; // SArray<SMqVgEp*>
} SMqConsumerEpInSub; } SMqConsumerEp;
SMqConsumerEpInSub* tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub* pEpInSub); SMqConsumerEp* tCloneSMqConsumerEp(const SMqConsumerEp* pEp);
void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub* pEpInSub); void tDeleteSMqConsumerEp(SMqConsumerEp* pEp);
int32_t tEncodeSMqConsumerEpInSub(void** buf, const SMqConsumerEpInSub* pEpInSub); int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp);
void* tDecodeSMqConsumerEpInSub(const void* buf, SMqConsumerEpInSub* pEpInSub); void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp);
typedef struct { typedef struct {
char key[TSDB_SUBSCRIBE_KEY_LEN]; char key[TSDB_SUBSCRIBE_KEY_LEN];
@ -514,9 +527,8 @@ typedef struct {
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub SHashObj* consumerHash; // consumerId -> SMqConsumerEp
// TODO put -1 into unassignVgs SArray* unassignedVgs; // SArray<SMqVgEp*>
// SArray* unassignedVgs;
} SMqSubscribeObj; } SMqSubscribeObj;
SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]); SMqSubscribeObj* tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]);
@ -527,7 +539,7 @@ void* tDecodeSubscribeObj(const void* buf, SMqSubscribeObj* pSub);
typedef struct { typedef struct {
int32_t epoch; int32_t epoch;
SArray* consumers; // SArray<SMqConsumerEpInSub*> SArray* consumers; // SArray<SMqConsumerEp*>
} SMqSubActionLogEntry; } SMqSubActionLogEntry;
SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry); SMqSubActionLogEntry* tCloneSMqSubActionLogEntry(SMqSubActionLogEntry* pEntry);

56
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -40,12 +40,12 @@ extern "C" {
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg); typedef int32_t (*MndMsgFp)(SNodeMsg *pMsg);
typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef int32_t (*MndInitFp)(SMnode *pMnode);
typedef void (*MndCleanupFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode);
typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); typedef int32_t (*ShowRetrieveFp)(SNodeMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter);
typedef struct SQWorkerMgmt SQHandle; typedef struct SQWorkerMgmt SQHandle;
@ -84,32 +84,32 @@ typedef struct {
int64_t timeseriesAllowed; int64_t timeseriesAllowed;
} SGrantInfo; } SGrantInfo;
struct SMnode { typedef struct SMnode {
int32_t selfId; int32_t selfId;
int64_t clusterId; int64_t clusterId;
int8_t replica; int8_t replica;
int8_t selfIndex; int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA]; SReplica replicas[TSDB_MAX_REPLICA];
tmr_h timer; tmr_h timer;
tmr_h transTimer; tmr_h transTimer;
tmr_h mqTimer; tmr_h mqTimer;
tmr_h telemTimer; tmr_h telemTimer;
char *path; char *path;
int64_t checkTime; int64_t checkTime;
SSdb *pSdb; SSdb *pSdb;
SMgmtWrapper *pWrapper; SMgmtWrapper *pWrapper;
SArray *pSteps; SArray *pSteps;
SQHandle *pQuery; SQHandle *pQuery;
SShowMgmt showMgmt; SShowMgmt showMgmt;
SProfileMgmt profileMgmt; SProfileMgmt profileMgmt;
STelemMgmt telemMgmt; STelemMgmt telemMgmt;
SSyncMgmt syncMgmt; SSyncMgmt syncMgmt;
SHashObj *infosMeta; SHashObj *infosMeta;
SHashObj *perfsMeta; SHashObj *perfsMeta;
SGrantInfo grant; SGrantInfo grant;
MndMsgFp msgFp[TDMT_MAX]; MndMsgFp msgFp[TDMT_MAX];
SMsgCb msgCb; SMsgCb msgCb;
}; } SMnode;
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
int64_t mndGenerateUid(char *name, int32_t len); int64_t mndGenerateUid(char *name, int32_t len);

10
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -33,6 +33,15 @@ typedef struct {
void *pCont; void *pCont;
} STransAction; } STransAction;
typedef enum {
TEST_TRANS_START_FUNC = 1,
TEST_TRANS_STOP_FUNC = 2,
CONSUME_TRANS_START_FUNC = 3,
CONSUME_TRANS_STOP_FUNC = 4,
} ETrnFuncType;
typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen);
int32_t mndInitTrans(SMnode *pMnode); int32_t mndInitTrans(SMnode *pMnode);
void mndCleanupTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode);
@ -44,6 +53,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen);
void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen);
void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);

1
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -30,7 +30,6 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
int32_t mndGetGlobalVgroupVersion(int32_t *vgId);
void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

20
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -17,8 +17,8 @@
#include "mndAcct.h" #include "mndAcct.h"
#include "mndShow.h" #include "mndShow.h"
#define TSDB_ACCT_VER_NUMBER 1 #define ACCT_VER_NUMBER 1
#define TSDB_ACCT_RESERVE_SIZE 128 #define ACCT_RESERVE_SIZE 128
static int32_t mndCreateDefaultAcct(SMnode *pMnode); static int32_t mndCreateDefaultAcct(SMnode *pMnode);
static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct);
@ -55,6 +55,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
acctObj.createdTime = taosGetTimestampMs(); acctObj.createdTime = taosGetTimestampMs();
acctObj.updateTime = acctObj.createdTime; acctObj.updateTime = acctObj.createdTime;
acctObj.acctId = 1; acctObj.acctId = 1;
acctObj.status = 0;
acctObj.cfg = (SAcctCfg){.maxUsers = INT32_MAX, acctObj.cfg = (SAcctCfg){.maxUsers = INT32_MAX,
.maxDbs = INT32_MAX, .maxDbs = INT32_MAX,
.maxStbs = INT32_MAX, .maxStbs = INT32_MAX,
@ -79,7 +80,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj) + TSDB_ACCT_RESERVE_SIZE); SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, ACCT_VER_NUMBER, sizeof(SAcctObj) + ACCT_RESERVE_SIZE);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
@ -100,7 +101,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxTopics, _OVER) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxTopics, _OVER)
SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage, _OVER) SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage, _OVER)
SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState, _OVER) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE, _OVER) SDB_SET_RESERVE(pRaw, dataPos, ACCT_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER)
terrno = 0; terrno = 0;
@ -122,7 +123,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != TSDB_ACCT_VER_NUMBER) { if (sver != ACCT_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER; goto _OVER;
} }
@ -151,7 +152,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.maxTopics, _OVER) SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.maxTopics, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pAcct->cfg.maxStorage, _OVER) SDB_GET_INT64(pRaw, dataPos, &pAcct->cfg.maxStorage, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.accessState, _OVER) SDB_GET_INT32(pRaw, dataPos, &pAcct->cfg.accessState, _OVER)
SDB_GET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE, _OVER) SDB_GET_RESERVE(pRaw, dataPos, ACCT_RESERVE_SIZE, _OVER)
terrno = 0; terrno = 0;
@ -178,7 +179,6 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) {
static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
mTrace("acct:%s, perform update action, old row:%p new row:%p", pOld->acct, pOld, pNew); mTrace("acct:%s, perform update action, old row:%p new row:%p", pOld->acct, pOld, pNew);
pOld->updateTime = pNew->updateTime; pOld->updateTime = pNew->updateTime;
pOld->status = pNew->status; pOld->status = pNew->status;
memcpy(&pOld->cfg, &pNew->cfg, sizeof(SAcctCfg)); memcpy(&pOld->cfg, &pNew->cfg, sizeof(SAcctCfg));
@ -186,19 +186,19 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) {
} }
static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq) { static int32_t mndProcessCreateAcctReq(SNodeMsg *pReq) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", terrstr());
return -1; return -1;
} }
static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq) { static int32_t mndProcessAlterAcctReq(SNodeMsg *pReq) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", terrstr());
return -1; return -1;
} }
static int32_t mndProcessDropAcctReq(SNodeMsg *pReq) { static int32_t mndProcessDropAcctReq(SNodeMsg *pReq) {
terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("failed to process create acct request since %s", terrstr()); mError("failed to process create acct request since %s", terrstr());
return -1; return -1;
} }

223
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -37,6 +37,8 @@
static int8_t mqInRebFlag = 0; static int8_t mqInRebFlag = 0;
static const char *mndConsumerStatusName(int status);
static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
@ -48,6 +50,7 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg);
static int32_t mndProcessAskEpReq(SNodeMsg *pMsg); static int32_t mndProcessAskEpReq(SNodeMsg *pMsg);
static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg); static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg);
static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg); static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg);
static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg);
int32_t mndInitConsumer(SMnode *pMnode) { int32_t mndInitConsumer(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_CONSUMER, SSdbTable table = {.sdbType = SDB_CONSUMER,
@ -59,9 +62,14 @@ int32_t mndInitConsumer(SMnode *pMnode) {
.deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; .deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndCancelGetNextConsumer);
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
@ -86,7 +94,31 @@ static int32_t mndProcessConsumerLostMsg(SNodeMsg *pMsg) {
mndTransDrop(pTrans); mndTransDrop(pTrans);
return 0; return 0;
FAIL: FAIL:
// TODO delete consumer tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans);
return -1;
}
static int32_t mndProcessConsumerRecoverMsg(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode;
SMqConsumerRecoverMsg *pRecoverMsg = pMsg->rpcMsg.pCont;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pRecoverMsg->consumerId);
ASSERT(pConsumer);
SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumer->consumerId, pConsumer->cgroup);
pConsumerNew->updateType = CONSUMER_UPDATE__RECOVER;
mndReleaseConsumer(pMnode, pConsumer);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_CONSUMER_RECOVER, &pMsg->rpcMsg);
if (pTrans == NULL) goto FAIL;
if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL;
if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL;
mndTransDrop(pTrans);
return 0;
FAIL:
tDeleteSMqConsumerObj(pConsumerNew);
mndTransDrop(pTrans); mndTransDrop(pTrans);
return -1; return -1;
} }
@ -197,11 +229,11 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) {
} }
static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) { static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode; SMnode *pMnode = pMsg->pNode;
SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->rpcMsg.pCont; SMqAskEpReq *pReq = (SMqAskEpReq *)pMsg->rpcMsg.pCont;
SMqCMGetSubEpRsp rsp = {0}; SMqAskEpRsp rsp = {0};
int64_t consumerId = be64toh(pReq->consumerId); int64_t consumerId = be64toh(pReq->consumerId);
int32_t epoch = ntohl(pReq->epoch); int32_t epoch = ntohl(pReq->epoch);
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId); SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pNode, consumerId);
if (pConsumer == NULL) { if (pConsumer == NULL) {
@ -216,8 +248,15 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
// 1. check consumer status // 1. check consumer status
int32_t status = atomic_load_32(&pConsumer->status); int32_t status = atomic_load_32(&pConsumer->status);
if (status == MQ_CONSUMER_STATUS__LOST) { if (status == MQ_CONSUMER_STATUS__LOST_REBD) {
// TODO: recover consumer SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg));
pRecoverMsg->consumerId = consumerId;
SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg));
pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER;
pRpcMsg->pCont = pRecoverMsg;
pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg);
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg);
} }
if (status != MQ_CONSUMER_STATUS__READY) { if (status != MQ_CONSUMER_STATUS__READY) {
@ -263,8 +302,8 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
// 2.2 iterate all vg assigned to the consumer of that topic // 2.2 iterate all vg assigned to the consumer of that topic
SMqConsumerEpInSub *pEpInSub = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t));
int32_t vgNum = taosArrayGetSize(pEpInSub->vgs); int32_t vgNum = taosArrayGetSize(pConsumerEp->vgs);
topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp)); topicEp.vgs = taosArrayInit(vgNum, sizeof(SMqSubVgEp));
if (topicEp.vgs == NULL) { if (topicEp.vgs == NULL) {
@ -274,7 +313,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
} }
for (int32_t j = 0; j < vgNum; j++) { for (int32_t j = 0; j < vgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
char offsetKey[TSDB_PARTITION_KEY_LEN]; char offsetKey[TSDB_PARTITION_KEY_LEN];
mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId);
// 2.2.1 build vg ep // 2.2.1 build vg ep
@ -300,7 +339,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
} }
// encode rsp // encode rsp
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqCMGetSubEpRsp(NULL, &rsp); int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqAskEpRsp(NULL, &rsp);
void *buf = rpcMallocCont(tlen); void *buf = rpcMallocCont(tlen);
if (buf == NULL) { if (buf == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -311,10 +350,10 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId;
void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); tEncodeSMqAskEpRsp(&abuf, &rsp);
// release consumer and free memory // release consumer and free memory
tDeleteSMqCMGetSubEpRsp(&rsp); tDeleteSMqAskEpRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
// send rsp // send rsp
@ -322,7 +361,7 @@ static int32_t mndProcessAskEpReq(SNodeMsg *pMsg) {
pMsg->rspLen = tlen; pMsg->rspLen = tlen;
return 0; return 0;
FAIL: FAIL:
tDeleteSMqCMGetSubEpRsp(&rsp); tDeleteSMqAskEpRsp(&rsp);
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
return -1; return -1;
} }
@ -366,10 +405,14 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
if (pConsumerOld == NULL) { if (pConsumerOld == NULL) {
pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup); pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup);
pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
/*pConsumerNew->waitingRebTopics = newSub;*/
pConsumerNew->rebNewTopics = newSub; pConsumerNew->rebNewTopics = newSub;
subscribe.topicNames = NULL; subscribe.topicNames = NULL;
for (int32_t i = 0; i < newTopicNum; i++) {
char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
if (pTrans == NULL) goto SUBSCRIBE_OVER; if (pTrans == NULL) goto SUBSCRIBE_OVER;
if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER; if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) goto SUBSCRIBE_OVER;
@ -389,7 +432,11 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
goto SUBSCRIBE_OVER; goto SUBSCRIBE_OVER;
} }
pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY;
/*pConsumerOld->waitingRebTopics = newSub;*/
for (int32_t i = 0; i < newTopicNum; i++) {
char *newTopicCopy = strdup(taosArrayGetP(newSub, i));
taosArrayPush(pConsumerNew->assignedTopics, &newTopicCopy);
}
int32_t oldTopicNum = 0; int32_t oldTopicNum = 0;
if (pConsumerOld->currentTopics) { if (pConsumerOld->currentTopics) {
@ -532,6 +579,7 @@ CM_DECODE_OVER:
static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId); mTrace("consumer:%" PRId64 ", perform insert action", pConsumer->consumerId);
pConsumer->subscribeTime = pConsumer->upTime;
return 0; return 0;
} }
@ -557,17 +605,45 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics;
pNewConsumer->rebRemovedTopics = tmp; pNewConsumer->rebRemovedTopics = tmp;
tmp = pOldConsumer->assignedTopics;
pOldConsumer->assignedTopics = pNewConsumer->assignedTopics;
pNewConsumer->assignedTopics = tmp;
pOldConsumer->subscribeTime = pNewConsumer->upTime;
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) {
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
ASSERT(taosArrayGetSize(pOldConsumer->rebRemovedTopics) == 0);
int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics);
pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *)); /*pOldConsumer->rebRemovedTopics = taosArrayInit(sz, sizeof(void *));*/
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i)); char *topic = strdup(taosArrayGetP(pOldConsumer->currentTopics, i));
taosArrayPush(pNewConsumer->rebRemovedTopics, &topic); taosArrayPush(pOldConsumer->rebRemovedTopics, &topic);
} }
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; pOldConsumer->status = MQ_CONSUMER_STATUS__LOST;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) {
ASSERT(taosArrayGetSize(pOldConsumer->currentTopics) == 0);
ASSERT(taosArrayGetSize(pOldConsumer->rebNewTopics) == 0);
int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) {
char *topic = strdup(taosArrayGetP(pOldConsumer->assignedTopics, i));
taosArrayPush(pOldConsumer->rebNewTopics, &topic);
}
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) {
atomic_add_fetch_32(&pOldConsumer->epoch, 1); atomic_add_fetch_32(&pOldConsumer->epoch, 1);
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) {
ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1); ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 1);
ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0); ASSERT(taosArrayGetSize(pNewConsumer->rebRemovedTopics) == 0);
@ -612,6 +688,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
} }
} }
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
atomic_add_fetch_32(&pOldConsumer->epoch, 1); atomic_add_fetch_32(&pOldConsumer->epoch, 1);
} else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) { } else if (pNewConsumer->updateType == CONSUMER_UPDATE__REMOVE) {
ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0); ASSERT(taosArrayGetSize(pNewConsumer->rebNewTopics) == 0);
@ -668,6 +747,9 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer,
pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB; pOldConsumer->status = MQ_CONSUMER_STATUS__LOST_IN_REB;
} }
} }
pOldConsumer->rebalanceTime = pNewConsumer->upTime;
atomic_add_fetch_32(&pOldConsumer->epoch, 1); atomic_add_fetch_32(&pOldConsumer->epoch, 1);
} }
@ -688,3 +770,104 @@ void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pConsumer); sdbRelease(pSdb, pConsumer);
} }
static int32_t mndRetrieveConsumer(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SMqConsumerObj *pConsumer = NULL;
while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
if (pShow->pIter == NULL) break;
SColumnInfoData *pColInfo;
int32_t cols = 0;
taosRLockLatch(&pConsumer->lock);
// consumer id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->consumerId, false);
// group id
char groupId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(varDataVal(groupId), pConsumer->cgroup, TSDB_CGROUP_LEN);
varDataSetLen(groupId, strlen(varDataVal(groupId)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)groupId, false);
// app id
char appId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(varDataVal(appId), pConsumer->appId, TSDB_CGROUP_LEN);
varDataSetLen(appId, strlen(varDataVal(appId)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)appId, false);
// status
char status[20 + VARSTR_HEADER_SIZE] = {0};
tstrncpy(varDataVal(status), mndConsumerStatusName(pConsumer->status), 20);
varDataSetLen(status, strlen(varDataVal(status)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)status, false);
// subscribed topics
char topics[TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE] = {0};
char *showStr = taosShowStrArray(pConsumer->assignedTopics);
tstrncpy(varDataVal(topics), showStr, TSDB_SHOW_LIST_LEN);
taosMemoryFree(showStr);
varDataSetLen(topics, strlen(varDataVal(topics)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)topics, false);
// pid
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->pid, true);
// end point
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->ep, true);
// up time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->upTime, false);
// subscribe time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->subscribeTime, false);
// rebalance time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0);
taosRUnLockLatch(&pConsumer->lock);
sdbRelease(pSdb, pConsumer);
numOfRows++;
}
pShow->numOfRows += numOfRows;
return numOfRows;
}
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}
static const char *mndConsumerStatusName(int status) {
switch (status) {
case MQ_CONSUMER_STATUS__READY:
return "ready";
case MQ_CONSUMER_STATUS__LOST:
case MQ_CONSUMER_STATUS__LOST_REBD:
case MQ_CONSUMER_STATUS__LOST_IN_REB:
return "lost";
case MQ_CONSUMER_STATUS__MODIFY:
case MQ_CONSUMER_STATUS__MODIFY_IN_REB:
return "rebalancing";
default:
return "unknown";
}
}

32
source/dnode/mnode/impl/src/mndDb.c
View File

@ -230,6 +230,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
return 0; return 0;
} }
static int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { return sdbGetTableVer(pMnode->pSdb, SDB_VGROUP); }
SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db);
@ -262,22 +264,22 @@ static int32_t mndCheckDbName(const char *dbName, SUserObj *pUser) {
static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1; if (pCfg->numOfVgroups < TSDB_MIN_VNODES_PER_DB || pCfg->numOfVgroups > TSDB_MAX_VNODES_PER_DB) return -1;
if (pCfg->numOfStables < TSDB_MIN_STBS_PER_DB || pCfg->numOfStables > TSDB_MAX_STBS_PER_DB) return -1; /*
if (pCfg->buffer < TSDB_MIN_BUFFER_SIZE || pCfg->buffer > TSDB_MAX_BUFFER_SIZE) return -1; if (pCfg->cacheBlockSize < TSDB_MIN_CACHE_BLOCK_SIZE || pCfg->cacheBlockSize > TSDB_MAX_CACHE_BLOCK_SIZE) return -1;
if (pCfg->pageSize < TSDB_MIN_PAGE_SIZE || pCfg->pageSize > TSDB_MAX_PAGE_SIZE) return -1; if (pCfg->totalBlocks < TSDB_MIN_TOTAL_BLOCKS || pCfg->totalBlocks > TSDB_MAX_TOTAL_BLOCKS) return -1;
if (pCfg->pages < TSDB_MIN_TOTAL_PAGES || pCfg->pages > TSDB_MAX_TOTAL_PAGES) return -1; */
if (pCfg->durationPerFile < TSDB_MIN_DURATION_PER_FILE || pCfg->durationPerFile > TSDB_MAX_DURATION_PER_FILE) if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) return -1;
return -1; if (pCfg->daysToKeep0 < TSDB_MIN_KEEP || pCfg->daysToKeep0 > TSDB_MAX_KEEP) return -1;
if (pCfg->durationToKeep0 < TSDB_MIN_KEEP || pCfg->durationToKeep0 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > TSDB_MAX_KEEP) return -1;
if (pCfg->durationToKeep1 < TSDB_MIN_KEEP || pCfg->durationToKeep1 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > TSDB_MAX_KEEP) return -1;
if (pCfg->durationToKeep2 < TSDB_MIN_KEEP || pCfg->durationToKeep2 > TSDB_MAX_KEEP) return -1; if (pCfg->daysToKeep0 < pCfg->daysPerFile) return -1;
if (pCfg->durationToKeep0 < pCfg->durationPerFile) return -1; if (pCfg->daysToKeep0 > pCfg->daysToKeep1) return -1;
if (pCfg->durationToKeep0 > pCfg->durationToKeep1) return -1; if (pCfg->daysToKeep1 > pCfg->daysToKeep2) return -1;
if (pCfg->durationToKeep1 > pCfg->durationToKeep2) return -1;
if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1; if (pCfg->minRows < TSDB_MIN_MINROWS_FBLOCK || pCfg->minRows > TSDB_MAX_MINROWS_FBLOCK) return -1;
if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1; if (pCfg->maxRows < TSDB_MIN_MAXROWS_FBLOCK || pCfg->maxRows > TSDB_MAX_MAXROWS_FBLOCK) return -1;
if (pCfg->minRows > pCfg->maxRows) return -1; if (pCfg->minRows > pCfg->maxRows) return -1;
if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1; if (pCfg->fsyncPeriod < TSDB_MIN_FSYNC_PERIOD || pCfg->fsyncPeriod > TSDB_MAX_FSYNC_PERIOD) return -1;
// if (pCfg->ttl < TSDB_MIN_TABLE_TTL) return -1;
if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1; if (pCfg->walLevel < TSDB_MIN_WAL_LEVEL || pCfg->walLevel > TSDB_MAX_WAL_LEVEL) return -1;
if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1; if (pCfg->precision < TSDB_MIN_PRECISION && pCfg->precision > TSDB_MAX_PRECISION) return -1;
if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1; if (pCfg->compression < TSDB_MIN_COMP_LEVEL || pCfg->compression > TSDB_MAX_COMP_LEVEL) return -1;
@ -304,6 +306,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK; if (pCfg->minRows < 0) pCfg->minRows = TSDB_DEFAULT_MINROWS_FBLOCK;
if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK; if (pCfg->maxRows < 0) pCfg->maxRows = TSDB_DEFAULT_MAXROWS_FBLOCK;
if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; if (pCfg->fsyncPeriod < 0) pCfg->fsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
if (pCfg->ttl < 0) pCfg->ttl = TSDB_DEFAULT_TABLE_TTL;
if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL; if (pCfg->walLevel < 0) pCfg->walLevel = TSDB_DEFAULT_WAL_LEVEL;
if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION; if (pCfg->precision < 0) pCfg->precision = TSDB_DEFAULT_PRECISION;
if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL; if (pCfg->compression < 0) pCfg->compression = TSDB_DEFAULT_COMP_LEVEL;
@ -624,7 +627,7 @@ static int32_t mndSetAlterDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOl
SSdbRaw *pRedoRaw = mndDbActionEncode(pOld); SSdbRaw *pRedoRaw = mndDbActionEncode(pOld);
if (pRedoRaw == NULL) return -1; if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_UPDATING) != 0) return -1; if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
return 0; return 0;
} }
@ -1199,8 +1202,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
char *p = strchr(usedbReq.db, '.'); char *p = strchr(usedbReq.db, '.');
if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
// mndGetGlobalVgroupVersion(); TODO int32_t vgVersion = mndGetGlobalVgroupVersion(pMnode);
static int32_t vgVersion = 1;
if (usedbReq.vgVersion < vgVersion) { if (usedbReq.vgVersion < vgVersion) {
usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
if (usedbRsp.pVgroupInfos == NULL) { if (usedbRsp.pVgroupInfos == NULL) {

142
source/dnode/mnode/impl/src/mndDef.c
View File

@ -34,15 +34,20 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L
pConsumer->currentTopics = taosArrayInit(0, sizeof(void *)); pConsumer->currentTopics = taosArrayInit(0, sizeof(void *));
pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *)); pConsumer->rebNewTopics = taosArrayInit(0, sizeof(void *));
pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *)); pConsumer->rebRemovedTopics = taosArrayInit(0, sizeof(void *));
pConsumer->assignedTopics = taosArrayInit(0, sizeof(void *));
if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL) { if (pConsumer->currentTopics == NULL || pConsumer->rebNewTopics == NULL || pConsumer->rebRemovedTopics == NULL ||
pConsumer->assignedTopics == NULL) {
taosArrayDestroy(pConsumer->currentTopics); taosArrayDestroy(pConsumer->currentTopics);
taosArrayDestroy(pConsumer->rebNewTopics); taosArrayDestroy(pConsumer->rebNewTopics);
taosArrayDestroy(pConsumer->rebRemovedTopics); taosArrayDestroy(pConsumer->rebRemovedTopics);
taosArrayDestroy(pConsumer->assignedTopics);
taosMemoryFree(pConsumer); taosMemoryFree(pConsumer);
return NULL; return NULL;
} }
pConsumer->upTime = taosGetTimestampMs();
return pConsumer; return pConsumer;
} }
@ -56,6 +61,9 @@ void tDeleteSMqConsumerObj(SMqConsumerObj *pConsumer) {
if (pConsumer->rebRemovedTopics) { if (pConsumer->rebRemovedTopics) {
taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree); taosArrayDestroyP(pConsumer->rebRemovedTopics, (FDelete)taosMemoryFree);
} }
if (pConsumer->assignedTopics) {
taosArrayDestroyP(pConsumer->assignedTopics, (FDelete)taosMemoryFree);
}
} }
int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
@ -67,6 +75,12 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
tlen += taosEncodeFixedI32(buf, pConsumer->epoch); tlen += taosEncodeFixedI32(buf, pConsumer->epoch);
tlen += taosEncodeFixedI32(buf, pConsumer->status); tlen += taosEncodeFixedI32(buf, pConsumer->status);
tlen += taosEncodeFixedI32(buf, pConsumer->pid);
tlen += taosEncodeSEpSet(buf, &pConsumer->ep);
tlen += taosEncodeFixedI64(buf, pConsumer->upTime);
tlen += taosEncodeFixedI64(buf, pConsumer->subscribeTime);
tlen += taosEncodeFixedI64(buf, pConsumer->rebalanceTime);
// current topics // current topics
if (pConsumer->currentTopics) { if (pConsumer->currentTopics) {
sz = taosArrayGetSize(pConsumer->currentTopics); sz = taosArrayGetSize(pConsumer->currentTopics);
@ -103,6 +117,18 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) {
tlen += taosEncodeFixedI32(buf, 0); tlen += taosEncodeFixedI32(buf, 0);
} }
// lost topics
if (pConsumer->assignedTopics) {
sz = taosArrayGetSize(pConsumer->assignedTopics);
tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) {
char *topic = taosArrayGetP(pConsumer->assignedTopics, i);
tlen += taosEncodeString(buf, topic);
}
} else {
tlen += taosEncodeFixedI32(buf, 0);
}
return tlen; return tlen;
} }
@ -114,6 +140,12 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
buf = taosDecodeFixedI32(buf, &pConsumer->epoch); buf = taosDecodeFixedI32(buf, &pConsumer->epoch);
buf = taosDecodeFixedI32(buf, &pConsumer->status); buf = taosDecodeFixedI32(buf, &pConsumer->status);
buf = taosDecodeFixedI32(buf, &pConsumer->pid);
buf = taosDecodeSEpSet(buf, &pConsumer->ep);
buf = taosDecodeFixedI64(buf, &pConsumer->upTime);
buf = taosDecodeFixedI64(buf, &pConsumer->subscribeTime);
buf = taosDecodeFixedI64(buf, &pConsumer->rebalanceTime);
// current topics // current topics
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *)); pConsumer->currentTopics = taosArrayInit(sz, sizeof(void *));
@ -141,6 +173,15 @@ void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) {
taosArrayPush(pConsumer->rebRemovedTopics, &topic); taosArrayPush(pConsumer->rebRemovedTopics, &topic);
} }
// reb removed topics
buf = taosDecodeFixedI32(buf, &sz);
pConsumer->assignedTopics = taosArrayInit(sz, sizeof(void *));
for (int32_t i = 0; i < sz; i++) {
char *topic;
buf = taosDecodeString(buf, &topic);
taosArrayPush(pConsumer->assignedTopics, &topic);
}
return (void *)buf; return (void *)buf;
} }
@ -170,42 +211,47 @@ void *tDecodeSMqVgEp(const void *buf, SMqVgEp *pVgEp) {
return (void *)buf; return (void *)buf;
} }
SMqConsumerEpInSub *tCloneSMqConsumerEpInSub(const SMqConsumerEpInSub *pEpInSub) { SMqConsumerEp *tCloneSMqConsumerEp(const SMqConsumerEp *pConsumerEpOld) {
SMqConsumerEpInSub *pEpInSubNew = taosMemoryMalloc(sizeof(SMqConsumerEpInSub)); SMqConsumerEp *pConsumerEpNew = taosMemoryMalloc(sizeof(SMqConsumerEp));
if (pEpInSubNew == NULL) return NULL; if (pConsumerEpNew == NULL) return NULL;
pEpInSubNew->consumerId = pEpInSub->consumerId; pConsumerEpNew->consumerId = pConsumerEpOld->consumerId;
pEpInSubNew->vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp); pConsumerEpNew->vgs = taosArrayDeepCopy(pConsumerEpOld->vgs, (FCopy)tCloneSMqVgEp);
return pEpInSubNew; return pConsumerEpNew;
} }
void tDeleteSMqConsumerEpInSub(SMqConsumerEpInSub *pEpInSub) { void tDeleteSMqConsumerEp(SMqConsumerEp *pConsumerEp) {
taosArrayDestroyEx(pEpInSub->vgs, (FDelete)tDeleteSMqVgEp); //
taosArrayDestroyP(pConsumerEp->vgs, (FDelete)tDeleteSMqVgEp);
} }
int32_t tEncodeSMqConsumerEpInSub(void **buf, const SMqConsumerEpInSub *pEpInSub) { int32_t tEncodeSMqConsumerEp(void **buf, const SMqConsumerEp *pConsumerEp) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pEpInSub->consumerId); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId);
int32_t sz = taosArrayGetSize(pEpInSub->vgs); tlen += taosEncodeArray(buf, pConsumerEp->vgs, (FEncode)tEncodeSMqVgEp);
#if 0
int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
tlen += taosEncodeFixedI32(buf, sz); tlen += taosEncodeFixedI32(buf, sz);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, i); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
tlen += tEncodeSMqVgEp(buf, pVgEp); tlen += tEncodeSMqVgEp(buf, pVgEp);
} }
/*tlen += taosEncodeArray(buf, pEpInSub->vgs, (FEncode)tEncodeSMqVgEp);*/ #endif
return tlen; return tlen;
} }
void *tDecodeSMqConsumerEpInSub(const void *buf, SMqConsumerEpInSub *pEpInSub) { void *tDecodeSMqConsumerEp(const void *buf, SMqConsumerEp *pConsumerEp) {
buf = taosDecodeFixedI64(buf, &pEpInSub->consumerId); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId);
/*buf = taosDecodeArray(buf, &pEpInSub->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));*/ buf = taosDecodeArray(buf, &pConsumerEp->vgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqSubVgEp));
#if 0
int32_t sz; int32_t sz;
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);
pEpInSub->vgs = taosArrayInit(sz, sizeof(void *)); pConsumerEp->vgs = taosArrayInit(sz, sizeof(void *));
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
buf = tDecodeSMqVgEp(buf, pVgEp); buf = tDecodeSMqVgEp(buf, pVgEp);
taosArrayPush(pEpInSub->vgs, &pVgEp); taosArrayPush(pConsumerEp->vgs, &pVgEp);
} }
#endif
return (void *)buf; return (void *)buf;
} }
@ -215,15 +261,13 @@ SMqSubscribeObj *tNewSubscribeObj(const char key[TSDB_SUBSCRIBE_KEY_LEN]) {
if (pSubNew == NULL) return NULL; if (pSubNew == NULL) return NULL;
memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN); memcpy(pSubNew->key, key, TSDB_SUBSCRIBE_KEY_LEN);
taosInitRWLatch(&pSubNew->lock); taosInitRWLatch(&pSubNew->lock);
pSubNew->vgNum = -1; pSubNew->vgNum = 0;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
// TODO set free fp // TODO set hash free fp
SMqConsumerEpInSub epInSub = { /*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
.consumerId = -1,
.vgs = taosArrayInit(0, sizeof(void *)), pSubNew->unassignedVgs = taosArrayInit(0, sizeof(void *));
};
int64_t unexistKey = -1;
taosHashPut(pSubNew->consumerHash, &unexistKey, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
return pSubNew; return pSubNew;
} }
@ -240,25 +284,27 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->vgNum = pSub->vgNum; pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
/*taosHashSetFreeFp(pSubNew->consumerHash, taosArrayDestroy);*/ // TODO set hash free fp
void *pIter = NULL; /*taosHashSetFreeFp(pSubNew->consumerHash, tDeleteSMqConsumerEp);*/
SMqConsumerEpInSub *pEpInSub = NULL; void *pIter = NULL;
SMqConsumerEp *pConsumerEp = NULL;
while (1) { while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter); pIter = taosHashIterate(pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
pEpInSub = (SMqConsumerEpInSub *)pIter; pConsumerEp = (SMqConsumerEp *)pIter;
SMqConsumerEpInSub newEp = { SMqConsumerEp newEp = {
.consumerId = pEpInSub->consumerId, .consumerId = pConsumerEp->consumerId,
.vgs = taosArrayDeepCopy(pEpInSub->vgs, (FCopy)tCloneSMqVgEp), .vgs = taosArrayDeepCopy(pConsumerEp->vgs, (FCopy)tCloneSMqVgEp),
}; };
taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEpInSub)); taosHashPut(pSubNew->consumerHash, &newEp.consumerId, sizeof(int64_t), &newEp, sizeof(SMqConsumerEp));
} }
pSubNew->unassignedVgs = taosArrayDeepCopy(pSub->unassignedVgs, (FCopy)tCloneSMqVgEp);
return pSubNew; return pSubNew;
} }
void tDeleteSubscribeObj(SMqSubscribeObj *pSub) { void tDeleteSubscribeObj(SMqSubscribeObj *pSub) {
/*taosArrayDestroyEx(pSub->consumerEps, (FDelete)tDeleteSMqConsumerEpInSub);*/
taosHashCleanup(pSub->consumerHash); taosHashCleanup(pSub->consumerHash);
taosArrayDestroyP(pSub->unassignedVgs, (FDelete)tDeleteSMqVgEp);
} }
int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
@ -278,12 +324,12 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
while (1) { while (1) {
pIter = taosHashIterate(pSub->consumerHash, pIter); pIter = taosHashIterate(pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
tlen += tEncodeSMqConsumerEpInSub(buf, pEpInSub); tlen += tEncodeSMqConsumerEp(buf, pConsumerEp);
cnt++; cnt++;
} }
ASSERT(cnt == sz); ASSERT(cnt == sz);
/*tlen += taosEncodeArray(buf, pSub->consumerEps, (FEncode)tEncodeSMqConsumerEpInSub);*/ tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp);
return tlen; return tlen;
} }
@ -301,13 +347,12 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); pSub->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
/*SMqConsumerEpInSub* pEpInSub = taosMemoryMalloc(sizeof(SMqConsumerEpInSub));*/ SMqConsumerEp consumerEp = {0};
SMqConsumerEpInSub epInSub = {0}; buf = tDecodeSMqConsumerEp(buf, &consumerEp);
buf = tDecodeSMqConsumerEpInSub(buf, &epInSub); taosHashPut(pSub->consumerHash, &consumerEp.consumerId, sizeof(int64_t), &consumerEp, sizeof(SMqConsumerEp));
taosHashPut(pSub->consumerHash, &epInSub.consumerId, sizeof(int64_t), &epInSub, sizeof(SMqConsumerEpInSub));
} }
/*buf = taosDecodeArray(buf, &pSub->consumerEps, (FDecode)tDecodeSMqConsumerEpInSub, sizeof(SMqConsumerEpInSub));*/ buf = taosDecodeArray(buf, &pSub->unassignedVgs, (FDecode)tDecodeSMqVgEp, sizeof(SMqVgEp));
return (void *)buf; return (void *)buf;
} }
@ -315,12 +360,12 @@ SMqSubActionLogEntry *tCloneSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry)); SMqSubActionLogEntry *pEntryNew = taosMemoryMalloc(sizeof(SMqSubActionLogEntry));
if (pEntryNew == NULL) return NULL; if (pEntryNew == NULL) return NULL;
pEntryNew->epoch = pEntry->epoch; pEntryNew->epoch = pEntry->epoch;
pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEpInSub); pEntryNew->consumers = taosArrayDeepCopy(pEntry->consumers, (FCopy)tCloneSMqConsumerEp);
return pEntryNew; return pEntryNew;
} }
void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) { void tDeleteSMqSubActionLogEntry(SMqSubActionLogEntry *pEntry) {
taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEpInSub); taosArrayDestroyEx(pEntry->consumers, (FDelete)tDeleteSMqConsumerEp);
} }
int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) { int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEntry) {
@ -329,6 +374,7 @@ int32_t tEncodeSMqSubActionLogEntry(void **buf, const SMqSubActionLogEntry *pEnt
tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry); tlen += taosEncodeArray(buf, pEntry->consumers, (FEncode)tEncodeSMqSubActionLogEntry);
return tlen; return tlen;
} }
void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) { void *tDecodeSMqSubActionLogEntry(const void *buf, SMqSubActionLogEntry *pEntry) {
buf = taosDecodeFixedI32(buf, &pEntry->epoch); buf = taosDecodeFixedI32(buf, &pEntry->epoch);
buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry)); buf = taosDecodeArray(buf, &pEntry->consumers, (FDecode)tDecodeSMqSubActionLogEntry, sizeof(SMqSubActionLogEntry));
@ -339,12 +385,12 @@ SMqSubActionLogObj *tCloneSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj)); SMqSubActionLogObj *pLogNew = taosMemoryMalloc(sizeof(SMqSubActionLogObj));
if (pLogNew == NULL) return pLogNew; if (pLogNew == NULL) return pLogNew;
memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN); memcpy(pLogNew->key, pLog->key, TSDB_SUBSCRIBE_KEY_LEN);
pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEpInSub); pLogNew->logs = taosArrayDeepCopy(pLog->logs, (FCopy)tCloneSMqConsumerEp);
return pLogNew; return pLogNew;
} }
void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) { void tDeleteSMqSubActionLogObj(SMqSubActionLogObj *pLog) {
taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEpInSub); taosArrayDestroyEx(pLog->logs, (FDelete)tDeleteSMqConsumerEp);
} }
int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) { int32_t tEncodeSMqSubActionLogObj(void **buf, const SMqSubActionLogObj *pLog) {

2
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -363,7 +363,7 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) {
pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH; pDnode->offlineReason = DND_REASON_VERSION_NOT_MATCH;
} }
mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion); mError("dnode:%d, status msg version:%d not match cluster:%d", statusReq.dnodeId, statusReq.sver, tsVersion);
terrno = TSDB_CODE_MND_INVALID_MSG_VERSION; terrno = TSDB_CODE_VERSION_NOT_COMPATIBLE;
goto PROCESS_STATUS_MSG_OVER; goto PROCESS_STATUS_MSG_OVER;
} }

21
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -142,6 +142,7 @@ static const SInfosTableSchema userTblsSchema[] = {
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "ttl", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "table_comment", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "type", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
}; };
static const SInfosTableSchema userTblDistSchema[] = { static const SInfosTableSchema userTblDistSchema[] = {
@ -164,7 +165,6 @@ static const SInfosTableSchema userUsersSchema[] = {
{.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "account", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
}; };
static const SInfosTableSchema grantsSchema[] = { static const SInfosTableSchema grantsSchema[] = {
@ -199,23 +199,6 @@ static const SInfosTableSchema vgroupsSchema[] = {
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
}; };
static const SInfosTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
// ep
// up time
// topics
};
static const SInfosTableSchema subscribeSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
};
static const SInfosTableSchema smaSchema[] = { static const SInfosTableSchema smaSchema[] = {
{.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "sma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
@ -282,8 +265,6 @@ static const SInfosTableMeta infosMeta[] = {
{TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
{TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
{TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
{TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)},
{TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)}, {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)},
{TSDB_INS_TABLE_SMAS, smaSchema, tListLen(smaSchema)}, {TSDB_INS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
{TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)}, {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)},

35
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -41,29 +41,39 @@ static const SPerfsTableSchema queriesSchema[] = {
static const SPerfsTableSchema topicSchema[] = { static const SPerfsTableSchema topicSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
/*{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},*/ {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
/*{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},*/ // TODO config
}; };
static const SPerfsTableSchema consumerSchema[] = { static const SPerfsTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "topics", .bytes = TSDB_SHOW_LIST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "end_point", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
// ep {.name = "up_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// up time {.name = "subscribe_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
// topics {.name = "rebalance_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
}; };
static const SPerfsTableSchema subscribeSchema[] = { static const SPerfsTableSchema subscriptionSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT}, {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, };
static const SPerfsTableSchema offsetSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "committed_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "current_offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "skip_log_cnt", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
}; };
static const SPerfsTableMeta perfsMeta[] = { static const SPerfsTableMeta perfsMeta[] = {
@ -71,7 +81,8 @@ static const SPerfsTableMeta perfsMeta[] = {
{TSDB_PERFS_TABLE_QUERIES, queriesSchema, tListLen(queriesSchema)}, {TSDB_PERFS_TABLE_QUERIES, queriesSchema, tListLen(queriesSchema)},
{TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)}, {TSDB_PERFS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
{TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, {TSDB_PERFS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
{TSDB_PERFS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)}, {TSDB_PERFS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema)},
{TSDB_PERFS_TABLE_OFFSETS, offsetSchema, tListLen(offsetSchema)},
}; };
// connection/application/ // connection/application/

2
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -196,7 +196,7 @@ static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
goto CONN_OVER; goto CONN_OVER;
} }
if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) {
mError("user:%s, failed to auth while acquire user\n %s \r\n %s", pReq->user, connReq.passwd, pUser->pass); mError("user:%s, failed to auth while acquire user, input:%s saved:%s", pReq->user, connReq.passwd, pUser->pass);
code = TSDB_CODE_RPC_AUTH_FAILURE; code = TSDB_CODE_RPC_AUTH_FAILURE;
goto CONN_OVER; goto CONN_OVER;
} }

30
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -478,6 +478,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
SVgObj* pVgroup = NULL; SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = NULL; SQueryPlan* pPlan = NULL;
SSubplan* plan = NULL; SSubplan* plan = NULL;
if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan); pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) { if (pPlan == NULL) {
@ -485,14 +486,10 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
return -1; return -1;
} }
ASSERT(pSub->vgNum == -1);
pSub->vgNum = 0;
int32_t levelNum = LIST_LENGTH(pPlan->pSubplans); int32_t levelNum = LIST_LENGTH(pPlan->pSubplans);
if (levelNum != 1) { if (levelNum != 1) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1; return -1;
} }
@ -501,17 +498,14 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t opNum = LIST_LENGTH(inner->pNodeList); int32_t opNum = LIST_LENGTH(inner->pNodeList);
if (opNum != 1) { if (opNum != 1) {
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; terrno = TSDB_CODE_MND_INVALID_TOPIC_QUERY;
return -1; return -1;
} }
plan = nodesListGetNode(inner->pNodeList, 0); plan = nodesListGetNode(inner->pNodeList, 0);
} }
int64_t unexistKey = -1; ASSERT(pSub->unassignedVgs);
SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
ASSERT(pEpInSub);
ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
void* pIter = NULL; void* pIter = NULL;
while (1) { while (1) {
@ -527,9 +521,9 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
pVgEp->vgId = pVgroup->vgId; pVgEp->vgId = pVgroup->vgId;
taosArrayPush(pEpInSub->vgs, &pVgEp); taosArrayPush(pSub->unassignedVgs, &pVgEp);
mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId); mDebug("init subscription %s, assign vg: %d", pSub->key, pVgEp->vgId);
if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
int32_t msgLen; int32_t msgLen;
@ -546,17 +540,11 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
} else { } else {
pVgEp->qmsg = strdup(""); pVgEp->qmsg = strdup("");
} }
ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
/*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
} }
pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); ASSERT(pSub->unassignedVgs->size > 0);
ASSERT(pEpInSub->vgs->size > 0); ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);

159
source/dnode/mnode/impl/src/mndStb.c
View File

@ -72,8 +72,8 @@ void mndCleanupStb(SMnode *pMnode) {}
SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags + pStb->numOfSmas) * sizeof(SSchema) + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen +
+ pStb->commentLen + pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE; pStb->ast1Len + pStb->ast2Len + TSDB_STB_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) goto _OVER;
@ -87,12 +87,10 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER)
SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER) SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfSmas, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER)
@ -113,14 +111,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
} }
for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
SSchema *pSchema = &pStb->pSmas[i];
SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER)
SDB_SET_INT16(pRaw, dataPos, pSchema->colId, _OVER)
SDB_SET_INT32(pRaw, dataPos, pSchema->bytes, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
}
if (pStb->commentLen > 0) { if (pStb->commentLen > 0) {
SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER) SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen, _OVER)
} }
@ -175,20 +165,17 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
int32_t xFilesFactor = 0; int32_t xFilesFactor = 0;
SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER) SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER)
pStb->xFilesFactor = xFilesFactor / 10000.0f; pStb->xFilesFactor = xFilesFactor / 10000.0f;
SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->delay, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->delay, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfSmas, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER)
pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema));
pStb->pSmas = taosMemoryCalloc(pStb->numOfSmas, sizeof(SSchema)); if (pStb->pColumns == NULL || pStb->pTags == NULL) {
if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pSmas == NULL) {
goto _OVER; goto _OVER;
} }
@ -208,14 +195,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER) SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
} }
for (int32_t i = 0; i < pStb->numOfSmas; ++i) {
SSchema *pSchema = &pStb->pSmas[i];
SDB_GET_INT8(pRaw, dataPos, &pSchema->type, _OVER)
SDB_GET_INT16(pRaw, dataPos, &pSchema->colId, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pSchema->bytes, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
}
if (pStb->commentLen > 0) { if (pStb->commentLen > 0) {
pStb->comment = taosMemoryCalloc(pStb->commentLen, 1); pStb->comment = taosMemoryCalloc(pStb->commentLen, 1);
if (pStb->comment == NULL) goto _OVER; if (pStb->comment == NULL) goto _OVER;
@ -293,18 +272,6 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
} }
} }
if (pOld->numOfSmas < pNew->numOfSmas) {
void *pSmas = taosMemoryMalloc(pNew->numOfSmas * sizeof(SSchema));
if (pSmas != NULL) {
taosMemoryFree(pOld->pSmas);
pOld->pSmas = pSmas;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mTrace("stb:%s, failed to perform update action since %s", pOld->name, terrstr());
taosWUnLockLatch(&pOld->lock);
}
}
if (pOld->commentLen < pNew->commentLen) { if (pOld->commentLen < pNew->commentLen) {
void *comment = taosMemoryMalloc(pNew->commentLen); void *comment = taosMemoryMalloc(pNew->commentLen);
if (comment != NULL) { if (comment != NULL) {
@ -395,86 +362,51 @@ static FORCE_INLINE int schemaExColIdCompare(const void *colId, const void *pSch
} }
static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) { static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen) {
SName name = {0}; SCoder coder = {0};
int32_t contLen;
SName name = {0};
SVCreateStbReq req = {0};
tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); tNameFromString(&name, pStb->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
char dbFName[TSDB_DB_FNAME_LEN] = {0}; char dbFName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFName); tNameGetFullDbName(&name, dbFName);
SVCreateTbReq req = {0};
req.name = (char *)tNameGetTableName(&name); req.name = (char *)tNameGetTableName(&name);
req.ttl = 0; req.suid = pStb->uid;
req.keep = 0; req.rollup = pStb->ast1Len > 0 ? 1 : 0;
req.rollup = pStb->aggregationMethod > -1 ? 1 : 0; req.schema.nCols = pStb->numOfColumns;
req.type = TD_SUPER_TABLE; req.schema.sver = 0;
req.stbCfg.suid = pStb->uid; req.schema.pSchema = pStb->pColumns;
req.stbCfg.nCols = pStb->numOfColumns; req.schemaTag.nCols = pStb->numOfTags;
req.stbCfg.nTagCols = pStb->numOfTags; req.schemaTag.pSchema = pStb->pTags;
req.stbCfg.pTagSchema = pStb->pTags;
req.stbCfg.nBSmaCols = pStb->numOfSmas;
req.stbCfg.pSchema = (SSchema *)taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
if (req.stbCfg.pSchema == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
memcpy(req.stbCfg.pSchema, pStb->pColumns, sizeof(SSchema) * pStb->numOfColumns);
for (int i = 0; i < pStb->numOfColumns; i++) {
req.stbCfg.pSchema[i].flags = SCHEMA_SMA_ON;
}
SRSmaParam *pRSmaParam = NULL;
if (req.rollup) { if (req.rollup) {
pRSmaParam = (SRSmaParam *)taosMemoryCalloc(1, sizeof(SRSmaParam)); req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
if (pRSmaParam == NULL) { req.pRSmaParam.delay = pStb->delay;
taosMemoryFreeClear(req.stbCfg.pSchema);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pRSmaParam->xFilesFactor = pStb->xFilesFactor;
pRSmaParam->delay = pStb->delay;
pRSmaParam->nFuncIds = 1; // only 1 aggregation method supported currently
pRSmaParam->pFuncIds = (func_id_t *)taosMemoryCalloc(pRSmaParam->nFuncIds, sizeof(func_id_t));
if (pRSmaParam->pFuncIds == NULL) {
taosMemoryFreeClear(req.stbCfg.pRSmaParam);
taosMemoryFreeClear(req.stbCfg.pSchema);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
for (int32_t f = 0; f < pRSmaParam->nFuncIds; ++f) {
*(pRSmaParam->pFuncIds + f) = pStb->aggregationMethod;
}
if (pStb->ast1Len > 0) { if (pStb->ast1Len > 0) {
if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &pRSmaParam->qmsg1, &pRSmaParam->qmsg1Len) != TSDB_CODE_SUCCESS) { if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len) != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pRSmaParam->pFuncIds);
taosMemoryFreeClear(req.stbCfg.pRSmaParam);
taosMemoryFreeClear(req.stbCfg.pSchema);
return NULL; return NULL;
} }
} }
if (pStb->ast2Len > 0) { if (pStb->ast2Len > 0) {
if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &pRSmaParam->qmsg2, &pRSmaParam->qmsg2Len) != TSDB_CODE_SUCCESS) { if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len) != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pRSmaParam->pFuncIds);
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(req.stbCfg.pRSmaParam);
taosMemoryFreeClear(req.stbCfg.pSchema);
return NULL; return NULL;
} }
} }
}
req.stbCfg.pRSmaParam = pRSmaParam; // get length
int32_t ret = 0;
tEncodeSize(tEncodeSVCreateStbReq, &req, contLen, ret);
if (ret < 0) {
return NULL;
} }
int32_t contLen = tSerializeSVCreateTbReq(NULL, &req) + sizeof(SMsgHead); contLen += sizeof(SMsgHead);
SMsgHead *pHead = taosMemoryMalloc(contLen); SMsgHead *pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) { if (pHead == NULL) {
if (pRSmaParam) { taosMemoryFreeClear(req.pRSmaParam.qmsg1);
taosMemoryFreeClear(pRSmaParam->pFuncIds); taosMemoryFreeClear(req.pRSmaParam.qmsg2);
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(pRSmaParam->qmsg2);
taosMemoryFreeClear(pRSmaParam);
}
taosMemoryFreeClear(req.stbCfg.pSchema);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -483,16 +415,15 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
pHead->vgId = htonl(pVgroup->vgId); pHead->vgId = htonl(pVgroup->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tSerializeSVCreateTbReq(&pBuf, &req); tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER);
if (tEncodeSVCreateStbReq(&coder, &req) < 0) {
return NULL;
}
tCoderClear(&coder);
*pContLen = contLen; *pContLen = contLen;
if (pRSmaParam) { taosMemoryFreeClear(req.pRSmaParam.qmsg1);
taosMemoryFreeClear(pRSmaParam->pFuncIds); taosMemoryFreeClear(req.pRSmaParam.qmsg2);
taosMemoryFreeClear(pRSmaParam->qmsg1);
taosMemoryFreeClear(pRSmaParam->qmsg2);
taosMemoryFreeClear(pRSmaParam);
}
taosMemoryFreeClear(req.stbCfg.pSchema);
return pHead; return pHead;
} }
@ -705,12 +636,10 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
stbObj.version = 1; stbObj.version = 1;
stbObj.nextColId = 1; stbObj.nextColId = 1;
stbObj.xFilesFactor = pCreate->xFilesFactor; stbObj.xFilesFactor = pCreate->xFilesFactor;
stbObj.aggregationMethod = pCreate->aggregationMethod;
stbObj.delay = pCreate->delay; stbObj.delay = pCreate->delay;
stbObj.ttl = pCreate->ttl; stbObj.ttl = pCreate->ttl;
stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfColumns = pCreate->numOfColumns;
stbObj.numOfTags = pCreate->numOfTags; stbObj.numOfTags = pCreate->numOfTags;
stbObj.numOfSmas = pCreate->numOfSmas;
stbObj.commentLen = pCreate->commentLen; stbObj.commentLen = pCreate->commentLen;
if (stbObj.commentLen > 0) { if (stbObj.commentLen > 0) {
stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1); stbObj.comment = taosMemoryCalloc(stbObj.commentLen, 1);
@ -743,8 +672,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema)); stbObj.pColumns = taosMemoryMalloc(stbObj.numOfColumns * sizeof(SSchema));
stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema)); stbObj.pTags = taosMemoryMalloc(stbObj.numOfTags * sizeof(SSchema));
stbObj.pSmas = taosMemoryMalloc(stbObj.numOfSmas * sizeof(SSchema)); if (stbObj.pColumns == NULL || stbObj.pTags == NULL) {
if (stbObj.pColumns == NULL || stbObj.pTags == NULL || stbObj.pSmas == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -754,6 +682,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
SSchema *pSchema = &stbObj.pColumns[i]; SSchema *pSchema = &stbObj.pColumns[i];
pSchema->type = pField->type; pSchema->type = pField->type;
pSchema->bytes = pField->bytes; pSchema->bytes = pField->bytes;
pSchema->flags = pField->flags;
memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN);
pSchema->colId = stbObj.nextColId; pSchema->colId = stbObj.nextColId;
stbObj.nextColId++; stbObj.nextColId++;
@ -769,18 +698,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
stbObj.nextColId++; stbObj.nextColId++;
} }
for (int32_t i = 0; i < stbObj.numOfSmas; ++i) {
SField *pField = taosArrayGet(pCreate->pSmas, i);
SSchema *pSchema = &stbObj.pSmas[i];
SSchema *pColSchema = mndFindStbColumns(&stbObj, pField->name);
if (pColSchema == NULL) {
mError("stb:%s, sma:%s not found in columns", stbObj.name, pField->name);
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
memcpy(pSchema, pColSchema, sizeof(SSchema));
}
int32_t code = -1; int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_CREATE_STB, &pReq->rpcMsg);
if (pTrans == NULL) goto _OVER; if (pTrans == NULL) goto _OVER;

93
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -85,7 +85,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
pSub->withSchema = pTopic->withSchema; pSub->withSchema = pTopic->withSchema;
pSub->withTag = pTopic->withTag; pSub->withTag = pTopic->withTag;
ASSERT(taosHashGetSize(pSub->consumerHash) == 1); ASSERT(pSub->unassignedVgs->size == 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) { if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
tDeleteSubscribeObj(pSub); tDeleteSubscribeObj(pSub);
@ -93,7 +94,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
return NULL; return NULL;
} }
ASSERT(taosHashGetSize(pSub->consumerHash) == 1); ASSERT(pSub->unassignedVgs->size > 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
return pSub; return pSub;
} }
@ -185,7 +187,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
if (pInput->pTopic != NULL) { if (pInput->pTopic != NULL) {
// create subscribe // create subscribe
pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key); pOutput->pSub = mndCreateSub(pMnode, pInput->pTopic, pInput->pRebInfo->key);
ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 1); ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) == 0);
} else { } else {
pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub); pOutput->pSub = tCloneSubscribeObj(pInput->pOldSub);
} }
@ -196,21 +198,20 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
// 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg // 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
// 2. check and get actual removed consumers, put their vg into hash // 2. check and get actual removed consumers, put their vg into hash
int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t removedNum = taosArrayGetSize(pInput->pRebInfo->removedConsumers);
int32_t actualRemoved = 0; int32_t actualRemoved = 0;
for (int32_t i = 0; i < removedNum; i++) { for (int32_t i = 0; i < removedNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
ASSERT(consumerId > 0); ASSERT(consumerId > 0);
SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
ASSERT(pEpInSub); ASSERT(pConsumerEp);
if (pEpInSub) { if (pConsumerEp) {
ASSERT(consumerId == pEpInSub->consumerId); ASSERT(consumerId == pConsumerEp->consumerId);
actualRemoved++; actualRemoved++;
int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs); int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
for (int32_t j = 0; j < consumerVgNum; j++) { for (int32_t j = 0; j < consumerVgNum; j++) {
SMqVgEp *pVgEp = taosArrayGetP(pEpInSub->vgs, j); SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j);
SMqRebOutputVg outputVg = { SMqRebOutputVg outputVg = {
.oldConsumerId = consumerId, .oldConsumerId = consumerId,
.newConsumerId = -1, .newConsumerId = -1,
@ -224,16 +225,12 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
} }
} }
ASSERT(removedNum == actualRemoved); ASSERT(removedNum == actualRemoved);
ASSERT(taosHashGetSize(pOutput->pSub->consumerHash) > 0);
// if previously no consumer, there are vgs not assigned // if previously no consumer, there are vgs not assigned
{ {
int64_t unexistKey = -1; int32_t consumerVgNum = taosArrayGetSize(pOutput->pSub->unassignedVgs);
SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
ASSERT(pEpInSub);
int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
for (int32_t i = 0; i < consumerVgNum; i++) { for (int32_t i = 0; i < consumerVgNum; i++) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pOutput->pSub->unassignedVgs);
SMqRebOutputVg rebOutput = { SMqRebOutputVg rebOutput = {
.oldConsumerId = -1, .oldConsumerId = -1,
.newConsumerId = -1, .newConsumerId = -1,
@ -246,7 +243,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
// 3. calc vg number of each consumer // 3. calc vg number of each consumer
int32_t oldSz = 0; int32_t oldSz = 0;
if (pInput->pOldSub) { if (pInput->pOldSub) {
oldSz = taosHashGetSize(pInput->pOldSub->consumerHash) - 1; oldSz = taosHashGetSize(pInput->pOldSub->consumerHash);
} }
int32_t afterRebConsumerNum = int32_t afterRebConsumerNum =
oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers); oldSz + taosArrayGetSize(pInput->pRebInfo->newConsumers) - taosArrayGetSize(pInput->pRebInfo->removedConsumers);
@ -264,23 +261,22 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
while (1) { while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
if (pEpInSub->consumerId == -1) continue; ASSERT(pConsumerEp->consumerId > 0);
ASSERT(pEpInSub->consumerId > 0); int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
int32_t consumerVgNum = taosArrayGetSize(pEpInSub->vgs);
// all old consumers still existing are touched // all old consumers still existing are touched
// TODO optimize: touch only consumer whose vgs changed // TODO optimize: touch only consumer whose vgs changed
taosArrayPush(pOutput->touchedConsumers, &pEpInSub->consumerId); taosArrayPush(pOutput->touchedConsumers, &pConsumerEp->consumerId);
if (consumerVgNum > minVgCnt) { if (consumerVgNum > minVgCnt) {
if (imbCnt < imbConsumerNum) { if (imbCnt < imbConsumerNum) {
if (consumerVgNum == minVgCnt + 1) { if (consumerVgNum == minVgCnt + 1) {
continue; continue;
} else { } else {
// pop until equal minVg + 1 // pop until equal minVg + 1
while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt + 1) { while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
SMqRebOutputVg outputVg = { SMqRebOutputVg outputVg = {
.oldConsumerId = pEpInSub->consumerId, .oldConsumerId = pConsumerEp->consumerId,
.newConsumerId = -1, .newConsumerId = -1,
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
@ -290,10 +286,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
} }
} else { } else {
// pop until equal minVg // pop until equal minVg
while (taosArrayGetSize(pEpInSub->vgs) > minVgCnt) { while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt) {
SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pEpInSub->vgs); SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(pConsumerEp->vgs);
SMqRebOutputVg outputVg = { SMqRebOutputVg outputVg = {
.oldConsumerId = pEpInSub->consumerId, .oldConsumerId = pConsumerEp->consumerId,
.newConsumerId = -1, .newConsumerId = -1,
.pVgEp = pVgEp, .pVgEp = pVgEp,
}; };
@ -309,12 +305,11 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
for (int32_t i = 0; i < consumerNum; i++) { for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
ASSERT(consumerId > 0); ASSERT(consumerId > 0);
SMqConsumerEpInSub newConsumerEp; SMqConsumerEp newConsumerEp;
newConsumerEp.consumerId = consumerId; newConsumerEp.consumerId = consumerId;
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *)); newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
sizeof(SMqConsumerEpInSub)); /*SMqConsumer* pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
/*SMqConsumerEpInSub *pTestNew = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));*/
/*ASSERT(pTestNew->consumerId == consumerId);*/ /*ASSERT(pTestNew->consumerId == consumerId);*/
/*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/ /*ASSERT(pTestNew->vgs == newConsumerEp.vgs);*/
taosArrayPush(pOutput->newConsumers, &consumerId); taosArrayPush(pOutput->newConsumers, &consumerId);
@ -329,25 +324,24 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
while (1) { while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) break; if (pIter == NULL) break;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
if (pEpInSub->consumerId == -1) continue; ASSERT(pConsumerEp->consumerId > 0);
ASSERT(pEpInSub->consumerId > 0);
// push until equal minVg // push until equal minVg
while (taosArrayGetSize(pEpInSub->vgs) < minVgCnt) { while (taosArrayGetSize(pConsumerEp->vgs) < minVgCnt) {
// iter hash and find one vg // iter hash and find one vg
pRemovedIter = taosHashIterate(pHash, pRemovedIter); pRemovedIter = taosHashIterate(pHash, pRemovedIter);
ASSERT(pRemovedIter); ASSERT(pRemovedIter);
pRebVg = (SMqRebOutputVg *)pRemovedIter; pRebVg = (SMqRebOutputVg *)pRemovedIter;
// push // push
taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
pRebVg->newConsumerId = pEpInSub->consumerId; pRebVg->newConsumerId = pConsumerEp->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
} }
} }
// 7. handle unassigned vg // 7. handle unassigned vg
if (taosHashGetSize(pOutput->pSub->consumerHash) != 1) { if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) {
// if has consumer, assign all left vg // if has consumer, assign all left vg
while (1) { while (1) {
pRemovedIter = taosHashIterate(pHash, pRemovedIter); pRemovedIter = taosHashIterate(pHash, pRemovedIter);
@ -355,20 +349,14 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter); pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
ASSERT(pIter); ASSERT(pIter);
pRebVg = (SMqRebOutputVg *)pRemovedIter; pRebVg = (SMqRebOutputVg *)pRemovedIter;
SMqConsumerEpInSub *pEpInSub = (SMqConsumerEpInSub *)pIter; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
if (pEpInSub->consumerId == -1) continue; ASSERT(pConsumerEp->consumerId > 0);
ASSERT(pEpInSub->consumerId > 0); taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
taosArrayPush(pEpInSub->vgs, &pRebVg->pVgEp); pRebVg->newConsumerId = pConsumerEp->consumerId;
pRebVg->newConsumerId = pEpInSub->consumerId;
taosArrayPush(pOutput->rebVgs, pRebVg); taosArrayPush(pOutput->rebVgs, pRebVg);
} }
} else { } else {
// if all consumer is removed, put all vg into unassigned // if all consumer is removed, put all vg into unassigned
int64_t unexistKey = -1;
SMqConsumerEpInSub *pEpInSub = taosHashGet(pOutput->pSub->consumerHash, &unexistKey, sizeof(int64_t));
ASSERT(pEpInSub);
ASSERT(pEpInSub->consumerId == -1);
pIter = NULL; pIter = NULL;
SMqRebOutputVg *pRebOutput = NULL; SMqRebOutputVg *pRebOutput = NULL;
while (1) { while (1) {
@ -376,7 +364,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
if (pIter == NULL) break; if (pIter == NULL) break;
pRebOutput = (SMqRebOutputVg *)pIter; pRebOutput = (SMqRebOutputVg *)pIter;
ASSERT(pRebOutput->newConsumerId == -1); ASSERT(pRebOutput->newConsumerId == -1);
taosArrayPush(pEpInSub->vgs, &pRebOutput->pVgEp); taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
taosArrayPush(pOutput->rebVgs, pRebOutput); taosArrayPush(pOutput->rebVgs, pRebOutput);
} }
} }
@ -512,6 +500,7 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
// possibly no vg is changed // possibly no vg is changed
/*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
// TODO replace assert with error check
ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
if (rebInput.pTopic) { if (rebInput.pTopic) {
@ -631,6 +620,10 @@ static int32_t mndSubActionUpdate(SSdb *pSdb, SMqSubscribeObj *pOldSub, SMqSubsc
pOldSub->consumerHash = pNewSub->consumerHash; pOldSub->consumerHash = pNewSub->consumerHash;
pNewSub->consumerHash = tmp; pNewSub->consumerHash = tmp;
SArray *tmp1 = pOldSub->unassignedVgs;
pOldSub->unassignedVgs = pNewSub->unassignedVgs;
pNewSub->unassignedVgs = tmp1;
taosWUnLockLatch(&pOldSub->lock); taosWUnLockLatch(&pOldSub->lock);
return 0; return 0;
} }

54
source/dnode/mnode/impl/src/mndSync.c
View File

@ -22,13 +22,15 @@ static int32_t mndInitWal(SMnode *pMnode) {
char path[PATH_MAX] = {0}; char path[PATH_MAX] = {0};
snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP); snprintf(path, sizeof(path), "%s%swal", pMnode->path, TD_DIRSEP);
SWalCfg cfg = {.vgId = 1, SWalCfg cfg = {
.fsyncPeriod = 0, .vgId = 1,
.rollPeriod = -1, .fsyncPeriod = 0,
.segSize = -1, .rollPeriod = -1,
.retentionPeriod = -1, .segSize = -1,
.retentionSize = -1, .retentionPeriod = -1,
.level = TAOS_WAL_FSYNC}; .retentionSize = -1,
.level = TAOS_WAL_FSYNC,
};
pMgmt->pWal = walOpen(path, &cfg); pMgmt->pWal = walOpen(path, &cfg);
if (pMgmt->pWal == NULL) return -1; if (pMgmt->pWal == NULL) return -1;
@ -54,62 +56,62 @@ static int32_t mndRestoreWal(SMnode *pMnode) {
int64_t first = walGetFirstVer(pWal); int64_t first = walGetFirstVer(pWal);
int64_t last = walGetLastVer(pWal); int64_t last = walGetLastVer(pWal);
mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); mDebug("start to restore wal, sdbver:%" PRId64 ", first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last);
first = TMAX(lastSdbVer + 1, first); first = TMAX(lastSdbVer + 1, first);
for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) {
if (walReadWithHandle(pHandle, ver) < 0) { if (walReadWithHandle(pHandle, ver) < 0) {
mError("failed to read by wal handle since %s, ver:%" PRId64, terrstr(), ver); mError("ver:%" PRId64 ", failed to read from wal since %s", ver, terrstr());
goto WAL_RESTORE_OVER; goto _OVER;
} }
SWalHead *pHead = pHandle->pHead; SWalHead *pHead = pHandle->pHead;
int64_t sdbVer = sdbUpdateVer(pSdb, 0); int64_t sdbVer = sdbUpdateVer(pSdb, 0);
if (sdbVer + 1 != ver) { if (sdbVer + 1 != ver) {
terrno = TSDB_CODE_SDB_INVALID_WAl_VER; terrno = TSDB_CODE_SDB_INVALID_WAl_VER;
mError("failed to read wal from sdb, sdbVer:%" PRId64 " inconsistent with ver:%" PRId64, sdbVer, ver); mError("ver:%" PRId64 ", failed to write to sdb, since inconsistent with sdbver:%" PRId64, ver, sdbVer);
goto WAL_RESTORE_OVER; goto _OVER;
} }
mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body); mTrace("ver:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body);
if (sdbWriteNotFree(pSdb, (void *)pHead->head.body) < 0) { if (sdbWriteWithoutFree(pSdb, (void *)pHead->head.body) < 0) {
mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver); mError("ver:%" PRId64 ", failed to write to sdb since %s", ver, terrstr());
goto WAL_RESTORE_OVER; goto _OVER;
} }
sdbUpdateVer(pSdb, 1); sdbUpdateVer(pSdb, 1);
mDebug("wal:%" PRId64 ", is restored", ver); mDebug("ver:%" PRId64 ", is restored", ver);
} }
int64_t sdbVer = sdbUpdateVer(pSdb, 0); int64_t sdbVer = sdbUpdateVer(pSdb, 0);
mDebug("restore sdb wal finished, sdb ver:%" PRId64, sdbVer); mDebug("restore wal finished, sdbver:%" PRId64, sdbVer);
mndTransPullup(pMnode); mndTransPullup(pMnode);
sdbVer = sdbUpdateVer(pSdb, 0); sdbVer = sdbUpdateVer(pSdb, 0);
mDebug("pullup trans finished, sdb ver:%" PRId64, sdbVer); mDebug("pullup trans finished, sdbver:%" PRId64, sdbVer);
if (sdbVer != lastSdbVer) { if (sdbVer != lastSdbVer) {
mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer); mInfo("sdb restored from %" PRId64 " to %" PRId64 ", write file", lastSdbVer, sdbVer);
if (sdbWriteFile(pSdb) != 0) { if (sdbWriteFile(pSdb) != 0) {
goto WAL_RESTORE_OVER; goto _OVER;
} }
if (walCommit(pWal, sdbVer) != 0) { if (walCommit(pWal, sdbVer) != 0) {
goto WAL_RESTORE_OVER; goto _OVER;
} }
if (walBeginSnapshot(pWal, sdbVer) < 0) { if (walBeginSnapshot(pWal, sdbVer) < 0) {
goto WAL_RESTORE_OVER; goto _OVER;
} }
if (walEndSnapshot(pWal) < 0) { if (walEndSnapshot(pWal) < 0) {
goto WAL_RESTORE_OVER; goto _OVER;
} }
} }
code = 0; code = 0;
WAL_RESTORE_OVER: _OVER:
walCloseReadHandle(pHandle); walCloseReadHandle(pHandle);
return code; return code;
} }
@ -158,11 +160,11 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) {
int64_t ver = sdbUpdateVer(pSdb, 1); int64_t ver = sdbUpdateVer(pSdb, 1);
if (walWrite(pWal, ver, 1, pRaw, sdbGetRawTotalSize(pRaw)) < 0) { if (walWrite(pWal, ver, 1, pRaw, sdbGetRawTotalSize(pRaw)) < 0) {
sdbUpdateVer(pSdb, -1); sdbUpdateVer(pSdb, -1);
mError("failed to write raw:%p since %s, ver:%" PRId64, pRaw, terrstr(), ver); mError("ver:%" PRId64 ", failed to write raw:%p to wal since %s", ver, pRaw, terrstr());
return -1; return -1;
} }
mTrace("raw:%p, write to wal, ver:%" PRId64, pRaw, ver); mTrace("ver:%" PRId64 ", write to wal, raw:%p", ver, pRaw);
walCommit(pWal, ver); walCommit(pWal, ver);
walFsync(pWal, true); walFsync(pWal, true);

1
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -146,7 +146,6 @@ int32_t mndInitTelem(SMnode* pMnode) {
taosGetEmail(pMgmt->email, sizeof(pMgmt->email)); taosGetEmail(pMgmt->email, sizeof(pMgmt->email));
mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer); mndSetMsgHandle(pMnode, TDMT_MND_TELEM_TIMER, mndProcessTelemTimer);
mDebug("mnode telemetry is initialized");
return 0; return 0;
} }

26
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -76,7 +76,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pTopic->subDbUid, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER);
@ -139,7 +138,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->subDbUid, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER);
@ -263,7 +261,7 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) { if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; terrno = TSDB_CODE_MND_INVALID_TOPIC;
return -1; return -1;
} }
@ -520,29 +518,33 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
if (pShow->pIter == NULL) break; if (pShow->pIter == NULL) break;
int32_t cols = 0; SColumnInfoData *pColInfo;
SName n;
int32_t cols = 0;
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pTopic->name, T_NAME_ACCT | T_NAME_DB);
SName n;
tNameFromString(&n, pTopic->name, T_NAME_ACCT|T_NAME_DB);
tNameGetDbName(&n, varDataVal(topicName)); tNameGetDbName(&n, varDataVal(topicName));
varDataSetLen(topicName, strlen(varDataVal(topicName))); varDataSetLen(topicName, strlen(varDataVal(topicName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
char dbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tNameFromString(&n, pTopic->db, T_NAME_ACCT | T_NAME_DB);
tNameGetDbName(&n, varDataVal(dbName));
varDataSetLen(dbName, strlen(varDataVal(dbName)));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)dbName, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN); tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN);
varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)sql, false); colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
// taosMemoryFree(sql);
numOfRows++; numOfRows++;
sdbRelease(pSdb, pTopic); sdbRelease(pSdb, pTopic);
} }

99
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -29,7 +29,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans);
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld);
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc);
static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction);
@ -174,6 +174,13 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER)
} }
SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER)
SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER)
if (pTrans->param != NULL) {
SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER)
}
SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER)
SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER)
@ -193,9 +200,9 @@ TRANS_ENCODE_OVER:
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
SSdbRow * pRow = NULL; SSdbRow *pRow = NULL;
STrans * pTrans = NULL; STrans *pTrans = NULL;
char * pData = NULL; char *pData = NULL;
int32_t dataLen = 0; int32_t dataLen = 0;
int8_t sver = 0; int8_t sver = 0;
int32_t redoLogNum = 0; int32_t redoLogNum = 0;
@ -305,6 +312,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
action.pCont = NULL; action.pCont = NULL;
} }
SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER)
SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER)
if (pTrans->paramLen != 0) {
pTrans->param = taosMemoryMalloc(pTrans->paramLen);
SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER);
}
SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER) SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER)
terrno = 0; terrno = 0;
@ -413,9 +428,36 @@ static const char *mndTransType(ETrnType type) {
} }
} }
static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) {
mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen);
}
static void mndTransTestStopFunc(SMnode *pMnode, void *param, int32_t paramLen) {
mInfo("test trans stop, param:%s, len:%d", (char *)param, paramLen);
}
static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) {
switch (ftype) {
case TEST_TRANS_START_FUNC:
return mndTransTestStartFunc;
case TEST_TRANS_STOP_FUNC:
return mndTransTestStopFunc;
default:
return NULL;
}
}
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
// pTrans->stage = TRN_STAGE_PREPARE; // pTrans->stage = TRN_STAGE_PREPARE;
mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage));
if (pTrans->startFunc > 0) {
TransCbFp fp = mndTransGetCbFp(pTrans->startFunc);
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
}
return 0; return 0;
} }
@ -430,10 +472,23 @@ static void mndTransDropData(STrans *pTrans) {
pTrans->rpcRsp = NULL; pTrans->rpcRsp = NULL;
pTrans->rpcRspLen = 0; pTrans->rpcRspLen = 0;
} }
if (pTrans->param != NULL) {
taosMemoryFree(pTrans->param);
pTrans->param = NULL;
pTrans->paramLen = 0;
}
} }
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) {
mTrace("trans:%d, perform delete action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); mDebug("trans:%d, perform delete action, row:%p stage:%s callfunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage),
callFunc);
if (pTrans->stopFunc > 0 && callFunc) {
TransCbFp fp = mndTransGetCbFp(pTrans->stopFunc);
if (fp) {
(*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen);
}
}
mndTransDropData(pTrans); mndTransDropData(pTrans);
return 0; return 0;
} }
@ -456,7 +511,7 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) {
} }
static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) { static STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
SSdb * pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId); STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId);
if (pTrans == NULL) { if (pTrans == NULL) {
terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
@ -498,7 +553,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S
return NULL; return NULL;
} }
mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans); mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans);
return pTrans; return pTrans;
} }
@ -525,7 +580,7 @@ static void mndTransDropActions(SArray *pArray) {
void mndTransDrop(STrans *pTrans) { void mndTransDrop(STrans *pTrans) {
if (pTrans != NULL) { if (pTrans != NULL) {
mndTransDropData(pTrans); mndTransDropData(pTrans);
mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans);
taosMemoryFreeClear(pTrans); taosMemoryFreeClear(pTrans);
} }
} }
@ -574,6 +629,13 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) {
pTrans->rpcRspLen = contLen; pTrans->rpcRspLen = contLen;
} }
void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen) {
pTrans->startFunc = startFunc;
pTrans->stopFunc = stopFunc;
pTrans->param = param;
pTrans->paramLen = paramLen;
}
void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) { void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) {
pTrans->dbUid = pDb->uid; pTrans->dbUid = pDb->uid;
memcpy(pTrans->dbname, pDb->name, TSDB_DB_FNAME_LEN); memcpy(pTrans->dbname, pDb->name, TSDB_DB_FNAME_LEN);
@ -626,7 +688,7 @@ static int32_t mndCheckTransCanBeStartedInParallel(SMnode *pMnode, STrans *pNewT
if (mndIsBasicTrans(pNewTrans)) return 0; if (mndIsBasicTrans(pNewTrans)) return 0;
STrans *pTrans = NULL; STrans *pTrans = NULL;
void * pIter = NULL; void *pIter = NULL;
int32_t code = 0; int32_t code = 0;
while (1) { while (1) {
@ -830,14 +892,14 @@ HANDLE_ACTION_RSP_OVER:
} }
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
SSdb * pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t arraySize = taosArrayGetSize(pArray); int32_t arraySize = taosArrayGetSize(pArray);
if (arraySize == 0) return 0; if (arraySize == 0) return 0;
for (int32_t i = 0; i < arraySize; ++i) { for (int32_t i = 0; i < arraySize; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i); SSdbRaw *pRaw = taosArrayGetP(pArray, i);
int32_t code = sdbWriteNotFree(pSdb, pRaw); int32_t code = sdbWriteWithoutFree(pSdb, pRaw);
if (code != 0) { if (code != 0) {
return code; return code;
} }
@ -1117,6 +1179,7 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) {
} }
mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes);
return continueExec; return continueExec;
} }
@ -1205,11 +1268,11 @@ static int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) {
} }
static int32_t mndProcessKillTransReq(SNodeMsg *pReq) { static int32_t mndProcessKillTransReq(SNodeMsg *pReq) {
SMnode * pMnode = pReq->pNode; SMnode *pMnode = pReq->pNode;
SKillTransReq killReq = {0}; SKillTransReq killReq = {0};
int32_t code = -1; int32_t code = -1;
SUserObj * pUser = NULL; SUserObj *pUser = NULL;
STrans * pTrans = NULL; STrans *pTrans = NULL;
if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { if (tDeserializeSKillTransReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
@ -1249,7 +1312,7 @@ KILL_OVER:
void mndTransPullup(SMnode *pMnode) { void mndTransPullup(SMnode *pMnode) {
STrans *pTrans = NULL; STrans *pTrans = NULL;
void * pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans); pIter = sdbFetch(pMnode->pSdb, SDB_TRANS, pIter, (void **)&pTrans);
@ -1264,11 +1327,11 @@ void mndTransPullup(SMnode *pMnode) {
static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveTrans(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode; SMnode *pMnode = pReq->pNode;
SSdb * pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
STrans *pTrans = NULL; STrans *pTrans = NULL;
int32_t cols = 0; int32_t cols = 0;
char * pWrite; char *pWrite;
while (numOfRows < rows) { while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_TRANS, pShow->pIter, (void **)&pTrans); pShow->pIter = sdbFetch(pSdb, SDB_TRANS, pShow->pIter, (void **)&pTrans);

195
source/dnode/mnode/impl/src/mndUser.c
View File

@ -21,8 +21,8 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "tbase64.h" #include "tbase64.h"
#define TSDB_USER_VER_NUMBER 1 #define USER_VER_NUMBER 1
#define TSDB_USER_RESERVE_SIZE 64 #define USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode); static int32_t mndCreateDefaultUsers(SMnode *pMnode);
static SSdbRaw *mndUserActionEncode(SUserObj *pUser); static SSdbRaw *mndUserActionEncode(SUserObj *pUser);
@ -35,7 +35,7 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq);
static int32_t mndProcessAlterUserReq(SNodeMsg *pReq); static int32_t mndProcessAlterUserReq(SNodeMsg *pReq);
static int32_t mndProcessDropUserReq(SNodeMsg *pReq); static int32_t mndProcessDropUserReq(SNodeMsg *pReq);
static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq); static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq);
static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter);
int32_t mndInitUser(SMnode *pMnode) { int32_t mndInitUser(SMnode *pMnode) {
@ -93,39 +93,39 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
int32_t size = sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN;
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
if (pRaw == NULL) goto USER_ENCODE_OVER; if (pRaw == NULL) goto _OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, USER_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER)
SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, USER_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->superUser, USER_ENCODE_OVER) SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, USER_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, USER_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER)
char *db = taosHashIterate(pUser->readDbs, NULL); char *db = taosHashIterate(pUser->readDbs, NULL);
while (db != NULL) { while (db != NULL) {
SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
db = taosHashIterate(pUser->readDbs, db); db = taosHashIterate(pUser->readDbs, db);
} }
db = taosHashIterate(pUser->writeDbs, NULL); db = taosHashIterate(pUser->writeDbs, NULL);
while (db != NULL) { while (db != NULL) {
SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER);
db = taosHashIterate(pUser->writeDbs, db); db = taosHashIterate(pUser->writeDbs, db);
} }
SDB_SET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_ENCODE_OVER) SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER)
terrno = 0; terrno = 0;
USER_ENCODE_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr()); mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr());
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
@ -140,55 +140,55 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != TSDB_USER_VER_NUMBER) { if (sver != USER_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto USER_DECODE_OVER; goto _OVER;
} }
SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj)); SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj));
if (pRow == NULL) goto USER_DECODE_OVER; if (pRow == NULL) goto _OVER;
SUserObj *pUser = sdbGetRowObj(pRow); SUserObj *pUser = sdbGetRowObj(pRow);
if (pUser == NULL) goto USER_DECODE_OVER; if (pUser == NULL) goto _OVER;
pUser->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
pUser->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true);
if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto USER_DECODE_OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER)
SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, USER_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, USER_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, USER_DECODE_OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER)
int32_t numOfReadDbs = 0; int32_t numOfReadDbs = 0;
int32_t numOfWriteDbs = 0; int32_t numOfWriteDbs = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, USER_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, USER_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER)
pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->writeDbs =
taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER;
for (int32_t i = 0; i < numOfReadDbs; ++i) { for (int32_t i = 0; i < numOfReadDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
int32_t len = strlen(db) + 1; int32_t len = strlen(db) + 1;
taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN); taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN);
} }
for (int32_t i = 0; i < numOfWriteDbs; ++i) { for (int32_t i = 0; i < numOfWriteDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER)
int32_t len = strlen(db) + 1; int32_t len = strlen(db) + 1;
taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN); taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN);
} }
SDB_GET_RESERVE(pRaw, dataPos, TSDB_USER_RESERVE_SIZE, USER_DECODE_OVER) SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER)
terrno = 0; terrno = 0;
USER_DECODE_OVER: _OVER:
if (terrno != 0) { if (terrno != 0) {
mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr()); mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr());
taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->readDbs);
@ -220,6 +220,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) {
mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser); mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser);
taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->readDbs);
taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->writeDbs);
pUser->readDbs = NULL;
pUser->writeDbs = NULL;
return 0; return 0;
} }
@ -228,13 +230,8 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
pOld->updateTime = pNew->updateTime; pOld->updateTime = pNew->updateTime;
void *tmp1 = pOld->readDbs; TSWAP(pOld->readDbs, pNew->readDbs);
pOld->readDbs = pNew->readDbs; TSWAP(pOld->writeDbs, pNew->writeDbs);
pNew->readDbs = tmp1;
void *tmp2 = pOld->writeDbs;
pOld->writeDbs = pNew->writeDbs;
pNew->writeDbs = tmp2;
return 0; return 0;
} }
@ -277,6 +274,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
} }
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
char *param = strdup("====> test code to be deleted later <=====");
mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1);
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); mndTransDrop(pTrans);
@ -296,41 +296,41 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) {
if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
goto CREATE_USER_OVER; goto _OVER;
} }
mDebug("user:%s, start to create", createReq.user); mDebug("user:%s, start to create", createReq.user);
if (createReq.user[0] == 0) { if (createReq.user[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
goto CREATE_USER_OVER; goto _OVER;
} }
if (createReq.pass[0] == 0) { if (createReq.pass[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
goto CREATE_USER_OVER; goto _OVER;
} }
pUser = mndAcquireUser(pMnode, createReq.user); pUser = mndAcquireUser(pMnode, createReq.user);
if (pUser != NULL) { if (pUser != NULL) {
terrno = TSDB_CODE_MND_USER_ALREADY_EXIST; terrno = TSDB_CODE_MND_USER_ALREADY_EXIST;
goto CREATE_USER_OVER; goto _OVER;
} }
pOperUser = mndAcquireUser(pMnode, pReq->user); pOperUser = mndAcquireUser(pMnode, pReq->user);
if (pOperUser == NULL) { if (pOperUser == NULL) {
terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
goto CREATE_USER_OVER; goto _OVER;
} }
if (mndCheckCreateUserAuth(pOperUser) != 0) { if (mndCheckCreateUserAuth(pOperUser) != 0) {
goto CREATE_USER_OVER; goto _OVER;
} }
code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq);
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
CREATE_USER_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to create since %s", createReq.user, terrstr()); mError("user:%s, failed to create since %s", createReq.user, terrstr());
} }
@ -341,13 +341,13 @@ CREATE_USER_OVER:
return code; return code;
} }
static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) { static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNodeMsg *pReq) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER, &pReq->rpcMsg); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_TYPE_ALTER_USER, &pReq->rpcMsg);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("user:%s, failed to update since %s", pOld->user, terrstr()); mError("user:%s, failed to alter since %s", pOld->user, terrstr());
return -1; return -1;
} }
mDebug("trans:%d, used to update user:%s", pTrans->id, pOld->user); mDebug("trans:%d, used to alter user:%s", pTrans->id, pOld->user);
SSdbRaw *pRedoRaw = mndUserActionEncode(pNew); SSdbRaw *pRedoRaw = mndUserActionEncode(pNew);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
@ -368,7 +368,8 @@ static int32_t mndUpdateUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SNo
} }
static SHashObj *mndDupDbHash(SHashObj *pOld) { static SHashObj *mndDupDbHash(SHashObj *pOld) {
SHashObj *pNew = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); SHashObj *pNew =
taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pNew == NULL) { if (pNew == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -379,8 +380,8 @@ static SHashObj *mndDupDbHash(SHashObj *pOld) {
int32_t len = strlen(db) + 1; int32_t len = strlen(db) + 1;
if (taosHashPut(pNew, db, len, db, TSDB_DB_FNAME_LEN) != 0) { if (taosHashPut(pNew, db, len, db, TSDB_DB_FNAME_LEN) != 0) {
taosHashCancelIterate(pOld, db); taosHashCancelIterate(pOld, db);
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosHashCleanup(pNew); taosHashCleanup(pNew);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
db = taosHashIterate(pOld, db); db = taosHashIterate(pOld, db);
@ -399,38 +400,38 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
goto ALTER_USER_OVER; goto _OVER;
} }
mDebug("user:%s, start to alter", alterReq.user); mDebug("user:%s, start to alter", alterReq.user);
if (alterReq.user[0] == 0) { if (alterReq.user[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
goto ALTER_USER_OVER; goto _OVER;
} }
if (alterReq.pass[0] == 0) { if (alterReq.pass[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT;
goto ALTER_USER_OVER; goto _OVER;
} }
pUser = mndAcquireUser(pMnode, alterReq.user); pUser = mndAcquireUser(pMnode, alterReq.user);
if (pUser == NULL) { if (pUser == NULL) {
terrno = TSDB_CODE_MND_USER_NOT_EXIST; terrno = TSDB_CODE_MND_USER_NOT_EXIST;
goto ALTER_USER_OVER; goto _OVER;
} }
pOperUser = mndAcquireUser(pMnode, pReq->user); pOperUser = mndAcquireUser(pMnode, pReq->user);
if (pOperUser == NULL) { if (pOperUser == NULL) {
terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
goto ALTER_USER_OVER; goto _OVER;
} }
memcpy(&newUser, pUser, sizeof(SUserObj)); memcpy(&newUser, pUser, sizeof(SUserObj));
newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.readDbs = mndDupDbHash(pUser->readDbs);
newUser.writeDbs = mndDupDbHash(pUser->writeDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs);
if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { if (newUser.readDbs == NULL || newUser.writeDbs == NULL) {
goto ALTER_USER_OVER; goto _OVER;
} }
int32_t len = strlen(alterReq.dbname) + 1; int32_t len = strlen(alterReq.dbname) + 1;
@ -440,56 +441,56 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) {
if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) {
char pass[TSDB_PASSWORD_LEN + 1] = {0}; char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass); taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass);
memcpy(pUser->pass, pass, TSDB_PASSWORD_LEN); memcpy(newUser.pass, pass, TSDB_PASSWORD_LEN);
} else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) { } else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) {
newUser.superUser = alterReq.superUser; newUser.superUser = alterReq.superUser;
} else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) { } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) {
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST; terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER; goto _OVER;
} }
if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto ALTER_USER_OVER; goto _OVER;
} }
} else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) {
if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST; terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER; goto _OVER;
} }
} else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) {
taosHashClear(newUser.readDbs); taosHashClear(newUser.readDbs);
} else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) {
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST; terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER; goto _OVER;
} }
if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto ALTER_USER_OVER; goto _OVER;
} }
} else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST; terrno = TSDB_CODE_MND_DB_NOT_EXIST;
goto ALTER_USER_OVER; goto _OVER;
} }
} else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) {
taosHashClear(newUser.writeDbs); taosHashClear(newUser.writeDbs);
} else { } else {
terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; terrno = TSDB_CODE_MND_INVALID_ALTER_OPER;
goto ALTER_USER_OVER; goto _OVER;
} }
newUser.updateTime = taosGetTimestampMs(); newUser.updateTime = taosGetTimestampMs();
if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) { if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) {
goto ALTER_USER_OVER; goto _OVER;
} }
code = mndUpdateUser(pMnode, pUser, &newUser, pReq); code = mndAlterUser(pMnode, pUser, &newUser, pReq);
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
ALTER_USER_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); mError("user:%s, failed to alter since %s", alterReq.user, terrstr());
} }
@ -537,36 +538,36 @@ static int32_t mndProcessDropUserReq(SNodeMsg *pReq) {
if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
goto DROP_USER_OVER; goto _OVER;
} }
mDebug("user:%s, start to drop", dropReq.user); mDebug("user:%s, start to drop", dropReq.user);
if (dropReq.user[0] == 0) { if (dropReq.user[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; terrno = TSDB_CODE_MND_INVALID_USER_FORMAT;
goto DROP_USER_OVER; goto _OVER;
} }
pUser = mndAcquireUser(pMnode, dropReq.user); pUser = mndAcquireUser(pMnode, dropReq.user);
if (pUser == NULL) { if (pUser == NULL) {
terrno = TSDB_CODE_MND_USER_NOT_EXIST; terrno = TSDB_CODE_MND_USER_NOT_EXIST;
goto DROP_USER_OVER; goto _OVER;
} }
pOperUser = mndAcquireUser(pMnode, pReq->user); pOperUser = mndAcquireUser(pMnode, pReq->user);
if (pOperUser == NULL) { if (pOperUser == NULL) {
terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; terrno = TSDB_CODE_MND_NO_USER_FROM_CONN;
goto DROP_USER_OVER; goto _OVER;
} }
if (mndCheckDropUserAuth(pOperUser) != 0) { if (mndCheckDropUserAuth(pOperUser) != 0) {
goto DROP_USER_OVER; goto _OVER;
} }
code = mndDropUser(pMnode, pReq, pUser); code = mndDropUser(pMnode, pReq, pUser);
if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS;
DROP_USER_OVER: _OVER:
if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("user:%s, failed to drop since %s", dropReq.user, terrstr()); mError("user:%s, failed to drop since %s", dropReq.user, terrstr());
} }
@ -586,7 +587,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
goto GET_AUTH_OVER; goto _OVER;
} }
mTrace("user:%s, start to get auth", authReq.user); mTrace("user:%s, start to get auth", authReq.user);
@ -594,7 +595,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
pUser = mndAcquireUser(pMnode, authReq.user); pUser = mndAcquireUser(pMnode, authReq.user);
if (pUser == NULL) { if (pUser == NULL) {
terrno = TSDB_CODE_MND_USER_NOT_EXIST; terrno = TSDB_CODE_MND_USER_NOT_EXIST;
goto GET_AUTH_OVER; goto _OVER;
} }
memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); memcpy(authRsp.user, pUser->user, TSDB_USER_LEN);
@ -622,7 +623,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
void *pRsp = rpcMallocCont(contLen); void *pRsp = rpcMallocCont(contLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto GET_AUTH_OVER; goto _OVER;
} }
tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp);
@ -631,15 +632,14 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) {
pReq->rspLen = contLen; pReq->rspLen = contLen;
code = 0; code = 0;
GET_AUTH_OVER: _OVER:
mndReleaseUser(pMnode, pUser); mndReleaseUser(pMnode, pUser);
taosHashCleanup(authRsp.readDbs); tFreeSGetUserAuthRsp(&authRsp);
taosHashCleanup(authRsp.writeDbs);
return code; return code;
} }
static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode; SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
@ -652,29 +652,24 @@ static int32_t mndRetrieveUsers(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pB
if (pShow->pIter == NULL) break; if (pShow->pIter == NULL) break;
cols = 0; cols = 0;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; char name[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]); STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->user, pShow->bytes[cols]);
colDataAppend(pColInfo, numOfRows, (const char*) name, false); colDataAppend(pColInfo, numOfRows, (const char *)name, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
const char* src = pUser->superUser? "super":"normal"; const char *src = pUser->superUser ? "super" : "normal";
char b[10+VARSTR_HEADER_SIZE] = {0}; char b[10 + VARSTR_HEADER_SIZE] = {0};
STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src)); STR_WITH_SIZE_TO_VARSTR(b, src, strlen(src));
colDataAppend(pColInfo, numOfRows, (const char*) b, false); colDataAppend(pColInfo, numOfRows, (const char *)b, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
colDataAppend(pColInfo, numOfRows, (const char*) &pUser->createdTime, false); colDataAppend(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
STR_WITH_MAXSIZE_TO_VARSTR(name, pUser->acct, pShow->bytes[cols]);
colDataAppend(pColInfo, numOfRows, (const char*) name, false);
numOfRows++; numOfRows++;
sdbRelease(pSdb, pUser); sdbRelease(pSdb, pUser);

4
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -45,8 +45,8 @@ int32_t mndInitVgroup(SMnode *pMnode) {
.encodeFp = (SdbEncodeFp)mndVgroupActionEncode, .encodeFp = (SdbEncodeFp)mndVgroupActionEncode,
.decodeFp = (SdbDecodeFp)mndVgroupActionDecode, .decodeFp = (SdbDecodeFp)mndVgroupActionDecode,
.insertFp = (SdbInsertFp)mndVgroupActionInsert, .insertFp = (SdbInsertFp)mndVgroupActionInsert,
.updateFp = (SdbUpdateFp)mndVgroupActionDelete, .updateFp = (SdbUpdateFp)mndVgroupActionUpdate,
.deleteFp = (SdbDeleteFp)mndVgroupActionUpdate}; .deleteFp = (SdbDeleteFp)mndVgroupActionDelete};
mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp);
mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp);

7
source/dnode/mnode/impl/src/mnode.c
View File

@ -43,7 +43,7 @@
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#define MQ_TIMER_MS 3000 #define MQ_TIMER_MS 2000
#define TRNAS_TIMER_MS 6000 #define TRNAS_TIMER_MS 6000
static void *mndBuildTimerMsg(int32_t *pContLen) { static void *mndBuildTimerMsg(int32_t *pContLen) {
@ -368,7 +368,7 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) {
} }
if (isReq && (pRpc->contLen == 0 || pRpc->pCont == NULL)) { if (isReq && (pRpc->contLen == 0 || pRpc->pCont == NULL)) {
terrno = TSDB_CODE_MND_INVALID_MSG_LEN; terrno = TSDB_CODE_INVALID_MSG_LEN;
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
return -1; return -1;
} }
@ -418,7 +418,6 @@ int64_t mndGenerateUid(char *name, int32_t len) {
} while (true); } while (true);
} }
int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo, int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgroupInfo *pVgroupInfo,
SMonGrantInfo *pGrantInfo) { SMonGrantInfo *pGrantInfo) {
if (!mndIsMaster(pMnode)) return -1; if (!mndIsMaster(pMnode)) return -1;
@ -528,4 +527,4 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) { int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad) {
pLoad->syncState = pMnode->syncMgmt.state; pLoad->syncState = pMnode->syncMgmt.state;
return 0; return 0;
} }

1
source/dnode/mnode/impl/test/CMakeLists.txt
View File

@ -8,6 +8,7 @@ add_subdirectory(func)
add_subdirectory(mnode) add_subdirectory(mnode)
add_subdirectory(profile) add_subdirectory(profile)
add_subdirectory(qnode) add_subdirectory(qnode)
add_subdirectory(sdb)
add_subdirectory(show) add_subdirectory(show)
add_subdirectory(sma) add_subdirectory(sma)
add_subdirectory(snode) add_subdirectory(snode)

6
source/dnode/mnode/impl/test/acct/acct.cpp
View File

@ -32,7 +32,7 @@ TEST_F(MndTestAcct, 01_Create_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
} }
TEST_F(MndTestAcct, 02_Alter_Acct) { TEST_F(MndTestAcct, 02_Alter_Acct) {
@ -42,7 +42,7 @@ TEST_F(MndTestAcct, 02_Alter_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
} }
TEST_F(MndTestAcct, 03_Drop_Acct) { TEST_F(MndTestAcct, 03_Drop_Acct) {
@ -52,5 +52,5 @@ TEST_F(MndTestAcct, 03_Drop_Acct) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_ACCT, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_MND_MSG_NOT_PROCESSED); ASSERT_EQ(pRsp->code, TSDB_CODE_MSG_NOT_PROCESSED);
} }

1
source/dnode/mnode/impl/test/db/db.cpp
View File

@ -90,7 +90,6 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
taosMsleep(1000); // Wait for the vnode to become the leader
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);

12
source/dnode/mnode/impl/test/sdb/CMakeLists.txt Normal file
View File

@ -0,0 +1,12 @@
aux_source_directory(. MNODE_SDB_TEST_SRC)
add_executable(sdbTest ${MNODE_SDB_TEST_SRC})
target_link_libraries(
sdbTest
PUBLIC sut
PUBLIC sdb
)
add_test(
NAME sdbTest
COMMAND sdbTest
)

83
source/dnode/mnode/impl/test/sdb/sdbTest.cpp Normal file
View File

@ -0,0 +1,83 @@
/**
* @file sdbTest.cpp
* @author slguan (slguan@taosdata.com)
* @brief MNODE module sdb tests
* @version 1.0
* @date 2022-04-27
*
* @copyright Copyright (c) 2022
*
*/
#include "sut.h"
class MndTestShow : public ::testing::Test {
protected:
static void SetUpTestSuite() { test.Init("/tmp/mnode_test_show", 9021); }
static void TearDownTestSuite() { test.Cleanup(); }
static Testbase test;
public:
void SetUp() override {}
void TearDown() override {}
};
Testbase MndTestShow::test;
TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) {
SShowReq showReq = {0};
showReq.type = TSDB_MGMT_TABLE_MAX;
int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSShowReq(pReq, contLen, &showReq);
tFreeSShowReq(&showReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_NE(pRsp->code, 0);
}
TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) {
SShowReq showReq = {0};
showReq.type = TSDB_MGMT_TABLE_START;
int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSShowReq(pReq, contLen, &showReq);
tFreeSShowReq(&showReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_SYSTABLE_RETRIEVE, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_NE(pRsp->code, 0);
}
TEST_F(MndTestShow, 03_ShowMsg_Conn) {
char passwd[] = "taosdata";
char secretEncrypt[TSDB_PASSWORD_LEN] = {0};
taosEncryptPass_c((uint8_t*)passwd, strlen(passwd), secretEncrypt);
SConnectReq connectReq = {0};
connectReq.pid = 1234;
strcpy(connectReq.app, "mnode_test_show");
strcpy(connectReq.db, "");
strcpy(connectReq.user, "root");
strcpy(connectReq.passwd, secretEncrypt);
int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSConnectReq(pReq, contLen, &connectReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
test.SendShowReq(TSDB_MGMT_TABLE_CONNS, "connections", "");
// EXPECT_EQ(test.GetShowRows(), 1);
}
TEST_F(MndTestShow, 04_ShowMsg_Cluster) {
test.SendShowReq(TSDB_MGMT_TABLE_CLUSTER, "cluster", "");
EXPECT_EQ(test.GetShowRows(), 1);
}

12
source/dnode/mnode/impl/test/sma/sma.cpp
View File

@ -115,18 +115,15 @@ void* MndTestSma::BuildCreateBSmaStbReq(const char* stbname, int32_t* pContLen)
SMCreateStbReq createReq = {0}; SMCreateStbReq createReq = {0};
createReq.numOfColumns = 3; createReq.numOfColumns = 3;
createReq.numOfTags = 1; createReq.numOfTags = 1;
createReq.numOfSmas = 1;
createReq.igExists = 0; createReq.igExists = 0;
createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField)); createReq.pColumns = taosArrayInit(createReq.numOfColumns, sizeof(SField));
createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField)); createReq.pTags = taosArrayInit(createReq.numOfTags, sizeof(SField));
createReq.pSmas = taosArrayInit(createReq.numOfSmas, sizeof(SField));
strcpy(createReq.name, stbname); strcpy(createReq.name, stbname);
PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts"); PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_TIMESTAMP, "ts");
PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1"); PushField(createReq.pColumns, 2, TSDB_DATA_TYPE_TINYINT, "col1");
PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2"); PushField(createReq.pColumns, 8, TSDB_DATA_TYPE_BIGINT, "col2");
PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1"); PushField(createReq.pTags, 2, TSDB_DATA_TYPE_TINYINT, "tag1");
PushField(createReq.pSmas, 2, TSDB_DATA_TYPE_TINYINT, "col1");
int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq);
void* pHead = rpcMallocCont(tlen); void* pHead = rpcMallocCont(tlen);
@ -191,7 +188,7 @@ void* MndTestSma::BuildDropTSmaReq(const char* smaname, int8_t igNotExists, int3
} }
TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) { TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) {
#if 0 #if 0
const char* dbname = "1.d1"; const char* dbname = "1.d1";
const char* stbname = "1.d1.stb"; const char* stbname = "1.d1.stb";
const char* smaname = "1.d1.sma"; const char* smaname = "1.d1.sma";
@ -245,7 +242,7 @@ TEST_F(MndTestSma, 01_Create_Show_Meta_Drop_Restart_Stb) {
test.SendShowRetrieveReq(); test.SendShowRetrieveReq();
EXPECT_EQ(test.GetShowRows(), 0); EXPECT_EQ(test.GetShowRows(), 0);
} }
#endif #endif
} }
TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) { TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
@ -259,14 +256,13 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateDbReq(dbname, &contLen); pReq = BuildCreateDbReq(dbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(1000); // Wait for the vnode to become the leader
} }
{ {
pReq = BuildCreateBSmaStbReq(stbname, &contLen); pReq = BuildCreateBSmaStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_CREATE_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
} }
@ -282,7 +278,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildDropStbReq(stbname, &contLen); pReq = BuildDropStbReq(stbname, &contLen);
pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_DROP_STB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables",dbname); test.SendShowReq(TSDB_MGMT_TABLE_STB, "user_stables", dbname);
EXPECT_EQ(test.GetShowRows(), 0); EXPECT_EQ(test.GetShowRows(), 0);
} }

8
source/dnode/mnode/impl/test/stb/stb.cpp
View File

@ -305,7 +305,6 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -439,7 +438,6 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -499,7 +497,6 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -541,7 +538,6 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -606,7 +602,6 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -660,7 +655,6 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -721,7 +715,6 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -782,7 +775,6 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {

2
source/dnode/mnode/sdb/CMakeLists.txt
View File

@ -6,5 +6,5 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
) )
target_link_libraries( target_link_libraries(
sdb os common util sdb os common util wal
) )

24
source/dnode/mnode/sdb/inc/sdbInt.h
View File

@ -19,10 +19,6 @@
#include "os.h" #include "os.h"
#include "sdb.h" #include "sdb.h"
#include "thash.h"
#include "tlockfree.h"
#include "tlog.h"
#include "tmsg.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
@ -53,26 +49,6 @@ typedef struct SSdbRow {
char pObj[]; char pObj[];
} SSdbRow; } SSdbRow;
typedef struct SSdb {
SMnode *pMnode;
char *currDir;
char *syncDir;
char *tmpDir;
int64_t lastCommitVer;
int64_t curVer;
int64_t tableVer[SDB_MAX];
int64_t maxId[SDB_MAX];
EKeyType keyTypes[SDB_MAX];
SHashObj *hashObjs[SDB_MAX];
SRWLatch locks[SDB_MAX];
SdbInsertFp insertFps[SDB_MAX];
SdbUpdateFp updateFps[SDB_MAX];
SdbDeleteFp deleteFps[SDB_MAX];
SdbDeployFp deployFps[SDB_MAX];
SdbEncodeFp encodeFps[SDB_MAX];
SdbDecodeFp decodeFps[SDB_MAX];
} SSdb;
const char *sdbTableName(ESdbType type); const char *sdbTableName(ESdbType type);
void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper); void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);

7
source/dnode/mnode/sdb/src/sdb.c
View File

@ -87,7 +87,7 @@ void sdbCleanup(SSdb *pSdb) {
SSdbRow *pRow = *ppRow; SSdbRow *pRow = *ppRow;
if (pRow == NULL) continue; if (pRow == NULL) continue;
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, true);
ppRow = taosHashIterate(hash, ppRow); ppRow = taosHashIterate(hash, ppRow);
} }
} }
@ -162,7 +162,4 @@ static int32_t sdbCreateDir(SSdb *pSdb) {
return 0; return 0;
} }
int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { int64_t sdbUpdateVer(SSdb *pSdb, int32_t val) { return atomic_add_fetch_64(&pSdb->curVer, val); }
pSdb->curVer += val;
return pSdb->curVer;
}

9
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "sdbInt.h" #include "sdbInt.h"
#include "tchecksum.h" #include "tchecksum.h"
#include "wal.h"
#define SDB_TABLE_SIZE 24 #define SDB_TABLE_SIZE 24
#define SDB_RESERVE_SIZE 512 #define SDB_RESERVE_SIZE 512
@ -28,7 +29,7 @@ static int32_t sdbRunDeployFp(SSdb *pSdb) {
if (fp == NULL) continue; if (fp == NULL) continue;
if ((*fp)(pSdb->pMnode) != 0) { if ((*fp)(pSdb->pMnode) != 0) {
mError("failed to deploy sdb:%d since %s", i, terrstr()); mError("failed to deploy sdb:%s since %s", sdbTableName(i), terrstr());
return -1; return -1;
} }
} }
@ -137,7 +138,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
int32_t readLen = 0; int32_t readLen = 0;
int64_t ret = 0; int64_t ret = 0;
SSdbRaw *pRaw = taosMemoryMalloc(SDB_MAX_SIZE); SSdbRaw *pRaw = taosMemoryMalloc(WAL_MAX_SIZE + 100);
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed read file since %s", terrstr()); mError("failed read file since %s", terrstr());
@ -202,7 +203,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
break; break;
} }
code = sdbWriteNotFree(pSdb, pRaw); code = sdbWriteWithoutFree(pSdb, pRaw);
if (code != 0) { if (code != 0) {
mError("failed to read file:%s since %s", file, terrstr()); mError("failed to read file:%s since %s", file, terrstr());
goto PARSE_SDB_DATA_ERROR; goto PARSE_SDB_DATA_ERROR;
@ -263,7 +264,7 @@ static int32_t sdbWriteFileImp(SSdb *pSdb) {
continue; continue;
} }
sdbPrintOper(pSdb, pRow, "writeFile"); sdbPrintOper(pSdb, pRow, "write");
SSdbRaw *pRaw = (*encodeFp)(pRow->pObj); SSdbRaw *pRaw = (*encodeFp)(pRow->pObj);
if (pRaw != NULL) { if (pRaw != NULL) {

50
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -51,7 +51,9 @@ const char *sdbTableName(ESdbType type) {
case SDB_TOPIC: case SDB_TOPIC:
return "topic"; return "topic";
case SDB_VGROUP: case SDB_VGROUP:
return "vgId"; return "vgroup";
case SDB_SMA:
return "sma";
case SDB_STB: case SDB_STB:
return "stb"; return "stb";
case SDB_DB: case SDB_DB:
@ -86,13 +88,13 @@ void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) {
EKeyType keyType = pSdb->keyTypes[pRow->type]; EKeyType keyType = pSdb->keyTypes[pRow->type];
if (keyType == SDB_KEY_BINARY) { if (keyType == SDB_KEY_BINARY) {
mTrace("%s:%s, refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, mTrace("%s:%s, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, oper,
oper, pRow->pObj, sdbStatusStr(pRow->status)); pRow->pObj, sdbStatusStr(pRow->status));
} else if (keyType == SDB_KEY_INT32) { } else if (keyType == SDB_KEY_INT32) {
mTrace("%s:%d, refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, mTrace("%s:%d, ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount,
pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status)); oper, pRow->pObj, sdbStatusStr(pRow->status));
} else if (keyType == SDB_KEY_INT64) { } else if (keyType == SDB_KEY_INT64) {
mTrace("%s:%" PRId64 ", refCount:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, mTrace("%s:%" PRId64 ", ref:%d oper:%s row:%p status:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj,
pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status)); pRow->refCount, oper, pRow->pObj, sdbStatusStr(pRow->status));
} else { } else {
} }
@ -135,18 +137,18 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (pOldRow != NULL) { if (pOldRow != NULL) {
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, false);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno; return terrno;
} }
pRow->refCount = 0; pRow->refCount = 0;
pRow->status = pRaw->status; pRow->status = pRaw->status;
sdbPrintOper(pSdb, pRow, "insertRow"); sdbPrintOper(pSdb, pRow, "insert");
if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) {
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, false);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno; return terrno;
} }
@ -162,7 +164,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosWLockLatch(pLock); taosWLockLatch(pLock);
taosHashRemove(hash, pRow->pObj, keySize); taosHashRemove(hash, pRow->pObj, keySize);
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, false);
terrno = code; terrno = code;
return terrno; return terrno;
} }
@ -191,7 +193,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow *pOldRow = *ppOldRow; SSdbRow *pOldRow = *ppOldRow;
pOldRow->status = pRaw->status; pOldRow->status = pRaw->status;
sdbPrintOper(pSdb, pOldRow, "updateRow"); sdbPrintOper(pSdb, pOldRow, "update");
taosRUnLockLatch(pLock); taosRUnLockLatch(pLock);
int32_t code = 0; int32_t code = 0;
@ -200,7 +202,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
} }
sdbFreeRow(pSdb, pNewRow); sdbFreeRow(pSdb, pNewRow, false);
pSdb->tableVer[pOldRow->type]++; pSdb->tableVer[pOldRow->type]++;
return code; return code;
@ -213,27 +215,27 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (ppOldRow == NULL || *ppOldRow == NULL) { if (ppOldRow == NULL || *ppOldRow == NULL) {
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, false);
terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; terrno = TSDB_CODE_SDB_OBJ_NOT_THERE;
return terrno; return terrno;
} }
SSdbRow *pOldRow = *ppOldRow; SSdbRow *pOldRow = *ppOldRow;
pOldRow->status = pRaw->status; pOldRow->status = pRaw->status;
sdbPrintOper(pSdb, pOldRow, "deleteRow"); sdbPrintOper(pSdb, pOldRow, "delete");
taosHashRemove(hash, pOldRow->pObj, keySize); taosHashRemove(hash, pOldRow->pObj, keySize);
taosWUnLockLatch(pLock); taosWUnLockLatch(pLock);
pSdb->tableVer[pOldRow->type]++; pSdb->tableVer[pOldRow->type]++;
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, false);
sdbCheck(pSdb, pOldRow); sdbCheck(pSdb, pOldRow);
// sdbRelease(pSdb, pOldRow->pObj); // sdbRelease(pSdb, pOldRow->pObj);
return 0; return 0;
} }
int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) { int32_t sdbWriteWithoutFree(SSdb *pSdb, SSdbRaw *pRaw) {
SHashObj *hash = sdbGetHash(pSdb, pRaw->type); SHashObj *hash = sdbGetHash(pSdb, pRaw->type);
if (hash == NULL) return terrno; if (hash == NULL) return terrno;
@ -266,7 +268,7 @@ int32_t sdbWriteNotFree(SSdb *pSdb, SSdbRaw *pRaw) {
} }
int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) { int32_t sdbWrite(SSdb *pSdb, SSdbRaw *pRaw) {
int32_t code = sdbWriteNotFree(pSdb, pRaw); int32_t code = sdbWriteWithoutFree(pSdb, pRaw);
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
return code; return code;
} }
@ -296,7 +298,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, const void *pKey) {
case SDB_STATUS_UPDATING: case SDB_STATUS_UPDATING:
atomic_add_fetch_32(&pRow->refCount, 1); atomic_add_fetch_32(&pRow->refCount, 1);
pRet = pRow->pObj; pRet = pRow->pObj;
sdbPrintOper(pSdb, pRow, "acquireRow"); sdbPrintOper(pSdb, pRow, "acquire");
break; break;
case SDB_STATUS_CREATING: case SDB_STATUS_CREATING:
terrno = TSDB_CODE_SDB_OBJ_CREATING; terrno = TSDB_CODE_SDB_OBJ_CREATING;
@ -318,9 +320,9 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) {
taosRLockLatch(pLock); taosRLockLatch(pLock);
int32_t ref = atomic_load_32(&pRow->refCount); int32_t ref = atomic_load_32(&pRow->refCount);
sdbPrintOper(pSdb, pRow, "checkRow"); sdbPrintOper(pSdb, pRow, "check");
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, true);
} }
taosRUnLockLatch(pLock); taosRUnLockLatch(pLock);
@ -330,15 +332,15 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
if (pObj == NULL) return; if (pObj == NULL) return;
SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow)); SSdbRow *pRow = (SSdbRow *)((char *)pObj - sizeof(SSdbRow));
if (pRow->type >= SDB_MAX ) return; if (pRow->type >= SDB_MAX) return;
SRWLatch *pLock = &pSdb->locks[pRow->type]; SRWLatch *pLock = &pSdb->locks[pRow->type];
taosRLockLatch(pLock); taosRLockLatch(pLock);
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "releaseRow"); sdbPrintOper(pSdb, pRow, "release");
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
sdbFreeRow(pSdb, pRow); sdbFreeRow(pSdb, pRow, true);
} }
taosRUnLockLatch(pLock); taosRUnLockLatch(pLock);
@ -372,7 +374,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
} }
atomic_add_fetch_32(&pRow->refCount, 1); atomic_add_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "fetchRow"); sdbPrintOper(pSdb, pRow, "fetch");
*ppObj = pRow->pObj; *ppObj = pRow->pObj;
break; break;
} }

6
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -27,7 +27,7 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) {
pRaw->sver = sver; pRaw->sver = sver;
pRaw->dataLen = dataLen; pRaw->dataLen = dataLen;
mTrace("raw:%p, is created, len:%d", pRaw, dataLen); mTrace("raw:%p, is created, len:%d table:%s", pRaw, dataLen, sdbTableName(type));
return pRaw; return pRaw;
} }
@ -107,7 +107,9 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_
return -1; return -1;
} }
memcpy(pRaw->pData + dataPos, pVal, valLen); if (pVal != NULL) {
memcpy(pRaw->pData + dataPos, pVal, valLen);
}
return 0; return 0;
} }

6
source/dnode/mnode/sdb/src/sdbRow.c
View File

@ -36,14 +36,14 @@ void *sdbGetRowObj(SSdbRow *pRow) {
return pRow->pObj; return pRow->pObj;
} }
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) { void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) {
// remove attached object such as trans // remove attached object such as trans
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type]; SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) { if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj); (*deleteFp)(pSdb, pRow->pObj, callFunc);
} }
sdbPrintOper(pSdb, pRow, "freeRow"); sdbPrintOper(pSdb, pRow, "free");
mTrace("row:%p, is freed", pRow->pObj); mTrace("row:%p, is freed", pRow->pObj);
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);

8
source/dnode/qnode/src/qnode.c
View File

@ -17,7 +17,7 @@
#include "qndInt.h" #include "qndInt.h"
#include "query.h" #include "query.h"
#include "qworker.h" #include "qworker.h"
//#include "tudf.h" #include "libs/function/function.h"
SQnode *qndOpen(const SQnodeOpt *pOption) { SQnode *qndOpen(const SQnodeOpt *pOption) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
@ -26,7 +26,9 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
return NULL; return NULL;
} }
//udfcOpen(); if (udfcOpen() != 0) {
qError("qnode can not open udfc");
}
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) {
taosMemoryFreeClear(pQnode); taosMemoryFreeClear(pQnode);
@ -40,7 +42,7 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
void qndClose(SQnode *pQnode) { void qndClose(SQnode *pQnode) {
qWorkerDestroy((void **)&pQnode->pQuery); qWorkerDestroy((void **)&pQnode->pQuery);
//udfcClose(); udfcClose();
taosMemoryFree(pQnode); taosMemoryFree(pQnode);
} }

15
source/dnode/vnode/CMakeLists.txt
View File

@ -5,9 +5,7 @@ target_sources(
PRIVATE PRIVATE
# vnode # vnode
"src/vnd/vnodeOpen.c" "src/vnd/vnodeOpen.c"
"src/vnd/vnodeArenaMAImpl.c" "src/vnd/vnodeBufPool.c"
"src/vnd/vnodeBufferPool.c"
# "src/vnd/vnodeBufferPool2.c"
"src/vnd/vnodeCfg.c" "src/vnd/vnodeCfg.c"
"src/vnd/vnodeCommit.c" "src/vnd/vnodeCommit.c"
"src/vnd/vnodeInt.c" "src/vnd/vnodeInt.c"
@ -21,18 +19,18 @@ target_sources(
"src/meta/metaOpen.c" "src/meta/metaOpen.c"
"src/meta/metaIdx.c" "src/meta/metaIdx.c"
"src/meta/metaTable.c" "src/meta/metaTable.c"
"src/meta/metaTDBImpl.c" "src/meta/metaQuery.c"
# "src/meta/metaBDBImpl.c" "src/meta/metaCommit.c"
"src/meta/metaEntry.c"
# tsdb # tsdb
"src/tsdb/tsdbTDBImpl.c" "src/tsdb/tsdbTDBImpl.c"
"src/tsdb/tsdbCommit.c" "src/tsdb/tsdbCommit.c"
"src/tsdb/tsdbCompact.c" "src/tsdb/tsdbCommit2.c"
"src/tsdb/tsdbFile.c" "src/tsdb/tsdbFile.c"
"src/tsdb/tsdbFS.c" "src/tsdb/tsdbFS.c"
"src/tsdb/tsdbMain.c" "src/tsdb/tsdbOpen.c"
"src/tsdb/tsdbMemTable.c" "src/tsdb/tsdbMemTable.c"
"src/tsdb/tsdbOptions.c"
"src/tsdb/tsdbRead.c" "src/tsdb/tsdbRead.c"
"src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbReadImpl.c"
"src/tsdb/tsdbScan.c" "src/tsdb/tsdbScan.c"
@ -71,6 +69,7 @@ target_link_libraries(
PUBLIC transport PUBLIC transport
PUBLIC stream PUBLIC stream
) )
target_compile_definitions(vnode PUBLIC -DMETA_REFACT)
if(${BUILD_TEST}) if(${BUILD_TEST})
add_subdirectory(test) add_subdirectory(test)

124
source/dnode/vnode/inc/vnode.h
View File

@ -32,6 +32,8 @@
#include "tmsg.h" #include "tmsg.h"
#include "trow.h" #include "trow.h"
#include "tdbInt.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
@ -59,34 +61,47 @@ int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg);
int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeCompact(SVnode *pVnode);
int32_t vnodeSync(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName); int vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode);
int64_t vnodeGetSyncHandle(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode);
void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId);
// meta // meta
typedef struct SMeta SMeta; // todo: remove typedef struct SMeta SMeta; // todo: remove
typedef struct SMTbCursor SMTbCursor; typedef struct SMetaReader SMetaReader;
typedef struct SMetaEntry SMetaEntry;
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
void metaReaderClear(SMetaReader *pReader);
int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
int metaReadNext(SMetaReader *pReader);
#if 1 // refact APIs below (TODO)
typedef SVCreateTbReq STbCfg; typedef SVCreateTbReq STbCfg;
typedef SVCreateTSmaReq SSmaCfg; typedef SVCreateTSmaReq SSmaCfg;
typedef struct SMTbCursor SMTbCursor;
SMTbCursor *metaOpenTbCursor(SMeta *pMeta); SMTbCursor *metaOpenTbCursor(SMeta *pMeta);
void metaCloseTbCursor(SMTbCursor *pTbCur); void metaCloseTbCursor(SMTbCursor *pTbCur);
char *metaTbCursorNext(SMTbCursor *pTbCur); int metaTbCursorNext(SMTbCursor *pTbCur);
#endif
// tsdb // tsdb
typedef struct STsdb STsdb; typedef struct STsdb STsdb;
typedef struct STsdbQueryCond STsdbQueryCond; typedef void *tsdbReaderT;
typedef void *tsdbReaderT;
#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 #define BLOCK_LOAD_OFFSET_SEQ_ORDER 1
#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 #define BLOCK_LOAD_TABLE_SEQ_ORDER 2
#define BLOCK_LOAD_TABLE_RR_ORDER 3 #define BLOCK_LOAD_TABLE_RR_ORDER 3
tsdbReaderT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, tsdbReaderT *tsdbQueryTables(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId,
uint64_t taskId); uint64_t taskId);
tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, tsdbReaderT tsdbQueryCacheLast(STsdb *tsdb, SQueryTableDataCond *pCond, STableGroupInfo *groupList, uint64_t qId,
void *pMemRef); void *pMemRef);
int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo); int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo);
bool isTsdbCacheLastRow(tsdbReaderT *pReader); bool isTsdbCacheLastRow(tsdbReaderT *pReader);
@ -98,6 +113,7 @@ bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle);
void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo);
int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis); int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReaderT *pTsdbReadHandle, SColumnDataAgg **pBlockStatis);
SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList); SArray *tsdbRetrieveDataBlock(tsdbReaderT *pTsdbReadHandle, SArray *pColumnIdList);
void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond *pCond);
void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); int32_t tsdbGetOneTableGroup(void *pMeta, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
@ -119,34 +135,28 @@ int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *p
// need to reposition // need to reposition
// structs // structs
struct SMetaCfg {
uint64_t lruSize;
};
struct STsdbCfg { struct STsdbCfg {
int8_t precision; int8_t precision;
int8_t update; int8_t update;
int8_t compression; int8_t compression;
int32_t days; int8_t slLevel;
int32_t minRows; int32_t days;
int32_t maxRows; int32_t minRows;
int32_t keep2; int32_t maxRows;
int32_t keep0; int32_t keep0;
int32_t keep1; int32_t keep1;
uint64_t lruCacheSize; int32_t keep2;
SArray *retentions; SArray *retentions;
}; };
struct SVnodeCfg { struct SVnodeCfg {
int32_t vgId; int32_t vgId;
char dbname[TSDB_DB_NAME_LEN]; char dbname[TSDB_DB_NAME_LEN];
uint64_t dbId; uint64_t dbId;
uint64_t wsize; int32_t szPage;
uint64_t ssize; int32_t szCache;
uint64_t lsize; uint64_t szBuf;
bool isHeapAllocator; bool isHeap;
uint32_t ttl;
uint32_t keep;
int8_t streamMode; int8_t streamMode;
bool isWeak; bool isWeak;
STsdbCfg tsdbCfg; STsdbCfg tsdbCfg;
@ -157,24 +167,52 @@ struct SVnodeCfg {
int8_t hashMethod; int8_t hashMethod;
}; };
struct STsdbQueryCond {
STimeWindow twindow;
int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo *colList;
bool loadExternalRows; // load external rows or not
int32_t type; // data block load type:
};
typedef struct { typedef struct {
TSKEY lastKey; TSKEY lastKey;
uint64_t uid; uint64_t uid;
} STableKeyInfo; } STableKeyInfo;
// sync integration struct SMetaEntry {
void vnodeSyncSetQ(SVnode *pVnode, void *qHandle); int64_t version;
void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle); int8_t type;
int32_t vnodeSyncStart(SVnode *pVnode); tb_uid_t uid;
const char *name;
union {
struct {
SSchemaWrapper schema;
SSchemaWrapper schemaTag;
} stbEntry;
struct {
int64_t ctime;
int32_t ttlDays;
tb_uid_t suid;
const void *pTags;
} ctbEntry;
struct {
int64_t ctime;
int32_t ttlDays;
SSchemaWrapper schema;
} ntbEntry;
};
};
struct SMetaReader {
int32_t flags;
SMeta *pMeta;
SCoder coder;
SMetaEntry me;
void *pBuf;
int szBuf;
};
struct SMTbCursor {
TDBC *pDbc;
void *pKey;
void *pVal;
int kLen;
int vLen;
SMetaReader mr;
};
#ifdef __cplusplus #ifdef __cplusplus
} }

102
source/dnode/vnode/src/inc/meta.h
View File

@ -16,13 +16,14 @@
#ifndef _TD_VNODE_META_H_ #ifndef _TD_VNODE_META_H_
#define _TD_VNODE_META_H_ #define _TD_VNODE_META_H_
#include "vnodeInt.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
typedef struct SMetaIdx SMetaIdx; typedef struct SMetaIdx SMetaIdx;
typedef struct SMetaDB SMetaDB; typedef struct SMetaDB SMetaDB;
typedef struct SMCtbCursor SMCtbCursor;
typedef struct SMSmaCursor SMSmaCursor; typedef struct SMSmaCursor SMSmaCursor;
// metaDebug ================== // metaDebug ==================
@ -36,8 +37,16 @@ typedef struct SMSmaCursor SMSmaCursor;
// clang-format on // clang-format on
// metaOpen ================== // metaOpen ==================
int metaOpen(SVnode* pVnode, SMeta** ppMeta);
int metaClose(SMeta* pMeta); // metaEntry ==================
int metaEncodeEntry(SCoder* pCoder, const SMetaEntry* pME);
int metaDecodeEntry(SCoder* pCoder, SMetaEntry* pME);
// metaTable ==================
int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq);
// metaQuery ==================
int metaGetTableEntryByVersion(SMetaReader* pReader, int64_t version, tb_uid_t uid);
// metaIdx ================== // metaIdx ==================
int metaOpenIdx(SMeta* pMeta); int metaOpenIdx(SMeta* pMeta);
@ -45,32 +54,63 @@ void metaCloseIdx(SMeta* pMeta);
int metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions); int metaSaveTableToIdx(SMeta* pMeta, const STbCfg* pTbOptions);
int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid); int metaRemoveTableFromIdx(SMeta* pMeta, tb_uid_t uid);
// metaCommit ==================
static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); } static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64(); }
#define META_SUPER_TABLE TD_SUPER_TABLE struct SMeta {
#define META_CHILD_TABLE TD_CHILD_TABLE char* path;
#define META_NORMAL_TABLE TD_NORMAL_TABLE SVnode* pVnode;
TENV* pEnv;
TXN txn;
TDB* pTbDb;
TDB* pSkmDb;
TDB* pUidIdx;
TDB* pNameIdx;
TDB* pCtbIdx;
TDB* pTagIdx;
TDB* pTtlIdx;
SMetaIdx* pIdx;
};
int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle); typedef struct {
int metaDropTable(SMeta* pMeta, tb_uid_t uid); int64_t version;
int metaCommit(SMeta* pMeta); tb_uid_t uid;
int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); } STbDbKey;
int32_t metaDropTSma(SMeta* pMeta, int64_t indexUid);
STbCfg* metaGetTbInfoByUid(SMeta* pMeta, tb_uid_t uid);
STbCfg* metaGetTbInfoByName(SMeta* pMeta, char* tbname, tb_uid_t* uid);
SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline);
STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver);
void* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid, bool isDecode);
STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid);
SArray* metaGetSmaTbUids(SMeta* pMeta, bool isDup);
int metaGetTbNum(SMeta* pMeta);
SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseSmaCursor(SMSmaCursor* pSmaCur);
int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur);
SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseCtbCurosr(SMCtbCursor* pCtbCur);
tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
#pragma pack(push, 1)
typedef struct {
tb_uid_t uid;
int32_t sver;
} SSkmDbKey;
#pragma pack(pop)
typedef struct {
tb_uid_t suid;
tb_uid_t uid;
} SCtbIdxKey;
#pragma pack(push, 1)
typedef struct {
tb_uid_t suid;
int16_t cid;
char data[];
} STagIdxKey;
#pragma pack(pop)
typedef struct {
int64_t dtime;
tb_uid_t uid;
} STtlIdxKey;
#if 1
// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaDropTable(SMeta* pMeta, tb_uid_t uid);
SMSmaCursor* metaOpenSmaCursor(SMeta* pMeta, tb_uid_t uid);
void metaCloseSmaCursor(SMSmaCursor* pSmaCur);
int64_t metaSmaCursorNext(SMSmaCursor* pSmaCur);
#ifndef META_REFACT
// SMetaDB // SMetaDB
int metaOpenDB(SMeta* pMeta); int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta);
@ -78,17 +118,9 @@ int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
#endif
// SMetaIdx #endif
tb_uid_t metaGenerateUid(SMeta* pMeta);
struct SMeta {
char* path;
SVnode* pVnode;
SMetaDB* pDB;
SMetaIdx* pIdx;
};
#ifdef __cplusplus #ifdef __cplusplus
} }

Some files were not shown because too many files have changed in this diff Show More