xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into vnode_refact_merge

This commit is contained in:
Hongze Cheng 2022-04-26 05:49:42 +00:00
parent 63e4884cce 3352f9520e
commit 6fe299cdcd
311 changed files with 18037 additions and 14326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -7,7 +7,7 @@ project(
) )
if (NOT DEFINED TD_SOURCE_DIR) if (NOT DEFINED TD_SOURCE_DIR)
set( TD_SOURCE_DIR ${CMAKE_SOURCE_DIR} ) set( TD_SOURCE_DIR ${PROJECT_SOURCE_DIR} )
endif() endif()
set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake") set(TD_SUPPORT_DIR "${TD_SOURCE_DIR}/cmake")

2
Jenkinsfile vendored
View File

@ -113,7 +113,7 @@ pipeline {
''' '''
sh''' sh'''
cd ${WKC}/debug cd ${WKC}/debug
ctest ctest -VV
''' '''
} }
} }

20
Jenkinsfile2
View File

@ -88,6 +88,12 @@ def pre_test(){
cmake .. > /dev/null cmake .. > /dev/null
make -j4> /dev/null make -j4> /dev/null
''' '''
sh'''
cd ${WKPY}
git reset --hard
git pull
pip3 install .
'''
return 1 return 1
} }
@ -97,6 +103,7 @@ pipeline {
environment{ environment{
WK = '/var/lib/jenkins/workspace/TDinternal' WK = '/var/lib/jenkins/workspace/TDinternal'
WKC= '/var/lib/jenkins/workspace/TDengine' WKC= '/var/lib/jenkins/workspace/TDengine'
WKPY= '/var/lib/jenkins/workspace/taos-connector-python'
} }
stages { stages {
stage('pre_build'){ stage('pre_build'){
@ -113,12 +120,17 @@ pipeline {
timeout(time: 45, unit: 'MINUTES'){ timeout(time: 45, unit: 'MINUTES'){
pre_test() pre_test()
sh''' sh'''
cd ${WKC}/tests cd ${WKC}/debug
./test-all.sh b1fq ctest -VV
''' '''
sh''' sh'''
cd ${WKC}/debug export LD_LIBRARY_PATH=${WKC}/debug/build/lib
ctest cd ${WKC}/tests/system-test
./fulltest.sh
'''
sh'''
cd ${WKC}/tests
./test-all.sh b1fq
''' '''
} }
} }

2
cmake/cmake.define
View File

@ -1,6 +1,6 @@
cmake_minimum_required(VERSION 3.16) cmake_minimum_required(VERSION 3.16)
set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_VERBOSE_MAKEFILE OFF)
#set output directory #set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)

91
contrib/CMakeLists.txt
View File

@ -14,24 +14,9 @@ if(${BUILD_PTHREAD})
cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif() endif()
# iconv # gnu regex
if(${BUILD_WITH_ICONV}) if(${BUILD_GNUREGEX})
cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# msvc regex
if(${BUILD_MSVCREGEX})
cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wcwidth
if(${BUILD_WCWIDTH})
cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wingetopt
if(${BUILD_WINGETOPT})
cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif() endif()
# googletest # googletest
@ -114,27 +99,8 @@ if(${BUILD_TEST})
target_include_directories( target_include_directories(
gtest gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
)
if(${TD_WINDOWS})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
)
endif(${TD_WINDOWS})
if(${TD_LINUX})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux> PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
) )
endif(${TD_LINUX})
if(${TD_DARWIN})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
)
endif(${TD_DARWIN})
endif(${BUILD_TEST}) endif(${BUILD_TEST})
# cJson # cJson
@ -216,53 +182,6 @@ if(${BUILD_WITH_NURAFT})
add_subdirectory(nuraft) add_subdirectory(nuraft)
endif(${BUILD_WITH_NURAFT}) endif(${BUILD_WITH_NURAFT})
# pthread
if(${BUILD_PTHREAD})
set(CMAKE_BUILD_TYPE release)
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
add_library(pthread STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
endif()
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
endif(${BUILD_WITH_ICONV})
# wingetopt
if(${BUILD_WINGETOPT})
add_subdirectory(wingetopt)
endif(${BUILD_WINGETOPT})
# msvcregex
if(${BUILD_MSVCREGEX})
add_library(msvcregex STATIC "")
target_sources(msvcregex
PRIVATE "msvcregex/regex.c"
)
target_include_directories(msvcregex
PRIVATE "msvcregex"
)
target_link_libraries(msvcregex
INTERFACE Shell32
)
SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
endif(${BUILD_MSVCREGEX})
# msvcregex
if(${BUILD_WCWIDTH})
add_library(wcwidth STATIC "")
target_sources(wcwidth
PRIVATE "wcwidth/wcwidth.c"
)
target_include_directories(wcwidth
PRIVATE "wcwidth"
)
SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
endif(${BUILD_WCWIDTH})
# CRAFT # CRAFT
if(${BUILD_WITH_CRAFT}) if(${BUILD_WITH_CRAFT})
add_library(craft STATIC IMPORTED GLOBAL) add_library(craft STATIC IMPORTED GLOBAL)
@ -319,12 +238,8 @@ if(${BUILD_WITH_SQLITE})
target_link_libraries(sqlite target_link_libraries(sqlite
INTERFACE m INTERFACE m
INTERFACE pthread INTERFACE pthread
)
if(NOT TD_WINDOWS)
target_link_libraries(sqlite
INTERFACE dl INTERFACE dl
) )
endif(NOT TD_WINDOWS)
endif(${BUILD_WITH_SQLITE}) endif(${BUILD_WITH_SQLITE})
# pthread # pthread

7
contrib/test/craft/CMakeLists.txt
View File

@ -1,9 +1,2 @@
add_executable(simulate_vnode "simulate_vnode.c") add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
if(${BUILD_WINGETOPT})
target_link_libraries(simulate_vnode PUBLIC wingetopt)
target_include_directories(
simulate_vnode
PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
)
endif()

72
contrib/test/tdev/src/main.c
View File

@ -6,39 +6,43 @@
#define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
#define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
static inline void tPutA(void **buf, uint64_t val) { #define tPutA(buf, val) \
memcpy(buf, &val, sizeof(val)); ({ \
*buf = POINTER_SHIFT(buf, sizeof(val)); memcpy(buf, &val, sizeof(val)); \
} POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutB(void **buf, uint64_t val) { #define tPutB(buf, val) \
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; ({ \
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
((uint8_t *)buf)[0] = (val)&0xff; ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \
*buf = POINTER_SHIFT(buf, sizeof(val)); ((uint8_t *)buf)[0] = (val)&0xff; \
} POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutC(void **buf, uint64_t val) { #define tPutC(buf, val) \
if (buf) { ({ \
((uint64_t *)buf)[0] = (val); if (buf) { \
POINTER_SHIFT(buf, sizeof(val)); ((uint64_t *)buf)[0] = (val); \
} POINTER_SHIFT(buf, sizeof(val)); \
*buf = NULL; } \
} NULL; \
})
static inline void tPutD(void **buf, uint64_t val) { #define tPutD(buf, val) \
uint64_t tmp = val; ({ \
for (size_t i = 0; i < sizeof(val); i++) { uint64_t tmp = val; \
((uint8_t *)buf)[i] = tmp & 0xff; for (size_t i = 0; i < sizeof(val); i++) { \
tmp >>= 8; ((uint8_t *)buf)[i] = tmp & 0xff; \
} tmp >>= 8; \
*buf = POINTER_SHIFT(buf, sizeof(val)); } \
} POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutE(void **buf, uint64_t val) { static inline void tPutE(void **buf, uint64_t val) {
if (buf) { if (buf) {
@ -57,7 +61,7 @@ static void func(T t) {
switch (t) { switch (t) {
case A: case A:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutA(pBuf, val); pBuf = tPutA(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }
@ -65,7 +69,7 @@ static void func(T t) {
break; break;
case B: case B:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutB(pBuf, val); pBuf = tPutB(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }
@ -73,7 +77,7 @@ static void func(T t) {
break; break;
case C: case C:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutC(pBuf, val); pBuf = tPutC(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }
@ -81,7 +85,7 @@ static void func(T t) {
break; break;
case D: case D:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutD(pBuf, val); pBuf = tPutD(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) { if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf; pBuf = buf;
} }

16
example/src/tmq.c
View File

@ -17,6 +17,7 @@
#include <stdio.h> #include <stdio.h>
#include <string.h> #include <string.h>
#include <time.h> #include <time.h>
#include <unistd.h>
#include "taos.h" #include "taos.h"
static int running = 1; static int running = 1;
@ -47,6 +48,7 @@ int32_t init_env() {
return -1; return -1;
} }
taos_free_result(pRes); taos_free_result(pRes);
sleep(1);
pRes = taos_query(pConn, "use abc1"); pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
@ -101,10 +103,8 @@ int32_t create_topic() {
} }
taos_free_result(pRes); taos_free_result(pRes);
/*const char* sql = "select * from tu1";*/ pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");
/*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/ /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");*/
/*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
return -1; return -1;
@ -162,9 +162,10 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "group.id", "tg2"); tmq_conf_set(conf, "group.id", "tg2");
tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.user", "root");
tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "td.connect.pass", "taosdata");
tmq_conf_set(conf, "td.connect.db", "abc1"); /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print); tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print);
tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
assert(tmq);
return tmq; return tmq;
} }
@ -267,10 +268,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
} }
int main(int argc, char* argv[]) { int main(int argc, char* argv[]) {
int code;
if (argc > 1) { if (argc > 1) {
printf("env init\n"); printf("env init\n");
code = init_env(); if (init_env() < 0) {
return -1;
}
create_topic(); create_topic();
} }
tmq_t* tmq = build_consumer(); tmq_t* tmq = build_consumer();

4
examples/c/CMakeLists.txt
View File

@ -1,7 +1,7 @@
PROJECT(TDengine) PROJECT(TDengine)
IF (TD_LINUX) IF (TD_LINUX)
INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc)
AUX_SOURCE_DIRECTORY(. SRC) AUX_SOURCE_DIRECTORY(. SRC)
ADD_EXECUTABLE(demo apitest.c) ADD_EXECUTABLE(demo apitest.c)
TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread ) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread )
@ -13,7 +13,7 @@ IF (TD_LINUX)
TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua) TARGET_LINK_LIBRARIES(epoll taos_static trpc tutil pthread lua)
ENDIF () ENDIF ()
IF (TD_DARWIN) IF (TD_DARWIN)
INCLUDE_DIRECTORIES(. ${TD_COMMUNITY_DIR}/src/inc ${TD_COMMUNITY_DIR}/src/client/inc ${TD_COMMUNITY_DIR}/inc) INCLUDE_DIRECTORIES(. ${TD_SOURCE_DIR}/src/inc ${TD_SOURCE_DIR}/src/client/inc ${TD_SOURCE_DIR}/inc)
AUX_SOURCE_DIRECTORY(. SRC) AUX_SOURCE_DIRECTORY(. SRC)
ADD_EXECUTABLE(demo demo.c) ADD_EXECUTABLE(demo demo.c)
TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua) TARGET_LINK_LIBRARIES(demo taos_static trpc tutil pthread lua)

6
examples/c/asyncdemo.c
View File

@ -45,7 +45,7 @@ typedef struct {
void taos_insert_call_back(void *param, TAOS_RES *tres, int code); void taos_insert_call_back(void *param, TAOS_RES *tres, int code);
void taos_select_call_back(void *param, TAOS_RES *tres, int code); void taos_select_call_back(void *param, TAOS_RES *tres, int code);
void taos_error(TAOS *taos); void shellPrintError(TAOS *taos);
static void queryDB(TAOS *taos, char *command) { static void queryDB(TAOS *taos, char *command) {
int i; int i;
@ -102,7 +102,7 @@ int main(int argc, char *argv[])
taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
if (taos == NULL) if (taos == NULL)
taos_error(taos); shellPrintError(taos);
printf("success to connect to server\n"); printf("success to connect to server\n");
@ -193,7 +193,7 @@ int main(int argc, char *argv[])
return 0; return 0;
} }
void taos_error(TAOS *con) void shellPrintError(TAOS *con)
{ {
fprintf(stderr, "TDengine error: %s\n", taos_errstr(con)); fprintf(stderr, "TDengine error: %s\n", taos_errstr(con));
taos_close(con); taos_close(con);

4
include/client/taos.h
View File

@ -222,6 +222,8 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *));
DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT void tmq_list_destroy(tmq_list_t *); DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *);
DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *);
#if 0 #if 0
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
@ -232,7 +234,7 @@ DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errst
DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t);
/* ------------------------TMQ CONSUMER INTERFACE------------------------ */ /* ------------------------TMQ CONSUMER INTERFACE------------------------ */
DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list);
DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq); DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t *tmq);
DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t *tmq, tmq_list_t **topics);
DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time);

13
include/common/tdatablock.h
View File

@ -238,10 +238,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32
static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
int8_t needCompress) { int8_t needCompress) {
int32_t* colSizes = (int32_t*)data; int32_t* actualLen = (int32_t*) data;
data += sizeof(int32_t);
uint64_t* groupId = (uint64_t*) data;
data += sizeof(uint64_t);
int32_t* colSizes = (int32_t*)data;
data += numOfCols * sizeof(int32_t); data += numOfCols * sizeof(int32_t);
*dataLen = (numOfCols * sizeof(int32_t));
*dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
int32_t numOfRows = pBlock->info.rows; int32_t numOfRows = pBlock->info.rows;
for (int32_t col = 0; col < numOfCols; ++col) { for (int32_t col = 0; col < numOfCols; ++col) {
@ -273,6 +279,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
colSizes[col] = htonl(colSizes[col]); colSizes[col] = htonl(colSizes[col]);
} }
*actualLen = *dataLen;
*groupId = pBlock->info.groupId;
} }
#ifdef __cplusplus #ifdef __cplusplus

16
include/common/tglobal.h
View File

@ -34,13 +34,9 @@ extern int32_t tsVersion;
extern int32_t tsStatusInterval; extern int32_t tsStatusInterval;
// common // common
extern int32_t tsRpcTimer;
extern int32_t tsRpcMaxTime;
extern bool tsRpcForceTcp; // all commands go to tcp protocol if this is enabled
extern int32_t tsMaxConnections; extern int32_t tsMaxConnections;
extern int32_t tsMaxShellConns; extern int32_t tsMaxShellConns;
extern int32_t tsShellActivityTimer; extern int32_t tsShellActivityTimer;
extern int32_t tsMaxTmrCtrl;
extern int32_t tsCompressMsgSize; extern int32_t tsCompressMsgSize;
extern int32_t tsCompressColData; extern int32_t tsCompressColData;
extern int32_t tsMaxNumOfDistinctResults; extern int32_t tsMaxNumOfDistinctResults;
@ -94,10 +90,10 @@ extern bool tsRetrieveBlockingModel; // retrieve threads will be blocked
extern bool tsKeepOriginalColumnName; extern bool tsKeepOriginalColumnName;
extern bool tsDeadLockKillQuery; extern bool tsDeadLockKillQuery;
// query client
extern int32_t tsQueryPolicy;
// client // client
extern int32_t tsMaxWildCardsLen;
extern int32_t tsMaxRegexStringLen;
extern int32_t tsMaxNumOfOrderedResults;
extern int32_t tsMinSlidingTime; extern int32_t tsMinSlidingTime;
extern int32_t tsMinIntervalTime; extern int32_t tsMinIntervalTime;
extern int32_t tsMaxStreamComputDelay; extern int32_t tsMaxStreamComputDelay;
@ -127,9 +123,9 @@ extern SDiskCfg tsDiskCfg[];
#define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,
const char *apolloUrl, SArray *pArgs, bool tsc); char *apolloUrl, SArray *pArgs, bool tsc);
int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc); int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc);
void taosCleanupCfg(); void taosCleanupCfg();
void taosCfgDynamicOptions(const char *option, const char *value); void taosCfgDynamicOptions(const char *option, const char *value);
void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary); void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary);

170
include/common/tmsg.h
View File

@ -229,6 +229,14 @@ typedef struct {
typedef struct { typedef struct {
int32_t totalLen; int32_t totalLen;
int32_t len; int32_t len;
// head of SSubmitBlk
// int64_t uid; // table unique id
// int64_t suid; // stable id
// int32_t sversion; // data schema version
// int32_t dataLen; // data part length, not including the SSubmitBlk head
// int32_t schemaLen; // schema length, if length is 0, no schema exists
// int16_t numOfRows; // total number of rows in current submit block
// head of SSubmitBlk
const void* pMsg; const void* pMsg;
} SSubmitMsgIter; } SSubmitMsgIter;
@ -237,6 +245,15 @@ int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
// TODO: KEEP one suite of iterator API finally.
// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts
// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later
// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter
// int32_t tInitSubmitMsgIterEx(const SSubmitReq* pMsg, SSubmitMsgIter* pIter);
// int32_t tGetSubmitMsgNextEx(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
// int32_t tInitSubmitBlkIterEx(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
// STSRow* tGetSubmitBlkNextEx(SSubmitBlkIter* pIter);
typedef struct { typedef struct {
int32_t index; // index of failed block in submit blocks int32_t index; // index of failed block in submit blocks
int32_t vnode; // vnode index of failed block int32_t vnode; // vnode index of failed block
@ -281,7 +298,6 @@ typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; char name[TSDB_TABLE_FNAME_LEN];
int8_t igExists; int8_t igExists;
float xFilesFactor; float xFilesFactor;
int32_t aggregationMethod;
int32_t delay; int32_t delay;
int32_t ttl; int32_t ttl;
int32_t numOfColumns; int32_t numOfColumns;
@ -645,13 +661,18 @@ int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
typedef struct { typedef struct {
SArray* epSetList; // SArray<SEpSet> SArray* addrsList; // SArray<SQueryNodeAddr>
} SQnodeListRsp; } SQnodeListRsp;
int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
void tFreeSQnodeListRsp(SQnodeListRsp* pRsp); void tFreeSQnodeListRsp(SQnodeListRsp* pRsp);
typedef struct SQueryNodeAddr {
int32_t nodeId; // vgId or qnodeId
SEpSet epSet;
} SQueryNodeAddr;
typedef struct { typedef struct {
SArray* pArray; // Array of SUseDbRsp SArray* pArray; // Array of SUseDbRsp
} SUseDbBatchRsp; } SUseDbBatchRsp;
@ -1289,15 +1310,13 @@ typedef struct {
} SMVCreateStreamRsp, SMSCreateStreamRsp; } SMVCreateStreamRsp, SMSCreateStreamRsp;
typedef struct { typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic
int8_t igExists; int8_t igExists;
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
int8_t withTagSchema;
char* sql; char* sql;
char* ast; char* ast;
int64_t subDbUid;
char subscribeDbName[TSDB_DB_NAME_LEN]; char subscribeDbName[TSDB_DB_NAME_LEN];
} SCMCreateTopicReq; } SCMCreateTopicReq;
@ -1317,30 +1336,34 @@ typedef struct {
} SMqConsumerLostMsg; } SMqConsumerLostMsg;
typedef struct { typedef struct {
int32_t topicNum;
int64_t consumerId; int64_t consumerId;
char cgroup[TSDB_CGROUP_LEN]; char cgroup[TSDB_CGROUP_LEN];
SArray* topicNames; // SArray<char*> SArray* topicNames; // SArray<char**>
} SCMSubscribeReq; } SCMSubscribeReq;
static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pReq->topicNum);
tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->cgroup); tlen += taosEncodeString(buf, pReq->cgroup);
for (int32_t i = 0; i < pReq->topicNum; i++) { int32_t topicNum = taosArrayGetSize(pReq->topicNames);
tlen += taosEncodeFixedI32(buf, topicNum);
for (int32_t i = 0; i < topicNum; i++) {
tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i));
} }
return tlen; return tlen;
} }
static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
buf = taosDecodeFixedI32(buf, &pReq->topicNum);
buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->cgroup); buf = taosDecodeStringTo(buf, pReq->cgroup);
pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*));
for (int32_t i = 0; i < pReq->topicNum; i++) { int32_t topicNum;
buf = taosDecodeFixedI32(buf, &topicNum);
pReq->topicNames = taosArrayInit(topicNum, sizeof(void*));
for (int32_t i = 0; i < topicNum; i++) {
char* name; char* name;
buf = taosDecodeString(buf, &name); buf = taosDecodeString(buf, &name);
taosArrayPush(pReq->topicNames, &name); taosArrayPush(pReq->topicNames, &name);
@ -1494,10 +1517,8 @@ typedef struct {
int32_t delay; int32_t delay;
int32_t qmsg1Len; int32_t qmsg1Len;
int32_t qmsg2Len; int32_t qmsg2Len;
func_id_t* pFuncIds; char* qmsg1; // pAst1:qmsg1:SRetention1 => trigger aggr task1
char* qmsg1; // not null: pAst1:qmsg1:SRetention1 => trigger aggr task1 char* qmsg2; // pAst2:qmsg2:SRetention2 => trigger aggr task2
char* qmsg2; // not null: pAst2:qmsg2:SRetention2 => trigger aggr task2
int8_t nFuncIds;
} SRSmaParam; } SRSmaParam;
int tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam); int tEncodeSRSmaParam(SCoder* pCoder, const SRSmaParam* pRSmaParam);
@ -1994,7 +2015,6 @@ typedef struct {
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
int8_t withTagSchema;
char* qmsg; char* qmsg;
} SMqRebVgReq; } SMqRebVgReq;
@ -2009,7 +2029,6 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR
tlen += taosEncodeFixedI8(buf, pReq->withTbName); tlen += taosEncodeFixedI8(buf, pReq->withTbName);
tlen += taosEncodeFixedI8(buf, pReq->withSchema); tlen += taosEncodeFixedI8(buf, pReq->withSchema);
tlen += taosEncodeFixedI8(buf, pReq->withTag); tlen += taosEncodeFixedI8(buf, pReq->withTag);
tlen += taosEncodeFixedI8(buf, pReq->withTagSchema);
if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
tlen += taosEncodeString(buf, pReq->qmsg); tlen += taosEncodeString(buf, pReq->qmsg);
} }
@ -2026,7 +2045,6 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq)
buf = taosDecodeFixedI8(buf, &pReq->withTbName); buf = taosDecodeFixedI8(buf, &pReq->withTbName);
buf = taosDecodeFixedI8(buf, &pReq->withSchema); buf = taosDecodeFixedI8(buf, &pReq->withSchema);
buf = taosDecodeFixedI8(buf, &pReq->withTag); buf = taosDecodeFixedI8(buf, &pReq->withTag);
buf = taosDecodeFixedI8(buf, &pReq->withTagSchema);
if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
buf = taosDecodeString(buf, &pReq->qmsg); buf = taosDecodeString(buf, &pReq->qmsg);
} }
@ -2076,80 +2094,6 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
return buf; return buf;
} }
typedef struct {
int64_t leftForVer;
int32_t vgId;
int32_t epoch;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
} SMqCancelConnReq;
static FORCE_INLINE int32_t tEncodeSMqCancelConnReq(void** buf, const SMqCancelConnReq* pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI32(buf, pReq->epoch);
tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->topicName);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqCancelConnReq(void* buf, SMqCancelConnReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI32(buf, &pReq->epoch);
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->topicName);
return buf;
}
typedef struct {
int8_t reserved;
} SMqCancelConnRsp;
typedef struct {
int64_t leftForVer;
int32_t vgId;
int64_t oldConsumerId;
int64_t newConsumerId;
char* topic;
} SMqMVRebReq;
static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->leftForVer);
tlen += taosEncodeFixedI32(buf, pReq->vgId);
tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId);
tlen += taosEncodeFixedI64(buf, pReq->newConsumerId);
tlen += taosEncodeString(buf, pReq->topic);
return tlen;
}
static FORCE_INLINE void* tDecodeSMqMVRebReq(void* buf, SMqMVRebReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->leftForVer);
buf = taosDecodeFixedI32(buf, &pReq->vgId);
buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId);
buf = taosDecodeFixedI64(buf, &pReq->newConsumerId);
buf = taosDecodeString(buf, &pReq->topic);
return buf;
}
typedef struct {
SMsgHead header;
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CGROUP_LEN];
} SMqSetCVgRsp;
typedef struct {
SMsgHead header;
int32_t vgId;
int64_t consumerId;
char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CGROUP_LEN];
} SMqMVRebRsp;
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
int64_t offset; int64_t offset;
@ -2171,6 +2115,25 @@ int32_t tDecodeSMqOffset(SCoder* decoder, SMqOffset* pOffset);
int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq); int32_t tEncodeSMqCMCommitOffsetReq(SCoder* encoder, const SMqCMCommitOffsetReq* pReq);
int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq); int32_t tDecodeSMqCMCommitOffsetReq(SCoder* decoder, SMqCMCommitOffsetReq* pReq);
static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
if (pSW == NULL) return pSW;
pSW->nCols = pSchemaWrapper->nCols;
pSW->sver = pSchemaWrapper->sver;
pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
if (pSW->pSchema == NULL) {
taosMemoryFree(pSW);
return NULL;
}
memcpy(pSW->pSchema, pSchemaWrapper->pSchema, pSW->nCols * sizeof(SSchema));
return pSW;
}
static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) {
taosMemoryFree(pSchemaWrapper->pSchema);
taosMemoryFree(pSchemaWrapper);
}
static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pSchema->type); tlen += taosEncodeFixedI8(buf, pSchema->type);
@ -2181,13 +2144,13 @@ static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema
return tlen; return tlen;
} }
static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { static FORCE_INLINE void* taosDecodeSSchema(const void* buf, SSchema* pSchema) {
buf = taosDecodeFixedI8(buf, &pSchema->type); buf = taosDecodeFixedI8(buf, &pSchema->type);
buf = taosDecodeFixedI8(buf, &pSchema->flags); buf = taosDecodeFixedI8(buf, &pSchema->flags);
buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeFixedI32(buf, &pSchema->bytes);
buf = taosDecodeFixedI16(buf, &pSchema->colId); buf = taosDecodeFixedI16(buf, &pSchema->colId);
buf = taosDecodeStringTo(buf, pSchema->name); buf = taosDecodeStringTo(buf, pSchema->name);
return buf; return (void*)buf;
} }
static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) {
@ -2218,7 +2181,7 @@ static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWr
return tlen; return tlen;
} }
static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { static FORCE_INLINE void* taosDecodeSSchemaWrapper(const void* buf, SSchemaWrapper* pSW) {
buf = taosDecodeVariantI32(buf, &pSW->nCols); buf = taosDecodeVariantI32(buf, &pSW->nCols);
buf = taosDecodeVariantI32(buf, &pSW->sver); buf = taosDecodeVariantI32(buf, &pSW->sver);
pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema));
@ -2229,7 +2192,7 @@ static FORCE_INLINE void* taosDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pS
for (int32_t i = 0; i < pSW->nCols; i++) { for (int32_t i = 0; i < pSW->nCols; i++) {
buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); buf = taosDecodeSSchema(buf, &pSW->pSchema[i]);
} }
return buf; return (void*)buf;
} }
static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) { static FORCE_INLINE int32_t tEncodeSSchemaWrapper(SCoder* pEncoder, const SSchemaWrapper* pSW) {
@ -2614,7 +2577,6 @@ typedef struct {
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
int8_t withTagSchema;
SArray* blockDataLen; // SArray<int32_t> SArray* blockDataLen; // SArray<int32_t>
SArray* blockData; // SArray<SRetrieveTableRsp*> SArray* blockData; // SArray<SRetrieveTableRsp*>
SArray* blockTbName; // SArray<char*> SArray* blockTbName; // SArray<char*>
@ -2633,13 +2595,16 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp
tlen += taosEncodeFixedI8(buf, pRsp->withTbName); tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
tlen += taosEncodeFixedI8(buf, pRsp->withSchema); tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
tlen += taosEncodeFixedI8(buf, pRsp->withTag); tlen += taosEncodeFixedI8(buf, pRsp->withTag);
tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema);
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
void* data = taosArrayGetP(pRsp->blockData, i); void* data = taosArrayGetP(pRsp->blockData, i);
tlen += taosEncodeFixedI32(buf, bLen); tlen += taosEncodeFixedI32(buf, bLen);
tlen += taosEncodeBinary(buf, data, bLen); tlen += taosEncodeBinary(buf, data, bLen);
if (pRsp->withSchema) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i);
tlen += taosEncodeSSchemaWrapper(buf, pSW);
}
} }
} }
return tlen; return tlen;
@ -2652,11 +2617,11 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
buf = taosDecodeFixedI32(buf, &pRsp->blockNum); buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*)); pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
if (pRsp->blockNum != 0) { if (pRsp->blockNum != 0) {
buf = taosDecodeFixedI8(buf, &pRsp->withTbName); buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
buf = taosDecodeFixedI8(buf, &pRsp->withSchema); buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
buf = taosDecodeFixedI8(buf, &pRsp->withTag); buf = taosDecodeFixedI8(buf, &pRsp->withTag);
buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema);
for (int32_t i = 0; i < pRsp->blockNum; i++) { for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = 0; int32_t bLen = 0;
@ -2665,6 +2630,11 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
buf = taosDecodeBinary(buf, &data, bLen); buf = taosDecodeBinary(buf, &data, bLen);
taosArrayPush(pRsp->blockDataLen, &bLen); taosArrayPush(pRsp->blockDataLen, &bLen);
taosArrayPush(pRsp->blockData, &data); taosArrayPush(pRsp->blockData, &data);
if (pRsp->withSchema) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper));
buf = taosDecodeSSchemaWrapper(buf, pSW);
taosArrayPush(pRsp->blockSchema, &pSW);
}
} }
} }
return (void*)buf; return (void*)buf;

22
include/common/tmsgdef.h
View File

@ -85,6 +85,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "dnode-server-status", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "dnode-net-test", NULL, NULL)
// Requests handled by MNODE // Requests handled by MNODE
TD_NEW_MSG_SEG(TDMT_MND_MSG) TD_NEW_MSG_SEG(TDMT_MND_MSG)
@ -144,7 +145,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp)
TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-mq-ask-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-tmr", SMTimerReq, SMTimerReq)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "mnode-mq-consumer-lost", SMTimerReq, SMTimerReq)
TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg)
@ -175,11 +176,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
@ -205,6 +202,21 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
// sync integration
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)

178
include/common/ttokendef.h
View File

@ -156,87 +156,96 @@
#define TK_INTERVAL 138 #define TK_INTERVAL 138
#define TK_TOPIC 139 #define TK_TOPIC 139
#define TK_AS 140 #define TK_AS 140
#define TK_DESC 141 #define TK_WITH 141
#define TK_DESCRIBE 142 #define TK_SCHEMA 142
#define TK_RESET 143 #define TK_DESC 143
#define TK_QUERY 144 #define TK_DESCRIBE 144
#define TK_EXPLAIN 145 #define TK_RESET 145
#define TK_ANALYZE 146 #define TK_QUERY 146
#define TK_VERBOSE 147 #define TK_EXPLAIN 147
#define TK_NK_BOOL 148 #define TK_ANALYZE 148
#define TK_RATIO 149 #define TK_VERBOSE 149
#define TK_COMPACT 150 #define TK_NK_BOOL 150
#define TK_VNODES 151 #define TK_RATIO 151
#define TK_IN 152 #define TK_COMPACT 152
#define TK_OUTPUTTYPE 153 #define TK_VNODES 153
#define TK_AGGREGATE 154 #define TK_IN 154
#define TK_BUFSIZE 155 #define TK_OUTPUTTYPE 155
#define TK_STREAM 156 #define TK_AGGREGATE 156
#define TK_INTO 157 #define TK_BUFSIZE 157
#define TK_TRIGGER 158 #define TK_STREAM 158
#define TK_AT_ONCE 159 #define TK_INTO 159
#define TK_WINDOW_CLOSE 160 #define TK_TRIGGER 160
#define TK_WATERMARK 161 #define TK_AT_ONCE 161
#define TK_KILL 162 #define TK_WINDOW_CLOSE 162
#define TK_CONNECTION 163 #define TK_WATERMARK 163
#define TK_MERGE 164 #define TK_KILL 164
#define TK_VGROUP 165 #define TK_CONNECTION 165
#define TK_REDISTRIBUTE 166 #define TK_MERGE 166
#define TK_SPLIT 167 #define TK_VGROUP 167
#define TK_SYNCDB 168 #define TK_REDISTRIBUTE 168
#define TK_NULL 169 #define TK_SPLIT 169
#define TK_NK_QUESTION 170 #define TK_SYNCDB 170
#define TK_NK_ARROW 171 #define TK_NULL 171
#define TK_ROWTS 172 #define TK_NK_QUESTION 172
#define TK_TBNAME 173 #define TK_NK_ARROW 173
#define TK_QSTARTTS 174 #define TK_ROWTS 174
#define TK_QENDTS 175 #define TK_TBNAME 175
#define TK_WSTARTTS 176 #define TK_QSTARTTS 176
#define TK_WENDTS 177 #define TK_QENDTS 177
#define TK_WDURATION 178 #define TK_WSTARTTS 178
#define TK_CAST 179 #define TK_WENDTS 179
#define TK_NOW 180 #define TK_WDURATION 180
#define TK_TODAY 181 #define TK_CAST 181
#define TK_TIMEZONE 182 #define TK_NOW 182
#define TK_COUNT 183 #define TK_TODAY 183
#define TK_FIRST 184 #define TK_TIMEZONE 184
#define TK_LAST 185 #define TK_COUNT 185
#define TK_LAST_ROW 186 #define TK_FIRST 186
#define TK_BETWEEN 187 #define TK_LAST 187
#define TK_IS 188 #define TK_LAST_ROW 188
#define TK_NK_LT 189 #define TK_BETWEEN 189
#define TK_NK_GT 190 #define TK_IS 190
#define TK_NK_LE 191 #define TK_NK_LT 191
#define TK_NK_GE 192 #define TK_NK_GT 192
#define TK_NK_NE 193 #define TK_NK_LE 193
#define TK_MATCH 194 #define TK_NK_GE 194
#define TK_NMATCH 195 #define TK_NK_NE 195
#define TK_CONTAINS 196 #define TK_MATCH 196
#define TK_JOIN 197 #define TK_NMATCH 197
#define TK_INNER 198 #define TK_CONTAINS 198
#define TK_SELECT 199 #define TK_JOIN 199
#define TK_DISTINCT 200 #define TK_INNER 200
#define TK_WHERE 201 #define TK_SELECT 201
#define TK_PARTITION 202 #define TK_DISTINCT 202
#define TK_BY 203 #define TK_WHERE 203
#define TK_SESSION 204 #define TK_PARTITION 204
#define TK_STATE_WINDOW 205 #define TK_BY 205
#define TK_SLIDING 206 #define TK_SESSION 206
#define TK_FILL 207 #define TK_STATE_WINDOW 207
#define TK_VALUE 208 #define TK_SLIDING 208
#define TK_NONE 209 #define TK_FILL 209
#define TK_PREV 210 #define TK_VALUE 210
#define TK_LINEAR 211 #define TK_NONE 211
#define TK_NEXT 212 #define TK_PREV 212
#define TK_GROUP 213 #define TK_LINEAR 213
#define TK_HAVING 214 #define TK_NEXT 214
#define TK_ORDER 215 #define TK_GROUP 215
#define TK_SLIMIT 216 #define TK_HAVING 216
#define TK_SOFFSET 217 #define TK_ORDER 217
#define TK_LIMIT 218 #define TK_SLIMIT 218
#define TK_OFFSET 219 #define TK_SOFFSET 219
#define TK_ASC 220 #define TK_LIMIT 220
#define TK_NULLS 221 #define TK_OFFSET 221
#define TK_ASC 222
#define TK_NULLS 223
#define TK_ID 224
#define TK_NK_BITNOT 225
#define TK_INSERT 226
#define TK_VALUES 227
#define TK_IMPORT 228
#define TK_NK_SEMI 229
#define TK_FILE 230
#define TK_NK_SPACE 300 #define TK_NK_SPACE 300
#define TK_NK_COMMENT 301 #define TK_NK_COMMENT 301
@ -244,13 +253,6 @@
#define TK_NK_HEX 303 // hex number 0x123 #define TK_NK_HEX 303 // hex number 0x123
#define TK_NK_OCT 304 // oct number #define TK_NK_OCT 304 // oct number
#define TK_NK_BIN 305 // bin format data 0b111 #define TK_NK_BIN 305 // bin format data 0b111
#define TK_NK_FILE 306
#define TK_NK_BITNOT 501
#define TK_INSERT 502
#define TK_VALUES 507
#define TK_IMPORT 509
#define TK_NK_SEMI 508
#define TK_NK_NIL 65535 #define TK_NK_NIL 65535

2
include/libs/catalog/catalog.h
View File

@ -51,7 +51,7 @@ typedef struct SMetaData {
SArray *pTableMeta; // STableMeta array SArray *pTableMeta; // STableMeta array
SArray *pVgroupInfo; // SVgroupInfo list SArray *pVgroupInfo; // SVgroupInfo list
SArray *pUdfList; // udf info list SArray *pUdfList; // udf info list
SArray *pEpSetList; // qnode epset list, SArray<SEpSet> SArray *pQnodeList; // qnode list, SArray<SQueryNodeAddr>
} SMetaData; } SMetaData;
typedef struct SCatalogCfg { typedef struct SCatalogCfg {

27
include/libs/function/function.h
View File

@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx); typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef struct SScalarFuncExecFuncs { typedef struct SScalarFuncExecFuncs {
@ -141,8 +141,7 @@ struct SResultRowEntryInfo;
//for selectivity query, the corresponding tag value is assigned if the data is qualified //for selectivity query, the corresponding tag value is assigned if the data is qualified
typedef struct SSubsidiaryResInfo { typedef struct SSubsidiaryResInfo {
int16_t bufLen; // keep the tags data for top/bottom query result int16_t num;
int16_t numOfCols;
struct SqlFunctionCtx **pCtx; struct SqlFunctionCtx **pCtx;
} SSubsidiaryResInfo; } SSubsidiaryResInfo;
@ -187,8 +186,8 @@ typedef struct SqlFunctionCtx {
uint8_t currentStage; // record current running step, default: 0 uint8_t currentStage; // record current running step, default: 0
bool isAggSet; bool isAggSet;
int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it
/////////////////////////////////////////////////////////////////
bool stableQuery; bool stableQuery;
/////////////////////////////////////////////////////////////////
int16_t functionId; // function id int16_t functionId; // function id
char * pOutput; // final result output buffer, point to sdata->data char * pOutput; // final result output buffer, point to sdata->data
int32_t numOfParams; int32_t numOfParams;
@ -198,11 +197,15 @@ typedef struct SqlFunctionCtx {
int32_t offset; int32_t offset;
SVariant tag; SVariant tag;
struct SResultRowEntryInfo *resultInfo; struct SResultRowEntryInfo *resultInfo;
SSubsidiaryResInfo subsidiaryRes; SSubsidiaryResInfo subsidiaries;
SPoint1 start; SPoint1 start;
SPoint1 end; SPoint1 end;
SFuncExecFuncs fpSet; SFuncExecFuncs fpSet;
SScalarFuncExecFuncs sfp; SScalarFuncExecFuncs sfp;
SExprInfo *pExpr;
struct SDiskbasedBuf *pBuf;
struct SSDataBlock *pSrcBlock;
int32_t curBufPage;
} SqlFunctionCtx; } SqlFunctionCtx;
enum { enum {
@ -319,6 +322,20 @@ struct SUdfInfo;
void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
/**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
* @return error code
*/
int32_t udfcOpen();
/**
* destroy udfd proxy
* @return error code
*/
int32_t udfcClose();
typedef void *UdfcFuncHandle;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

6
include/libs/function/functionMgt.h
View File

@ -125,15 +125,15 @@ typedef struct SFmGetFuncInfoParam {
struct SCatalog* pCtg; struct SCatalog* pCtg;
void *pRpc; void *pRpc;
const SEpSet* pMgmtEps; const SEpSet* pMgmtEps;
char* pErrBuf;
int32_t errBufLen;
} SFmGetFuncInfoParam; } SFmGetFuncInfoParam;
int32_t fmFuncMgtInit(); int32_t fmFuncMgtInit();
void fmFuncMgtDestroy(); void fmFuncMgtDestroy();
int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); int32_t fmGetFuncInfo(SFmGetFuncInfoParam* pParam, SFunctionNode* pFunc);
int32_t fmGetFuncResultType(SFunctionNode* pFunc, char* pErrBuf, int32_t len);
bool fmIsAggFunc(int32_t funcId); bool fmIsAggFunc(int32_t funcId);
bool fmIsScalarFunc(int32_t funcId); bool fmIsScalarFunc(int32_t funcId);

8
include/libs/nodes/cmdnodes.h
View File

@ -241,12 +241,20 @@ typedef struct SDropComponentNodeStmt {
int32_t dnodeId; int32_t dnodeId;
} SDropComponentNodeStmt; } SDropComponentNodeStmt;
typedef struct STopicOptions {
ENodeType type;
bool withTable;
bool withSchema;
bool withTag;
} STopicOptions;
typedef struct SCreateTopicStmt { typedef struct SCreateTopicStmt {
ENodeType type; ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN]; char topicName[TSDB_TABLE_NAME_LEN];
char subscribeDbName[TSDB_DB_NAME_LEN]; char subscribeDbName[TSDB_DB_NAME_LEN];
bool ignoreExists; bool ignoreExists;
SNode* pQuery; SNode* pQuery;
STopicOptions* pOptions;
} SCreateTopicStmt; } SCreateTopicStmt;
typedef struct SDropTopicStmt { typedef struct SDropTopicStmt {

1
include/libs/nodes/nodes.h
View File

@ -84,6 +84,7 @@ typedef enum ENodeType {
QUERY_NODE_INDEX_OPTIONS, QUERY_NODE_INDEX_OPTIONS,
QUERY_NODE_EXPLAIN_OPTIONS, QUERY_NODE_EXPLAIN_OPTIONS,
QUERY_NODE_STREAM_OPTIONS, QUERY_NODE_STREAM_OPTIONS,
QUERY_NODE_TOPIC_OPTIONS,
// Statement nodes are used in parser and planner module. // Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR, QUERY_NODE_SET_OPERATOR,

7
include/libs/nodes/plannodes.h
View File

@ -46,7 +46,7 @@ typedef struct SScanLogicNode {
struct STableMeta* pMeta; struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList; SVgroupsInfo* pVgroupList;
EScanType scanType; EScanType scanType;
uint8_t scanFlag; // denotes reversed scan of data or not uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange; STimeWindow scanRange;
SName tableName; SName tableName;
bool showRewrite; bool showRewrite;
@ -189,9 +189,6 @@ typedef struct SScanPhysiNode {
SNodeList* pScanCols; SNodeList* pScanCols;
uint64_t uid; // unique id of the table uint64_t uid; // unique id of the table
int8_t tableType; int8_t tableType;
int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC
int32_t count; // repeat count
int32_t reverse; // reverse scan count
SName tableName; SName tableName;
} SScanPhysiNode; } SScanPhysiNode;
@ -207,7 +204,7 @@ typedef struct SSystemTableScanPhysiNode {
typedef struct STableScanPhysiNode { typedef struct STableScanPhysiNode {
SScanPhysiNode scan; SScanPhysiNode scan;
uint8_t scanFlag; // denotes reversed scan of data or not uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count
STimeWindow scanRange; STimeWindow scanRange;
double ratio; double ratio;
int32_t dataRequired; int32_t dataRequired;

10
include/libs/nodes/querynodes.h
View File

@ -46,7 +46,7 @@ typedef struct SExprNode {
ENodeType type; ENodeType type;
SDataType resType; SDataType resType;
char aliasName[TSDB_COL_NAME_LEN]; char aliasName[TSDB_COL_NAME_LEN];
SNodeList* pAssociationList; SArray* pAssociation;
} SExprNode; } SExprNode;
typedef enum EColumnType { typedef enum EColumnType {
@ -81,7 +81,6 @@ typedef struct SValueNode {
char* literal; char* literal;
bool isDuration; bool isDuration;
bool translate; bool translate;
bool genByCalc;
int16_t placeholderNo; int16_t placeholderNo;
union { union {
bool b; bool b;
@ -119,6 +118,7 @@ typedef struct SFunctionNode {
int32_t funcId; int32_t funcId;
int32_t funcType; int32_t funcType;
SNodeList* pParameterList; SNodeList* pParameterList;
int32_t udfBufSize;
} SFunctionNode; } SFunctionNode;
typedef struct STableNode { typedef struct STableNode {
@ -135,7 +135,7 @@ typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta; struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList; SVgroupsInfo* pVgroupList;
char useDbName[TSDB_DB_NAME_LEN]; char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES
double ratio; double ratio;
} SRealTableNode; } SRealTableNode;
@ -237,8 +237,8 @@ typedef struct SSelectStmt {
SNodeList* pGroupByList; // SGroupingSetNode SNodeList* pGroupByList; // SGroupingSetNode
SNode* pHaving; SNode* pHaving;
SNodeList* pOrderByList; // SOrderByExprNode SNodeList* pOrderByList; // SOrderByExprNode
SNode* pLimit; SLimitNode* pLimit;
SNode* pSlimit; SLimitNode* pSlimit;
char stmtName[TSDB_TABLE_NAME_LEN]; char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision; uint8_t precision;
bool isEmptyResult; bool isEmptyResult;

4
include/libs/planner/planner.h
View File

@ -35,6 +35,10 @@ typedef struct SPlanContext {
int8_t triggerType; int8_t triggerType;
int64_t watermark; int64_t watermark;
bool isStmtQuery; bool isStmtQuery;
void* pTransporter;
struct SCatalog* pCatalog;
char* pMsg;
int32_t msgLen;
} SPlanContext; } SPlanContext;
// Create the physical plan for the query, according to the AST. // Create the physical plan for the query, according to the AST.

6
include/libs/qcom/query.h
View File

@ -133,12 +133,6 @@ typedef struct SMsgSendInfo {
SDataBuf msgInfo; SDataBuf msgInfo;
} SMsgSendInfo; } SMsgSendInfo;
typedef struct SQueryNodeAddr {
int32_t nodeId; // vgId or qnodeId
SEpSet epSet;
} SQueryNodeAddr;
typedef struct SQueryNodeStat { typedef struct SQueryNodeStat {
int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
} SQueryNodeStat; } SQueryNodeStat;

128
include/libs/sync/sync.h
View File

@ -20,11 +20,14 @@
extern "C" { extern "C" {
#endif #endif
#include <stdbool.h>
#include <stdint.h> #include <stdint.h>
#include <tdatablock.h> //#include <tdatablock.h>
#include "taosdef.h" #include "cJSON.h"
#include "trpc.h" #include "tdef.h"
#include "wal.h" //#include "taosdef.h"
//#include "trpc.h"
//#include "wal.h"
typedef uint64_t SyncNodeId; typedef uint64_t SyncNodeId;
typedef int32_t SyncGroupId; typedef int32_t SyncGroupId;
@ -38,14 +41,9 @@ typedef enum {
TAOS_SYNC_STATE_ERROR = 103, TAOS_SYNC_STATE_ERROR = 103,
} ESyncState; } ESyncState;
typedef struct SSyncBuffer {
void* data;
size_t len;
} SSyncBuffer;
typedef struct SNodeInfo { typedef struct SNodeInfo {
uint16_t nodePort; // node sync Port uint16_t nodePort;
char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN char nodeFqdn[TSDB_FQDN_LEN];
} SNodeInfo; } SNodeInfo;
typedef struct SSyncCfg { typedef struct SSyncCfg {
@ -54,47 +52,46 @@ typedef struct SSyncCfg {
SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
} SSyncCfg; } SSyncCfg;
typedef struct SNodesRole {
int32_t replicaNum;
SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
ESyncState role[TSDB_MAX_REPLICA];
} SNodesRole;
// abstract definition of snapshot
typedef struct SSnapshot { typedef struct SSnapshot {
void* data; void* data;
SyncIndex lastApplyIndex; SyncIndex lastApplyIndex;
SyncTerm lastApplyTerm;
} SSnapshot; } SSnapshot;
typedef enum {
TAOS_SYNC_FSM_CB_SUCCESS = 0,
TAOS_SYNC_FSM_CB_OTHER_ERROR,
} ESyncFsmCbCode;
typedef struct SFsmCbMeta {
SyncIndex index;
bool isWeak;
int32_t code;
ESyncState state;
uint64_t seqNum;
} SFsmCbMeta;
struct SRpcMsg;
typedef struct SRpcMsg SRpcMsg;
typedef struct SSyncFSM { typedef struct SSyncFSM {
void* data; void* data;
// when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
// user can do something according to the code and isWeak. for example, write data into tsdb void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code, void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
ESyncState state);
// when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
// user can do something according to the code and isWeak. for example, write data into tsdb int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
ESyncState state);
// when log entry is updated by a new one, FpRollBackCb is called
// user can do something to roll back. for example, delete data from tsdb, or just ignore it
void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
ESyncState state);
// user should implement this function, use "data" to take snapshot into "snapshot"
int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
// user should implement this function, restore "data" from "snapshot"
int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
} SSyncFSM; } SSyncFSM;
struct SSyncRaftEntry; struct SSyncRaftEntry;
typedef struct SSyncRaftEntry SSyncRaftEntry; typedef struct SSyncRaftEntry SSyncRaftEntry;
#define SYNC_INDEX_BEGIN 0
#define SYNC_INDEX_INVALID -1
// abstract definition of log store in raft // abstract definition of log store in raft
// SWal implements it // SWal implements it
typedef struct SSyncLogStore { typedef struct SSyncLogStore {
@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
} SSyncLogStore; } SSyncLogStore;
// raft need to persist two variables in storage: currentTerm, voteFor struct SWal;
typedef struct SStateMgr { typedef struct SWal SWal;
void* data;
int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm); struct SEpSet;
int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm); typedef struct SEpSet SEpSet;
int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
} SStateMgr;
typedef struct SSyncInfo { typedef struct SSyncInfo {
SyncGroupId vgId; SyncGroupId vgId;
@ -152,25 +140,33 @@ typedef struct SSyncInfo {
} SSyncInfo; } SSyncInfo;
int32_t syncInit();
void syncCleanUp();
int64_t syncOpen(const SSyncInfo* pSyncInfo);
void syncStart(int64_t rid);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
ESyncState syncGetMyRole(int64_t rid);
const char* syncGetMyRoleStr(int64_t rid);
SyncTerm syncGetMyTerm(int64_t rid);
typedef enum {
TAOS_SYNC_PROPOSE_SUCCESS = 0,
TAOS_SYNC_PROPOSE_NOT_LEADER,
TAOS_SYNC_PROPOSE_OTHER_ERROR,
} ESyncProposeCode;
int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
extern int32_t sDebugFlag;
//-----------------------------------------
struct SSyncNode; struct SSyncNode;
typedef struct SSyncNode SSyncNode; typedef struct SSyncNode SSyncNode;
int32_t syncInit(); struct SSyncBuffer;
void syncCleanUp(); typedef struct SSyncBuffer SSyncBuffer;
//-----------------------------------------
int64_t syncStart(const SSyncInfo* pSyncInfo);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
ESyncState syncGetMyRole(int64_t rid);
// propose with sequence number, to implement linearizable semantics
int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
// for compatibility, the same as syncPropose
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
extern int32_t sDebugFlag;
const char* syncStr(ESyncState state); const char* syncStr(ESyncState state);

414
include/libs/sync/syncTools.h Normal file
View File

@ -0,0 +1,414 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_TOOLS_H
#define _TD_LIBS_SYNC_TOOLS_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
//#include <tdatablock.h>
#include "cJSON.h"
//#include "taosdef.h"
#include "trpc.h"
//#include "wal.h"
// ------------------ ds -------------------
typedef struct SRaftId {
SyncNodeId addr;
SyncGroupId vgId;
} SRaftId;
// ------------------ control -------------------
struct SSyncNode;
typedef struct SSyncNode SSyncNode;
SSyncNode* syncNodeAcquire(int64_t rid);
void syncNodeRelease(SSyncNode* pNode);
int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
void syncSetQ(int64_t rid, void* queueHandle);
void syncSetRpc(int64_t rid, void* rpcHandle);
char* sync2SimpleStr(int64_t rid);
// set timer ms
void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
void setElectTimerMS(int64_t rid, int32_t electTimerMS);
void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
// for compatibility, the same as syncPropose
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
// utils
const char* syncUtilState2String(ESyncState state);
// ------------------ for debug -------------------
void syncRpcMsgPrint(SRpcMsg* pMsg);
void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
void syncRpcMsgLog(SRpcMsg* pMsg);
void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
// ------------------ for compile -------------------
typedef struct SSyncBuffer {
void* data;
size_t len;
} SSyncBuffer;
typedef struct SNodesRole {
int32_t replicaNum;
SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
ESyncState role[TSDB_MAX_REPLICA];
} SNodesRole;
typedef struct SStateMgr {
void* data;
int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
} SStateMgr;
// ------------------ for message process -------------------
// ---------------------------------------------
typedef struct SyncPing {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
uint32_t dataLen;
char data[];
} SyncPing;
SyncPing* syncPingBuild(uint32_t dataLen);
SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
void syncPingDestroy(SyncPing* pMsg);
void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
int32_t syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncPing2Json(const SyncPing* pMsg);
char* syncPing2Str(const SyncPing* pMsg);
// for debug ----------------------
void syncPingPrint(const SyncPing* pMsg);
void syncPingPrint2(char* s, const SyncPing* pMsg);
void syncPingLog(const SyncPing* pMsg);
void syncPingLog2(char* s, const SyncPing* pMsg);
// ---------------------------------------------
typedef struct SyncPingReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
uint32_t dataLen;
char data[];
} SyncPingReply;
SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
void syncPingReplyDestroy(SyncPingReply* pMsg);
void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
int32_t syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncPingReply2Json(const SyncPingReply* pMsg);
char* syncPingReply2Str(const SyncPingReply* pMsg);
// for debug ----------------------
void syncPingReplyPrint(const SyncPingReply* pMsg);
void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
void syncPingReplyLog(const SyncPingReply* pMsg);
void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
// ---------------------------------------------
typedef enum ESyncTimeoutType {
SYNC_TIMEOUT_PING = 100,
SYNC_TIMEOUT_ELECTION,
SYNC_TIMEOUT_HEARTBEAT,
} ESyncTimeoutType;
typedef struct SyncTimeout {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
ESyncTimeoutType timeoutType;
uint64_t logicClock;
int32_t timerMS;
void* data; // need optimized
} SyncTimeout;
SyncTimeout* syncTimeoutBuild();
SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
void* data);
void syncTimeoutDestroy(SyncTimeout* pMsg);
void syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
void syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
void syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncTimeout2Json(const SyncTimeout* pMsg);
char* syncTimeout2Str(const SyncTimeout* pMsg);
// for debug ----------------------
void syncTimeoutPrint(const SyncTimeout* pMsg);
void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
void syncTimeoutLog(const SyncTimeout* pMsg);
void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
// ---------------------------------------------
typedef struct SyncClientRequest {
uint32_t bytes;
int32_t vgId;
uint32_t msgType; // SyncClientRequest msgType
uint32_t originalRpcType; // user RpcMsg msgType
uint64_t seqNum;
bool isWeak;
uint32_t dataLen; // user RpcMsg.contLen
char data[]; // user RpcMsg.pCont
} SyncClientRequest;
SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
int32_t vgId); // step 1
void syncClientRequestDestroy(SyncClientRequest* pMsg);
void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2
void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3
cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg);
char* syncClientRequest2Str(const SyncClientRequest* pMsg);
// for debug ----------------------
void syncClientRequestPrint(const SyncClientRequest* pMsg);
void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
void syncClientRequestLog(const SyncClientRequest* pMsg);
void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
// ---------------------------------------------
typedef struct SyncClientRequestReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
int32_t errCode;
SRaftId leaderHint;
} SyncClientRequestReply;
// ---------------------------------------------
typedef struct SyncRequestVote {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
SyncIndex lastLogIndex;
SyncTerm lastLogTerm;
} SyncRequestVote;
SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
void syncRequestVoteDestroy(SyncRequestVote* pMsg);
void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg);
char* syncRequestVote2Str(const SyncRequestVote* pMsg);
// for debug ----------------------
void syncRequestVotePrint(const SyncRequestVote* pMsg);
void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
void syncRequestVoteLog(const SyncRequestVote* pMsg);
void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
// ---------------------------------------------
typedef struct SyncRequestVoteReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
bool voteGranted;
} SyncRequestVoteReply;
SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
char* syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
// for debug ----------------------
void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
// ---------------------------------------------
typedef struct SyncAppendEntries {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
SyncIndex prevLogIndex;
SyncTerm prevLogTerm;
SyncIndex commitIndex;
uint32_t dataLen;
char data[];
} SyncAppendEntries;
SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
void syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg);
char* syncAppendEntries2Str(const SyncAppendEntries* pMsg);
// for debug ----------------------
void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
// ---------------------------------------------
typedef struct SyncAppendEntriesReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
bool success;
SyncIndex matchIndex;
} SyncAppendEntriesReply;
SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
char* syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
// for debug ----------------------
void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
// ---------------------------------------------
typedef struct SyncApplyMsg {
uint32_t bytes;
int32_t vgId;
uint32_t msgType; // user SyncApplyMsg msgType
uint32_t originalRpcType; // user RpcMsg msgType
SFsmCbMeta fsmMeta;
uint32_t dataLen; // user RpcMsg.contLen
char data[]; // user RpcMsg.pCont
} SyncApplyMsg;
SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
void syncApplyMsgDestroy(SyncApplyMsg* pMsg);
void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg); // SyncApplyMsg to SRpcMsg, put it into ApplyQ
void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg); // get SRpcMsg from ApplyQ, to SyncApplyMsg
SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg); // SyncApplyMsg to OriginalRpcMsg
cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
char* syncApplyMsg2Str(const SyncApplyMsg* pMsg);
// for debug ----------------------
void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
void syncApplyMsgLog(const SyncApplyMsg* pMsg);
void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
// on message ----------------------
int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
// ---------------------------------------------
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_TOOLS_H*/

1
include/libs/transport/trpc.h
View File

@ -43,6 +43,7 @@ typedef struct SRpcMsg {
int32_t code; int32_t code;
void * handle; // rpc handle returned to app void * handle; // rpc handle returned to app
void * ahandle; // app handle set by client void * ahandle; // app handle set by client
int64_t refId; // refid, used by server
int noResp; // has response or not(default 0, 0: resp, 1: no resp); int noResp; // has response or not(default 0, 0: resp, 1: no resp);
int persistHandle; // persist handle or not int persistHandle; // persist handle or not

10
include/libs/wal/wal.h
View File

@ -92,7 +92,7 @@ typedef struct SWalReadHead {
int8_t headVer; int8_t headVer;
int8_t reserved; int8_t reserved;
int16_t msgType; int16_t msgType;
int32_t len; int32_t bodyLen;
int64_t ingestTs; // not implemented int64_t ingestTs; // not implemented
int64_t version; int64_t version;
@ -192,7 +192,13 @@ int32_t walEndSnapshot(SWal *);
SWalReadHandle *walOpenReadHandle(SWal *); SWalReadHandle *walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *); void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
// only for tq usage
// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead);
// deprecated // deprecated
#if 0 #if 0

7
include/os/osSemaphore.h
View File

@ -23,17 +23,24 @@ extern "C" {
#include <semaphore.h> #include <semaphore.h>
#if defined(_TD_DARWIN_64) #if defined(_TD_DARWIN_64)
typedef struct tsem_s *tsem_t; typedef struct tsem_s *tsem_t;
int tsem_init(tsem_t *sem, int pshared, unsigned int value); int tsem_init(tsem_t *sem, int pshared, unsigned int value);
int tsem_wait(tsem_t *sem); int tsem_wait(tsem_t *sem);
int tsem_timewait(tsem_t *sim, int64_t nanosecs);
int tsem_post(tsem_t *sem); int tsem_post(tsem_t *sem);
int tsem_destroy(tsem_t *sem); int tsem_destroy(tsem_t *sem);
#else #else
#define tsem_t sem_t #define tsem_t sem_t
#define tsem_init sem_init #define tsem_init sem_init
int tsem_wait(tsem_t *sem); int tsem_wait(tsem_t *sem);
int tsem_timewait(tsem_t *sim, int64_t nanosecs);
#define tsem_post sem_post #define tsem_post sem_post
#define tsem_destroy sem_destroy #define tsem_destroy sem_destroy
#endif #endif
#if defined(_TD_DARWIN_64) #if defined(_TD_DARWIN_64)

6
include/os/osSystem.h
View File

@ -41,9 +41,9 @@ void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void* handle); void taosCloseDll(void* handle);
int32_t taosSetConsoleEcho(bool on); int32_t taosSetConsoleEcho(bool on);
void setTerminalMode(); void taosSetTerminalMode();
int32_t getOldTerminalMode(); int32_t taosGetOldTerminalMode();
void resetTerminalMode(); void taosResetTerminalMode();
#ifdef __cplusplus #ifdef __cplusplus
} }

13
include/util/taoserror.h
View File

@ -252,7 +252,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE)
// mnode-infoSchema // mnode-infoSchema
#define TSDB_CODE_MND_INVALID_INFOS_TBL TAOS_DEF_ERROR_CODE(0, 0x03B0) #define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x03B0)
// mnode-func // mnode-func
#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0)
@ -411,6 +411,10 @@ int32_t* taosGetErrno();
#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909)
#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A)
// sync integration
#define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x0910)
#define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF)
// tq // tq
#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) #define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00)
#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) #define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01)
@ -569,7 +573,6 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) #define TSDB_CODE_PAR_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602)
#define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) #define TSDB_CODE_PAR_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603)
#define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604) #define TSDB_CODE_PAR_WRONG_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x2604)
#define TSDB_CODE_PAR_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2605)
#define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608) #define TSDB_CODE_PAR_ILLEGAL_USE_AGG_FUNCTION TAOS_DEF_ERROR_CODE(0, 0x2608)
#define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609) #define TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT TAOS_DEF_ERROR_CODE(0, 0x2609)
#define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A)
@ -615,6 +618,11 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632) #define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632)
#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633) #define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633)
#define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL TAOS_DEF_ERROR_CODE(0, 0x2634) #define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL TAOS_DEF_ERROR_CODE(0, 0x2634)
#define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL TAOS_DEF_ERROR_CODE(0, 0x2635)
#define TSDB_CODE_PAR_INVALID_DAYS_VALUE TAOS_DEF_ERROR_CODE(0, 0x2636)
#define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637)
#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638)
#define TSDB_CODE_PAR_INVALID_TOPIC_QUERY TAOS_DEF_ERROR_CODE(0, 0x2639)
//planner //planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700) #define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
@ -624,6 +632,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801) #define TSDB_CODE_FUNC_FUNTION_PARA_NUM TAOS_DEF_ERROR_CODE(0, 0x2801)
#define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802)
#define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803)
#define TSDB_CODE_FUNC_INVALID_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2604)
#ifdef __cplusplus #ifdef __cplusplus
} }

4
include/util/tcache.h
View File

@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem;
/** /**
* initialize the cache object * initialize the cache object
* @param keyType key type * @param keyType key type
* @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired * @param refreshTimeInMs refresh operation interval time, the maximum survival time when one element is expired
* and not referenced by other objects * and not referenced by other objects
* @param extendLifespan auto extend lifespan, if accessed * @param extendLifespan auto extend lifespan, if accessed
* @param fn free resource callback function * @param fn free resource callback function
* @return * @return
*/ */
SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn, SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn,
const char *cacheName); const char *cacheName);
/** /**

5
include/util/tconfig.h
View File

@ -30,6 +30,7 @@ typedef enum {
CFG_STYPE_CFG_FILE, CFG_STYPE_CFG_FILE,
CFG_STYPE_ENV_FILE, CFG_STYPE_ENV_FILE,
CFG_STYPE_ENV_VAR, CFG_STYPE_ENV_VAR,
CFG_STYPE_ENV_CMD,
CFG_STYPE_APOLLO_URL, CFG_STYPE_APOLLO_URL,
CFG_STYPE_ARG_LIST, CFG_STYPE_ARG_LIST,
CFG_STYPE_TAOS_OPTIONS CFG_STYPE_TAOS_OPTIONS
@ -82,7 +83,7 @@ typedef struct SConfig {
} SConfig; } SConfig;
SConfig *cfgInit(); SConfig *cfgInit();
int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const char *sourceStr); int32_t cfgLoad(SConfig *pCfg, ECfgSrcType cfgType, const void *sourceStr);
int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair int32_t cfgLoadFromArray(SConfig *pCfg, SArray *pArgs); // SConfigPair
void cfgCleanup(SConfig *pCfg); void cfgCleanup(SConfig *pCfg);
@ -105,6 +106,8 @@ const char *cfgDtypeStr(ECfgDataType type);
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

3
include/util/tdef.h
View File

@ -76,8 +76,6 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_DEFAULT_PASS "taosdata" #define TSDB_DEFAULT_PASS "taosdata"
#endif #endif
#define SHELL_MAX_PASSWORD_LEN 20
#define TSDB_TRUE 1 #define TSDB_TRUE 1
#define TSDB_FALSE 0 #define TSDB_FALSE 0
#define TSDB_OK 0 #define TSDB_OK 0
@ -118,7 +116,6 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_USER_USERS "user_users"
#define TSDB_INS_TABLE_LICENCES "grants" #define TSDB_INS_TABLE_LICENCES "grants"
#define TSDB_INS_TABLE_VGROUPS "vgroups" #define TSDB_INS_TABLE_VGROUPS "vgroups"
#define TSDB_INS_TABLE_TOPICS "topics"
#define TSDB_INS_TABLE_CONSUMERS "consumers" #define TSDB_INS_TABLE_CONSUMERS "consumers"
#define TSDB_INS_TABLE_SUBSCRIBES "subscribes" #define TSDB_INS_TABLE_SUBSCRIBES "subscribes"
#define TSDB_INS_TABLE_TRANS "trans" #define TSDB_INS_TABLE_TRANS "trans"

12
tools/shell/src/backup/tnettest.h → include/util/tenv.h
View File

@ -1,3 +1,4 @@
/* /*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com> * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
* *
@ -13,17 +14,20 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#ifndef TDENGINE_TNETTEST_H #ifndef _TD_ENV_H_
#define TDENGINE_TNETTEST_H #define _TD_ENV_H_
#include "os.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {
#endif #endif
void taosNetTest(char *role, char *host, int32_t port, int32_t pkgLen, int32_t pkgNum, char *pkgType); int32_t taosEnvNameToCfgName(const char *envNameStr, char *cfgNameStr, int32_t cfgNameMaxLen);
int32_t taosEnvToCfg(const char *envStr, char *cfgStr);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif
#endif // TDENGINE_TNETTEST_H #endif /*_TD_ENV_H_*/

3
include/util/tjson.h
View File

@ -49,6 +49,8 @@ int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem); int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName); SJson* tjsonGetObjectItem(const SJson* pJson, const char* pName);
int32_t tjsonGetObjectName(const SJson* pJson, char** pName);
int32_t tjsonGetObjectValueString(const SJson* pJson, char** pStringValue);
int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal); int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal);
int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal); int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal);
int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal); int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal);
@ -81,6 +83,7 @@ char* tjsonToUnformattedString(const SJson* pJson);
SJson* tjsonParse(const char* pStr); SJson* tjsonParse(const char* pStr);
bool tjsonValidateJson(const char* pJson); bool tjsonValidateJson(const char* pJson);
const char* tjsonGetError();
#ifdef __cplusplus #ifdef __cplusplus
} }

4
include/util/tprocess.h
View File

@ -53,8 +53,8 @@ int32_t taosProcRun(SProcObj *pProc);
void taosProcStop(SProcObj *pProc); void taosProcStop(SProcObj *pProc);
int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
void *handle, EProcFuncType ftype); void *handle, int64_t handleRef, EProcFuncType ftype);
void taosProcRemoveHandle(SProcObj *pProc, void *handle); int64_t taosProcRemoveHandle(SProcObj *pProc, void *handle);
void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle)); void taosProcCloseHandles(SProcObj *pProc, void (*HandleFp)(void *handle));
void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, void taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen,
EProcFuncType ftype); EProcFuncType ftype);

4
include/util/ttimer.h
View File

@ -41,10 +41,6 @@ bool taosTmrReset(TAOS_TMR_CALLBACK fp, int32_t mseconds, void *param, void *han
void taosTmrCleanUp(void *handle); void taosTmrCleanUp(void *handle);
int32_t taosInitTimer(void (*callback)(int32_t), int32_t ms);
void taosUninitTimer();
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

4
source/client/inc/clientInt.h
View File

@ -231,6 +231,10 @@ static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool conver
msg->resIter++; msg->resIter++;
if (msg->resIter < msg->rsp.blockNum) { if (msg->resIter < msg->rsp.blockNum) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter); SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(msg->rsp.blockData, msg->resIter);
if (msg->rsp.withSchema) {
SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(msg->rsp.blockSchema, msg->resIter);
setResSchemaInfo(&msg->resInfo, pSW->pSchema, pSW->nCols);
}
setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4); setQueryResultFromRsp(&msg->resInfo, pRetrieve, convertUcs4);
return &msg->resInfo; return &msg->resInfo;
} }

4
source/client/src/clientEnv.c
View File

@ -254,12 +254,12 @@ void taos_init_imp(void) {
deltaToUtcInitOnce(); deltaToUtcInitOnce();
if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, 1) != 0) { if (taosCreateLog("taoslog", 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) {
tscInitRes = -1; tscInitRes = -1;
return; return;
} }
if (taosInitCfg(configDir, NULL, NULL, NULL, 1) != 0) { if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1) != 0) {
tscInitRes = -1; tscInitRes = -1;
return; return;
} }

8
source/client/src/clientHb.c
View File

@ -14,9 +14,9 @@
*/ */
#include "catalog.h" #include "catalog.h"
#include "scheduler.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "scheduler.h"
#include "trpc.h" #include "trpc.h"
static SClientHbMgr clientHbMgr = {0}; static SClientHbMgr clientHbMgr = {0};
@ -110,7 +110,8 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); SHbConnInfo *info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey));
if (NULL == info) { if (NULL == info) {
tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid, pRsp->connKey.connType); tscWarn("fail to get connInfo, may be dropped, refId:%" PRIx64 ", type:%d", pRsp->connKey.tscRid,
pRsp->connKey.connType);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -641,7 +642,6 @@ void appHbMgrCleanup(void) {
taosHashCleanup(pTarget->activeInfo); taosHashCleanup(pTarget->activeInfo);
pTarget->activeInfo = NULL; pTarget->activeInfo = NULL;
pIter = taosHashIterate(pTarget->connInfo, NULL); pIter = taosHashIterate(pTarget->connInfo, NULL);
while (pIter != NULL) { while (pIter != NULL) {
SHbConnInfo *info = pIter; SHbConnInfo *info = pIter;
@ -668,7 +668,7 @@ int hbMgrInit() {
hbMgrInitHandle(); hbMgrInitHandle();
// init backgroud thread // init backgroud thread
//hbCreateThread(); /*hbCreateThread();*/
return 0; return 0;
} }

54
source/client/src/clientImpl.c
View File

@ -95,19 +95,19 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
if (initEpSetFromCfg(ip, NULL, &epSet) < 0) { if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
return NULL; return NULL;
} }
if (port) {
epSet.epSet.eps[0].port = port;
}
} else { } else {
if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) { if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
return NULL; return NULL;
} }
} }
char* key = getClusterKey(user, secretEncrypt, ip, port); if (port) {
SAppInstInfo** pInst = NULL; epSet.epSet.eps[0].port = port;
}
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL;
taosThreadMutexLock(&appInfo.mutex); taosThreadMutexLock(&appInfo.mutex);
pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
@ -230,10 +230,13 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra
.acctId = pRequest->pTscObj->acctId, .acctId = pRequest->pTscObj->acctId,
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot, .pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite}; .showRewrite = pQuery->showRewrite,
int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList); .pTransporter = pRequest->pTscObj->pAppInfo->pTransporter,
if (code != 0) { .pMsg = pRequest->msgBuf,
return code; .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE};
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (TSDB_CODE_SUCCESS == code) {
code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
} }
return code; return code;
} }
@ -242,6 +245,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t
ASSERT(pSchema != NULL && numOfCols > 0); ASSERT(pSchema != NULL && numOfCols > 0);
pResInfo->numOfCols = numOfCols; pResInfo->numOfCols = numOfCols;
// TODO handle memory leak
pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->fields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD)); pResInfo->userFields = taosMemoryCalloc(numOfCols, sizeof(TAOS_FIELD));
@ -302,8 +306,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
} }
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) { SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
switch (pQuery->execMode) { switch (pQuery->execMode) {
case QUERY_EXEC_MODE_LOCAL: case QUERY_EXEC_MODE_LOCAL:
@ -312,12 +314,15 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
case QUERY_EXEC_MODE_RPC: case QUERY_EXEC_MODE_RPC:
code = execDdlQuery(pRequest, pQuery); code = execDdlQuery(pRequest, pQuery);
break; break;
case QUERY_EXEC_MODE_SCHEDULE: case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
} }
taosArrayDestroy(pNodeList);
break; break;
}
case QUERY_EXEC_MODE_EMPTY_RESULT: case QUERY_EXEC_MODE_EMPTY_RESULT:
pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT; pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
break; break;
@ -326,7 +331,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
} }
} }
taosArrayDestroy(pNodeList);
if (!keepQuery) { if (!keepQuery) {
qDestroyQuery(pQuery); qDestroyQuery(pQuery);
} }
@ -436,7 +440,12 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
return -1; return -1;
} }
taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return terrno;
}
mgmtEpSet->numOfEps++; mgmtEpSet->numOfEps++;
} }
@ -830,10 +839,21 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
return code; return code;
} }
int32_t* colLength = (int32_t*)pResultInfo->pData; char* p = (char*)pResultInfo->pData;
char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols;
int32_t dataLen = *(int32_t*)p;
p += sizeof(int32_t);
uint64_t groupId = *(uint64_t*)p;
p += sizeof(uint64_t);
int32_t* colLength = (int32_t*)p;
p += sizeof(int32_t) * numOfCols;
char* pStart = p;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
colLength[i] = htonl(colLength[i]); colLength[i] = htonl(colLength[i]);
ASSERT(colLength[i] < dataLen);
if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
pResultInfo->pCol[i].offset = (int32_t*)pStart; pResultInfo->pCol[i].offset = (int32_t*)pStart;

2
source/client/src/clientMain.c
View File

@ -120,7 +120,7 @@ const char *taos_errstr(TAOS_RES *res) {
return (const char *)tstrerror(terrno); return (const char *)tstrerror(terrno);
} }
if (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) {
return pRequest->msgBuf; return pRequest->msgBuf;
} else { } else {
return (const char *)tstrerror(pRequest->code); return (const char *)tstrerror(pRequest->code);

23
source/client/src/clientMsgHandler.c
View File

@ -13,13 +13,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h" #include "catalog.h"
#include "tdef.h"
#include "tname.h"
#include "clientInt.h" #include "clientInt.h"
#include "clientLog.h" #include "clientLog.h"
#include "catalog.h" #include "os.h"
#include "query.h" #include "query.h"
#include "tdef.h"
#include "tname.h"
int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
@ -50,7 +50,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SConnectRsp connectRsp = {0}; SConnectRsp connectRsp = {0};
tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp); tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp);
assert(connectRsp.epSet.numOfEps > 0); /*assert(connectRsp.epSet.numOfEps > 0);*/
if (connectRsp.epSet.numOfEps == 0) {
taosMemoryFree(pMsg->pData);
setErrno(pRequest, TSDB_CODE_MND_APP_ERROR);
tsem_post(&pRequest->body.rspSem);
return code;
}
if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) {
updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet); updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet);
@ -93,7 +99,9 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) {
assert(pRequest != NULL); assert(pRequest != NULL);
pMsgSendInfo->msgInfo = pRequest->body.requestMsg; pMsgSendInfo->msgInfo = pRequest->body.requestMsg;
pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)
? genericRspCallback
: handleRequestRspFp[TMSG_INDEX(pRequest->type)];
return pMsgSendInfo; return pMsgSendInfo;
} }
@ -151,13 +159,14 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
taosMemoryFreeClear(output.dbVgroup); taosMemoryFreeClear(output.dbVgroup);
tscError("failed to build use db output since %s", terrstr()); tscError("failed to build use db output since %s", terrstr());
} else { } else if (output.dbVgroup) {
struct SCatalog* pCatalog = NULL; struct SCatalog* pCatalog = NULL;
int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code1 != TSDB_CODE_SUCCESS) { if (code1 != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId, tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
tstrerror(code1)); tstrerror(code1));
taosMemoryFreeClear(output.dbVgroup);
} else { } else {
catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup); catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
} }

334
source/client/src/tmq.c
View File

@ -23,6 +23,9 @@
#include "tmsgtype.h" #include "tmsgtype.h"
#include "tqueue.h" #include "tqueue.h"
#include "tref.h" #include "tref.h"
#include "ttimer.h"
int32_t tmqAskEp(tmq_t* tmq, bool sync);
typedef struct { typedef struct {
int8_t tmqRspType; int8_t tmqRspType;
@ -61,29 +64,40 @@ struct tmq_conf_t {
tmq_commit_cb* commit_cb; tmq_commit_cb* commit_cb;
}; };
typedef struct {
int8_t inited;
tmr_h timer;
} SMqMgmt;
static SMqMgmt tmqMgmt = {0};
struct tmq_t { struct tmq_t {
// conf // conf
char groupId[TSDB_CGROUP_LEN]; char groupId[TSDB_CGROUP_LEN];
char clientId[256]; char clientId[256];
int8_t autoCommit; int8_t autoCommit;
/*int8_t inWaiting;*/
int64_t consumerId; int64_t consumerId;
int32_t epoch;
int32_t resetOffsetCfg; int32_t resetOffsetCfg;
int64_t status;
STscObj* pTscObj;
tmq_commit_cb* commit_cb; tmq_commit_cb* commit_cb;
/*int32_t nextTopicIdx;*/
// status
int8_t status;
int8_t epStatus; int8_t epStatus;
int32_t epoch;
int32_t epSkipCnt; int32_t epSkipCnt;
/*int32_t waitingRequest;*/
/*int32_t readyRequest;*/
SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of tmq_message_t
STaosQall* qall;
tsem_t rspSem;
// stat
int64_t pollCnt; int64_t pollCnt;
// connection
STscObj* pTscObj;
// container
SArray* clientTopics; // SArray<SMqClientTopic>
STaosQueue* mqueue; // queue of rsp
STaosQall* qall;
STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit
// ctl
tsem_t rspSem;
}; };
enum { enum {
@ -93,6 +107,7 @@ enum {
enum { enum {
TMQ_CONSUMER_STATUS__INIT = 0, TMQ_CONSUMER_STATUS__INIT = 0,
TMQ_CONSUMER_STATUS__SUBSCRIBED,
TMQ_CONSUMER_STATUS__READY, TMQ_CONSUMER_STATUS__READY,
}; };
@ -110,13 +125,11 @@ typedef struct {
typedef struct { typedef struct {
// subscribe info // subscribe info
int32_t sqlLen;
char* sql;
char* topicName; char* topicName;
int64_t topicId;
SArray* vgs; // SArray<SMqClientVg> SArray* vgs; // SArray<SMqClientVg>
int8_t isSchemaAdaptive; int8_t isSchemaAdaptive;
int32_t numOfFields;
SSchemaWrapper schema; SSchemaWrapper schema;
} SMqClientTopic; } SMqClientTopic;
@ -156,7 +169,6 @@ typedef struct {
int32_t async; int32_t async;
tsem_t rspSem; tsem_t rspSem;
tmq_resp_err_t rspErr; tmq_resp_err_t rspErr;
/*SMqClientVg* pVg;*/
} SMqCommitCbParam; } SMqCommitCbParam;
tmq_conf_t* tmq_conf_new() { tmq_conf_t* tmq_conf_new() {
@ -251,13 +263,17 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) {
void tmq_list_destroy(tmq_list_t* list) { void tmq_list_destroy(tmq_list_t* list) {
SArray* container = &list->container; SArray* container = &list->container;
/*taosArrayDestroy(container);*/ taosArrayDestroyP(container, taosMemoryFree);
int32_t sz = taosArrayGetSize(container);
for (int32_t i = 0; i < sz; i++) {
char* str = taosArrayGetP(container, i);
taosMemoryFree(str);
} }
taosArrayDestroy(container);
int32_t tmq_list_get_size(const tmq_list_t* list) {
const SArray* container = &list->container;
return taosArrayGetSize(container);
}
char** tmq_list_to_c_array(const tmq_list_t* list) {
const SArray* container = &list->container;
return container->pData;
} }
static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
@ -288,6 +304,8 @@ void tmqClearUnhandleMsg(tmq_t* tmq) {
int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) {
SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param;
pParam->rspErr = code; pParam->rspErr = code;
tmq_t* tmq = pParam->tmq;
atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__SUBSCRIBED);
tsem_post(&pParam->rspSem); tsem_post(&pParam->rspSem);
return 0; return 0;
} }
@ -325,12 +343,9 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
return NULL; return NULL;
} }
pTmq->pTscObj = (STscObj*)conn; pTmq->pTscObj = (STscObj*)conn;
/*pTmq->inWaiting = 0;*/
pTmq->status = 0; pTmq->status = 0;
pTmq->pollCnt = 0; pTmq->pollCnt = 0;
pTmq->epoch = 0; pTmq->epoch = 0;
/*pTmq->waitingRequest = 0;*/
/*pTmq->readyRequest = 0;*/
pTmq->epStatus = 0; pTmq->epStatus = 0;
pTmq->epSkipCnt = 0; pTmq->epSkipCnt = 0;
// set conf // set conf
@ -357,29 +372,45 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs
#endif #endif
tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
// init timer
int8_t inited = atomic_val_compare_exchange_8(&tmqMgmt.inited, 0, 1);
if (inited == 0) {
tmqMgmt.timer = taosTmrInit(1000, 100, 360000, "TMQ");
if (tmqMgmt.timer == NULL) {
atomic_store_8(&tmqMgmt.inited, 0);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
}
tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t)); tmq_t* pTmq = taosMemoryCalloc(1, sizeof(tmq_t));
if (pTmq == NULL) { if (pTmq == NULL) {
return NULL; return NULL;
} }
const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user; const char* user = conf->user == NULL ? TSDB_DEFAULT_USER : conf->user;
const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass; const char* pass = conf->pass == NULL ? TSDB_DEFAULT_PASS : conf->pass;
ASSERT(user); ASSERT(user);
ASSERT(pass); ASSERT(pass);
ASSERT(conf->db);
ASSERT(conf->groupId[0]); ASSERT(conf->groupId[0]);
pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
if (pTmq->pTscObj == NULL) return NULL; pTmq->mqueue = taosOpenQueue();
pTmq->qall = taosAllocateQall();
pTmq->delayedTask = taosOpenQueue();
/*pTmq->inWaiting = 0;*/ if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL) {
pTmq->status = 0; goto FAIL;
}
// init status
pTmq->status = TMQ_CONSUMER_STATUS__INIT;
pTmq->pollCnt = 0; pTmq->pollCnt = 0;
pTmq->epoch = 0; pTmq->epoch = 0;
/*pTmq->waitingRequest = 0;*/
/*pTmq->readyRequest = 0;*/
pTmq->epStatus = 0; pTmq->epStatus = 0;
pTmq->epSkipCnt = 0; pTmq->epSkipCnt = 0;
// set conf // set conf
strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->clientId, conf->clientId);
strcpy(pTmq->groupId, conf->groupId); strcpy(pTmq->groupId, conf->groupId);
@ -387,19 +418,30 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->commit_cb = conf->commit_cb; pTmq->commit_cb = conf->commit_cb;
pTmq->resetOffsetCfg = conf->resetOffset; pTmq->resetOffsetCfg = conf->resetOffset;
pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); // assign consumerId
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); pTmq->consumerId = tGenIdPI64();
if (pTmq->clientTopics == NULL) {
taosMemoryFree(pTmq); // init semaphore
return NULL; if (tsem_init(&pTmq->rspSem, 0, 0) != 0) {
goto FAIL;
} }
pTmq->mqueue = taosOpenQueue(); // init connection
pTmq->qall = taosAllocateQall(); pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, NULL, conf->port, CONN_TYPE__TMQ);
if (pTmq->pTscObj == NULL) {
tsem_init(&pTmq->rspSem, 0, 0); tsem_destroy(&pTmq->rspSem);
goto FAIL;
}
return pTmq; return pTmq;
FAIL:
if (pTmq->clientTopics) taosArrayDestroy(pTmq->clientTopics);
if (pTmq->mqueue) taosCloseQueue(pTmq->mqueue);
if (pTmq->delayedTask) taosCloseQueue(pTmq->delayedTask);
if (pTmq->qall) taosFreeQall(pTmq->qall);
taosMemoryFree(pTmq);
return NULL;
} }
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) { tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, int32_t async) {
@ -490,81 +532,64 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets, in
return resp; return resp;
} }
tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tmq_resp_err_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) {
SRequestObj* pRequest = NULL; const SArray* container = &topic_list->container;
SArray* container = &topic_list->container;
int32_t sz = taosArrayGetSize(container); int32_t sz = taosArrayGetSize(container);
// destroy ex void* buf = NULL;
taosArrayDestroy(tmq->clientTopics); SCMSubscribeReq req = {0};
tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); int32_t code = -1;
SCMSubscribeReq req;
req.topicNum = sz;
req.consumerId = tmq->consumerId; req.consumerId = tmq->consumerId;
strcpy(req.cgroup, tmq->groupId); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN);
req.topicNames = taosArrayInit(sz, sizeof(void*)); req.topicNames = taosArrayInit(sz, sizeof(void*));
if (req.topicNames == NULL) goto FAIL;
for (int i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
/*char* topicName = topic_list->elems[i];*/ char* topic = taosArrayGetP(container, i);
char* topicName = taosArrayGetP(container, i);
SName name = {0}; SName name = {0};
char* dbName = getDbOfConnection(tmq->pTscObj); tNameSetDbName(&name, tmq->pTscObj->acctId, topic, strlen(topic));
if (dbName == NULL) {
return TMQ_RESP_ERR__FAIL;
}
tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName));
tNameFromString(&name, topicName, T_NAME_TABLE);
char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); char* topicFName = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
if (topicFname == NULL) { if (topicFName == NULL) {
goto _return; goto FAIL;
} }
tNameExtractFullName(&name, topicFname); tNameExtractFullName(&name, topicFName);
tscDebug("subscribe topic: %s", topicFname);
SMqClientTopic topic = { tscDebug("subscribe topic: %s", topicFName);
.sql = NULL,
.sqlLen = 0, taosArrayPush(req.topicNames, &topicFName);
.topicId = 0,
.topicName = topicFname,
.vgs = NULL,
};
topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
taosArrayPush(tmq->clientTopics, &topic);
taosArrayPush(req.topicNames, &topicFname);
taosMemoryFree(dbName);
} }
int tlen = tSerializeSCMSubscribeReq(NULL, &req); int32_t tlen = tSerializeSCMSubscribeReq(NULL, &req);
void* buf = taosMemoryMalloc(tlen); buf = taosMemoryMalloc(tlen);
if (buf == NULL) { if (buf == NULL) goto FAIL;
goto _return;
}
void* abuf = buf; void* abuf = buf;
tSerializeSCMSubscribeReq(&abuf, &req); tSerializeSCMSubscribeReq(&abuf, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); SMsgSendInfo* sendInfo = taosMemoryMalloc(sizeof(SMsgSendInfo));
if (pRequest == NULL) { if (sendInfo == NULL) goto FAIL;
tscError("failed to malloc request");
}
SMqSubscribeCbParam param = { SMqSubscribeCbParam param = {
.rspErr = TMQ_RESP_ERR__SUCCESS, .rspErr = TMQ_RESP_ERR__SUCCESS,
.tmq = tmq, .tmq = tmq,
}; };
tsem_init(&param.rspSem, 0, 0);
pRequest->body.requestMsg = (SDataBuf){ if (tsem_init(&param.rspSem, 0, 0) != 0) goto FAIL;
sendInfo->msgInfo = (SDataBuf){
.pData = buf, .pData = buf,
.len = tlen, .len = tlen,
.handle = NULL, .handle = NULL,
}; };
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); sendInfo->requestId = generateRequestId();
sendInfo->requestObjRefId = 0;
sendInfo->param = &param; sendInfo->param = &param;
sendInfo->fp = tmqSubscribeCb; sendInfo->fp = tmqSubscribeCb;
sendInfo->msgType = TDMT_MND_SUBSCRIBE;
SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0; int64_t transporterId = 0;
@ -573,15 +598,28 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
tsem_wait(&param.rspSem); tsem_wait(&param.rspSem);
tsem_destroy(&param.rspSem); tsem_destroy(&param.rspSem);
_return: code = param.rspErr;
/*if (sendInfo != NULL) {*/ if (code != 0) goto FAIL;
/*destroySendMsgInfo(sendInfo);*/
/*}*/
return param.rspErr; // TODO: add max retry cnt
while (TSDB_CODE_MND_CONSUMER_NOT_READY == tmqAskEp(tmq, true)) {
tscDebug("not ready, retry\n");
taosMsleep(500);
} }
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { conf->commit_cb = cb; } code = 0;
FAIL:
if (req.topicNames != NULL) taosArrayDestroyP(req.topicNames, taosMemoryFree);
if (code != 0) {
taosMemoryFree(buf);
}
return code;
}
void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) {
//
conf->commit_cb = cb;
}
TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) { TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbName, const char* sql) {
STscObj* pTscObj = (STscObj*)taos; STscObj* pTscObj = (STscObj*)taos;
@ -615,9 +653,6 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
int32_t code = 0; int32_t code = 0;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return); CHECK_CODE_GOTO(parseSql(pRequest, false, &pQueryNode, NULL), _return);
// todo check for invalid sql statement and return with error code
CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return); CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
/*printf("%s\n", pStr);*/ /*printf("%s\n", pStr);*/
@ -641,7 +676,6 @@ TAOS_RES* tmq_create_stream(TAOS* taos, const char* streamName, const char* tbNa
} }
tSerializeSCMCreateStreamReq(buf, tlen, &req); tSerializeSCMCreateStreamReq(buf, tlen, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest->body.requestMsg = (SDataBuf){ pRequest->body.requestMsg = (SDataBuf){
.pData = buf, .pData = buf,
@ -672,94 +706,6 @@ _return:
return pRequest; return pRequest;
} }
#if 0
TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos;
SRequestObj* pRequest = NULL;
SQuery* pQueryNode = NULL;
char* astStr = NULL;
terrno = TSDB_CODE_SUCCESS;
if (taos == NULL || topicName == NULL || sql == NULL) {
tscError("invalid parameters for creating topic, connObj:%p, topic name:%s, sql:%s", taos, topicName, sql);
terrno = TSDB_CODE_TSC_INVALID_INPUT;
goto _return;
}
if (strlen(topicName) >= TSDB_TOPIC_NAME_LEN) {
tscError("topic name too long, max length:%d", TSDB_TOPIC_NAME_LEN - 1);
terrno = TSDB_CODE_TSC_INVALID_INPUT;
goto _return;
}
if (sqlLen > TSDB_MAX_ALLOWED_SQL_LEN) {
tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN);
terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT;
goto _return;
}
tscDebug("start to create topic: %s", topicName);
int32_t code = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, true, &pQueryNode), _return);
// todo check for invalid sql statement and return with error code
CHECK_CODE_GOTO(nodesNodeToString(pQueryNode->pRoot, false, &astStr, NULL), _return);
/*printf("%s\n", pStr);*/
SName name = {.acctId = pTscObj->acctId, .type = TSDB_TABLE_NAME_T};
strcpy(name.dbname, pRequest->pDb);
strcpy(name.tname, topicName);
SCMCreateTopicReq req = {
.igExists = 1,
.ast = astStr,
.sql = (char*)sql,
};
tNameExtractFullName(&name, req.name);
int tlen = tSerializeSCMCreateTopicReq(NULL, 0, &req);
void* buf = taosMemoryMalloc(tlen);
if (buf == NULL) {
goto _return;
}
tSerializeSCMCreateTopicReq(buf, tlen, &req);
/*printf("formatted: %s\n", dagStr);*/
pRequest->body.requestMsg = (SDataBuf){
.pData = buf,
.len = tlen,
.handle = NULL,
};
pRequest->type = TDMT_MND_CREATE_TOPIC;
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
SEpSet epSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo);
tsem_wait(&pRequest->body.rspSem);
_return:
taosMemoryFreeClear(astStr);
qDestroyQuery(pQueryNode);
/*if (sendInfo != NULL) {*/
/*destroySendMsgInfo(sendInfo);*/
/*}*/
if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
pRequest->code = terrno;
}
return pRequest;
}
#endif
#if 0 #if 0
int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return 0; if (tmq_message == NULL) return 0;
@ -942,7 +888,7 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
/*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/ /*printf("rsp epoch %ld sz %ld\n", rsp.epoch, rsp.topics->size);*/
/*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/ /*printf("tmq epoch %ld sz %ld\n", tmq->epoch, tmq->clientTopics->size);*/
if (tmqUpdateEp(tmq, head->epoch, &rsp)) { if (tmqUpdateEp(tmq, head->epoch, &rsp)) {
atomic_store_64(&tmq->status, TMQ_CONSUMER_STATUS__READY); atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY);
} }
tDeleteSMqCMGetSubEpRsp(&rsp); tDeleteSMqCMGetSubEpRsp(&rsp);
} else { } else {
@ -1106,7 +1052,9 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.totalRows = 0;
pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
if (!pWrapper->msg.withSchema) {
setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
}
taosFreeQitem(pWrapper); taosFreeQitem(pWrapper);
return pRspObj; return pRspObj;
@ -1175,7 +1123,6 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
int64_t transporterId = 0; int64_t transporterId = 0;
/*printf("send poll\n");*/ /*printf("send poll\n");*/
/*atomic_add_fetch_32(&tmq->waitingRequest, 1);*/
tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId,
pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId);
/*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
@ -1254,12 +1201,14 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
SMqRspObj* rspObj; SMqRspObj* rspObj;
int64_t startTime = taosGetTimestampMs(); int64_t startTime = taosGetTimestampMs();
// TODO: put into another thread or delayed queue // TODO: put into delayed queue
int64_t status = atomic_load_64(&tmq->status); #if 0
while (0 != tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT)) { int8_t status = atomic_load_8(&tmq->status);
while (0 != tmqAskEp(tmq, status != TMQ_CONSUMER_STATUS__READY)) {
tscDebug("not ready, retry\n"); tscDebug("not ready, retry\n");
taosSsleep(1); taosSsleep(1);
} }
#endif
rspObj = tmqHandleAllRsp(tmq, blocking_time, false); rspObj = tmqHandleAllRsp(tmq, blocking_time, false);
if (rspObj) { if (rspObj) {
@ -1267,7 +1216,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
} }
while (1) { while (1) {
/*printf("cycle\n");*/
tmqAskEp(tmq, false); tmqAskEp(tmq, false);
tmqPollImpl(tmq, blocking_time); tmqPollImpl(tmq, blocking_time);

6
source/common/src/tdatablock.c
View File

@ -31,7 +31,6 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
if (pEp->port == 0) { if (pEp->port == 0) {
pEp->port = tsServerPort; pEp->port = tsServerPort;
return -1;
} }
return 0; return 0;
@ -645,12 +644,12 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
/** /**
* @refitem blockDataToBuf for the meta size * @refitem blockDataToBuf for the meta size
*
* @param pBlock * @param pBlock
* @return * @return
*/ */
size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t); // | total rows/total length | block group id | each column length |
return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t);
} }
double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@ -1220,7 +1219,6 @@ void colDataDestroy(SColumnInfoData* pColData) {
taosMemoryFree(pColData->pData); taosMemoryFree(pColData->pData);
} }
static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
int32_t len = BitmapLen(total); int32_t len = BitmapLen(total);

69
source/common/src/tglobal.c
View File

@ -32,13 +32,9 @@ int32_t tsVersion = 30000000;
int32_t tsStatusInterval = 1; // second int32_t tsStatusInterval = 1; // second
// common // common
int32_t tsRpcTimer = 300;
int32_t tsRpcMaxTime = 600; // seconds;
bool tsRpcForceTcp = true; // disable this, means query, show command use udp protocol as default
int32_t tsMaxShellConns = 50000; int32_t tsMaxShellConns = 50000;
int32_t tsMaxConnections = 50000; int32_t tsMaxConnections = 50000;
int32_t tsShellActivityTimer = 3; // second int32_t tsShellActivityTimer = 3; // second
int32_t tsMaxBinaryDisplayWidth = 30;
bool tsEnableSlaveQuery = true; bool tsEnableSlaveQuery = true;
bool tsPrintAuth = false; bool tsPrintAuth = false;
@ -80,6 +76,9 @@ int32_t tsTelemInterval = 86400;
char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com"; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com";
uint16_t tsTelemPort = 80; uint16_t tsTelemPort = 80;
// query
int32_t tsQueryPolicy = 1;
/* /*
* denote if the server needs to compress response message at the application layer to client, including query rsp, * denote if the server needs to compress response message at the application layer to client, including query rsp,
* metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server. * metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@ -102,14 +101,6 @@ int32_t tsCompressColData = -1;
*/ */
int32_t tsCompatibleModel = 1; int32_t tsCompatibleModel = 1;
// client
int32_t tsMaxWildCardsLen = TSDB_PATTERN_STRING_DEFAULT_LEN;
int32_t tsMaxRegexStringLen = TSDB_REGEX_STRING_DEFAULT_LEN;
// the maximum number of results for projection query on super table that are returned from
// one virtual node, to order according to timestamp
int32_t tsMaxNumOfOrderedResults = 100000;
// 10 ms for sliding time, the value will changed in case of time precision changed // 10 ms for sliding time, the value will changed in case of time precision changed
int32_t tsMinSlidingTime = 10; int32_t tsMinSlidingTime = 10;
@ -229,23 +220,27 @@ struct SConfig *taosGetCfg() {
return tsCfg; return tsCfg;
} }
static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *envFile, const char *apolloUrl) { static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) {
char cfgDir[PATH_MAX] = {0}; char cfgDir[PATH_MAX] = {0};
char cfgFile[PATH_MAX + 100] = {0}; char cfgFile[PATH_MAX + 100] = {0};
taosExpandDir(inputCfgDir, cfgDir, PATH_MAX); taosExpandDir(inputCfgDir, cfgDir, PATH_MAX);
if (taosIsDir(cfgDir)) {
snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir); snprintf(cfgFile, sizeof(cfgFile), "%s" TD_DIRSEP "taos.cfg", cfgDir);
} else {
tstrncpy(cfgFile, cfgDir, sizeof(cfgDir));
}
if (apolloUrl == NULL || apolloUrl[0] == '\0') cfgGetApollUrl(envCmd, envFile, apolloUrl);
if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) { if (cfgLoad(pCfg, CFG_STYPE_APOLLO_URL, apolloUrl) != 0) {
uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr()); uError("failed to load from apollo url:%s since %s", apolloUrl, terrstr());
return -1; return -1;
} }
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgDir) != 0) {
if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) { if (cfgLoad(pCfg, CFG_STYPE_CFG_FILE, cfgFile) != 0) {
uInfo("cfg file:%s not read since %s", cfgFile, terrstr()); uError("failed to load from cfg file:%s since %s", cfgFile, terrstr());
return 0; return -1;
}
} }
if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) { if (cfgLoad(pCfg, CFG_STYPE_ENV_FILE, envFile) != 0) {
@ -258,6 +253,11 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char *inputCfgDir, const char *e
return -1; return -1;
} }
if (cfgLoad(pCfg, CFG_STYPE_ENV_CMD, envCmd) != 0) {
uError("failed to load from cmd env variables since %s", terrstr());
return -1;
}
return 0; return 0;
} }
@ -309,19 +309,11 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1; if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, 1) != 0) return -1;
if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1; if (cfgAddDir(pCfg, "tempDir", tsTempDir, 1) != 0) return -1;
if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1; if (cfgAddFloat(pCfg, "minimalTempDirGB", 1.0f, 0.001f, 10000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxTmrCtrl", tsMaxTmrCtrl, 8, 2048, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcTimer", tsRpcTimer, 100, 3000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "rpcMaxTime", tsRpcMaxTime, 100, 7200, 1) != 0) return -1;
if (cfgAddBool(pCfg, "rpcForceTcp", tsRpcForceTcp, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1; if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxWildCardsLength", tsMaxWildCardsLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxRegexStringLen", tsMaxRegexStringLen, 0, TSDB_MAX_FIELD_LEN, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxNumOfOrderedRes", tsMaxNumOfOrderedResults, 128, TSDB_MAX_ALLOWED_SQL_LEN, 1) != 0)
return -1;
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 4; tsNumOfTaskQueueThreads = tsNumOfCores / 4;
tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2); tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2);
@ -340,8 +332,8 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1; if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 0, 100000, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "pageSize(KB)", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddInt64(pCfg, "totalMemory(KB)", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1; if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, 1) != 0) return -1;
if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1; if (cfgAddString(pCfg, "os sysname", info.sysname, 1) != 0) return -1;
if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1; if (cfgAddString(pCfg, "os nodename", info.nodename, 1) != 0) return -1;
if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1; if (cfgAddString(pCfg, "os release", info.release, 1) != 0) return -1;
@ -502,19 +494,12 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
return -1; return -1;
} }
tsMaxTmrCtrl = cfgGetItem(pCfg, "maxTmrCtrl")->i32;
tsRpcTimer = cfgGetItem(pCfg, "rpcTimer")->i32;
tsRpcMaxTime = cfgGetItem(pCfg, "rpcMaxTime")->i32;
tsRpcForceTcp = cfgGetItem(pCfg, "rpcForceTcp")->i32;
tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32;
tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32;
tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32;
tsMaxWildCardsLen = cfgGetItem(pCfg, "maxWildCardsLength")->i32;
tsMaxRegexStringLen = cfgGetItem(pCfg, "maxRegexStringLen")->i32;
tsMaxNumOfOrderedResults = cfgGetItem(pCfg, "maxNumOfOrderedRes")->i32;
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
return 0; return 0;
} }
@ -594,8 +579,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
return 0; return 0;
} }
int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char *envFile, int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile,
const char *apolloUrl, SArray *pArgs, bool tsc) { char *apolloUrl, SArray *pArgs, bool tsc) {
osDefaultInit(); osDefaultInit();
SConfig *pCfg = cfgInit(); SConfig *pCfg = cfgInit();
@ -610,7 +595,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
if (taosAddServerLogCfg(pCfg) != 0) return -1; if (taosAddServerLogCfg(pCfg) != 0) return -1;
} }
if (taosLoadCfg(pCfg, cfgDir, envFile, apolloUrl) != 0) { if (taosLoadCfg(pCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr()); uError("failed to load cfg since %s", terrstr());
cfgCleanup(pCfg); cfgCleanup(pCfg);
return -1; return -1;
@ -647,7 +632,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
return 0; return 0;
} }
int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloUrl, SArray *pArgs, bool tsc) { int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) {
if (tsCfg != NULL) return 0; if (tsCfg != NULL) return 0;
tsCfg = cfgInit(); tsCfg = cfgInit();
@ -662,7 +647,7 @@ int32_t taosInitCfg(const char *cfgDir, const char *envFile, const char *apolloU
} }
taosAddSystemCfg(tsCfg); taosAddSystemCfg(tsCfg);
if (taosLoadCfg(tsCfg, cfgDir, envFile, apolloUrl) != 0) { if (taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl) != 0) {
uError("failed to load cfg since %s", terrstr()); uError("failed to load cfg since %s", terrstr());
cfgCleanup(tsCfg); cfgCleanup(tsCfg);
tsCfg = NULL; tsCfg = NULL;
@ -703,6 +688,6 @@ void taosCfgDynamicOptions(const char *option, const char *value) {
if (strcasecmp(option, "resetlog") == 0) { if (strcasecmp(option, "resetlog") == 0) {
taosResetLog(); taosResetLog();
cfgDumpCfg(tsCfg, 1, false); cfgDumpCfg(tsCfg, 0, false);
} }
} }

133
source/common/src/tmsg.c
View File

@ -93,7 +93,87 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
return row; return row;
} }
} }
#if 0
// TODO: KEEP one suite of iterator API finally.
// 1) use tInitSubmitMsgIterEx firstly as not decrease the merge conflicts
// 2) replace tInitSubmitMsgIterEx with tInitSubmitMsgIter later
// 3) finally, rename tInitSubmitMsgIterEx to tInitSubmitMsgIter
int32_t tInitSubmitMsgIterEx(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
if (pMsg == NULL) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return -1;
}
pIter->totalLen = htonl(pMsg->length);
ASSERT(pIter->totalLen > 0);
pIter->len = 0;
pIter->pMsg = pMsg;
if (pIter->totalLen <= sizeof(SSubmitReq)) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
return -1;
}
return 0;
}
int32_t tGetSubmitMsgNextEx(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
ASSERT(pIter->len >= 0);
if (pIter->len == 0) {
pIter->len += sizeof(SSubmitReq);
} else {
if (pIter->len >= pIter->totalLen) {
ASSERT(0);
}
SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
pIter->len += (sizeof(SSubmitBlk) + pIter->dataLen + pIter->schemaLen);
ASSERT(pIter->len > 0);
}
if (pIter->len > pIter->totalLen) {
terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP;
*pPBlock = NULL;
return -1;
}
if (pIter->len == pIter->totalLen) {
*pPBlock = NULL;
} else {
*pPBlock = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
pIter->uid = htobe64((*pPBlock)->uid);
pIter->suid = htobe64((*pPBlock)->suid);
pIter->sversion = htonl((*pPBlock)->sversion);
pIter->dataLen = htonl((*pPBlock)->dataLen);
pIter->schemaLen = htonl((*pPBlock)->schemaLen);
pIter->numOfRows = htons((*pPBlock)->numOfRows);
}
return 0;
}
int32_t tInitSubmitBlkIterEx(SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
if (pMsgIter->dataLen <= 0) return -1;
pIter->totalLen = pMsgIter->dataLen;
pIter->len = 0;
pIter->row = (STSRow *)(pBlock->data + pMsgIter->schemaLen);
return 0;
}
STSRow *tGetSubmitBlkNextEx(SSubmitBlkIter *pIter) {
STSRow *row = pIter->row;
if (pIter->len >= pIter->totalLen) {
return NULL;
} else {
pIter->len += TD_ROW_LEN(row);
if (pIter->len < pIter->totalLen) {
pIter->row = POINTER_SHIFT(row, TD_ROW_LEN(row));
}
return row;
}
}
#endif
int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) { int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) {
if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1;
if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1;
@ -114,6 +194,18 @@ int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) {
return 0; return 0;
} }
int32_t tEncodeSQueryNodeAddr(SCoder *pEncoder, SQueryNodeAddr *pAddr) {
if (tEncodeI32(pEncoder, pAddr->nodeId) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pAddr->epSet) < 0) return -1;
return 0;
}
int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) {
if (tDecodeI32(pDecoder, &pAddr->nodeId) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pAddr->epSet) < 0) return -1;
return 0;
}
int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pEp->inUse); tlen += taosEncodeFixedI8(buf, pEp->inUse);
@ -421,7 +513,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1;
if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1; if (tEncodeFloat(&encoder, pReq->xFilesFactor) < 0) return -1;
if (tEncodeI32(&encoder, pReq->aggregationMethod) < 0) return -1;
if (tEncodeI32(&encoder, pReq->delay) < 0) return -1; if (tEncodeI32(&encoder, pReq->delay) < 0) return -1;
if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1; if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
@ -476,7 +567,6 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1;
if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1; if (tDecodeFloat(&decoder, &pReq->xFilesFactor) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->aggregationMethod) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1; if (tDecodeI32(&decoder, &pReq->delay) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1; if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
@ -1356,13 +1446,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1; if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1;
if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; if (tEncodeI64(&encoder, pReq->signature) < 0) return -1;
int32_t codeSize = 0;
if (pReq->pCode != NULL) { if (pReq->pCode != NULL) {
codeSize = strlen(pReq->pCode) + 1; if (tEncodeBinary(&encoder, pReq->pCode, pReq->codeLen) < 0) return -1;
}
if (tEncodeI32(&encoder, codeSize) < 0) return -1;
if (pReq->pCode != NULL) {
if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1;
} }
int32_t commentSize = 0; int32_t commentSize = 0;
@ -1396,10 +1481,8 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1; if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1;
int32_t codeSize = 0; if (pReq->codeLen > 0) {
if (tDecodeI32(&decoder, &codeSize) < 0) return -1; pReq->pCode = taosMemoryCalloc(1, pReq->codeLen);
if (codeSize > 0) {
pReq->pCode = taosMemoryCalloc(1, codeSize);
if (pReq->pCode == NULL) { if (pReq->pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
@ -1522,7 +1605,7 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *
if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1;
if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1;
if (pInfo->codeSize) { if (pInfo->codeSize) {
if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; if (tEncodeBinary(&encoder, pInfo->pCode, pInfo->codeSize) < 0) return -1;
} }
if (pInfo->commentSize) { if (pInfo->commentSize) {
if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;
@ -1859,11 +1942,11 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp)
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1; if (tStartEncode(&encoder) < 0) return -1;
int32_t num = taosArrayGetSize(pRsp->epSetList); int32_t num = taosArrayGetSize(pRsp->addrsList);
if (tEncodeI32(&encoder, num) < 0) return -1; if (tEncodeI32(&encoder, num) < 0) return -1;
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SEpSet *epSet = taosArrayGet(pRsp->epSetList, i); SQueryNodeAddr *addr = taosArrayGet(pRsp->addrsList, i);
if (tEncodeSEpSet(&encoder, epSet) < 0) return -1; if (tEncodeSQueryNodeAddr(&encoder, addr) < 0) return -1;
} }
tEndEncode(&encoder); tEndEncode(&encoder);
@ -1879,10 +1962,15 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
if (tStartDecode(&decoder) < 0) return -1; if (tStartDecode(&decoder) < 0) return -1;
int32_t num = 0; int32_t num = 0;
if (tDecodeI32(&decoder, &num) < 0) return -1; if (tDecodeI32(&decoder, &num) < 0) return -1;
pRsp->epSetList = taosArrayInit(num, sizeof(SEpSet)); if (NULL == pRsp->addrsList) {
if (NULL == pRsp->epSetList) return -1; pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
if (NULL == pRsp->addrsList) return -1;
}
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
if (tDecodeSEpSet(&decoder, TARRAY_GET_ELEM(pRsp->epSetList, i)) < 0) return -1; SQueryNodeAddr addr = {0};
if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1;
taosArrayPush(pRsp->addrsList, &addr);
} }
tEndDecode(&decoder); tEndDecode(&decoder);
@ -1890,7 +1978,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
return 0; return 0;
} }
void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); } void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->addrsList); }
int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
SCoder encoder = {0}; SCoder encoder = {0};
@ -2535,7 +2623,7 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1; if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1; if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withTagSchema) < 0) return -1; if (tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1;
if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
if (tEncodeI32(&encoder, astLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1;
if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
@ -2561,7 +2649,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withTagSchema) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1;
if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
if (tDecodeI32(&decoder, &astLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1;
@ -2575,6 +2663,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
pReq->ast = taosMemoryCalloc(1, astLen + 1); pReq->ast = taosMemoryCalloc(1, astLen + 1);
if (pReq->ast == NULL) return -1; if (pReq->ast == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
} else {
} }
tEndDecode(&decoder); tEndDecode(&decoder);

2
source/common/src/ttime.c
View File

@ -412,7 +412,7 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) { int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) {
int32_t charLen = varDataLen(inputData); int32_t charLen = varDataLen(inputData);
char *newColData; char *newColData;
if (type == TSDB_DATA_TYPE_BINARY) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
newColData = taosMemoryCalloc(1, charLen + 1); newColData = taosMemoryCalloc(1, charLen + 1);
memcpy(newColData, varDataVal(inputData), charLen); memcpy(newColData, varDataVal(inputData), charLen);
bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight); bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);

28
source/dnode/mgmt/exe/dmMain.c
View File

@ -24,6 +24,7 @@ static struct {
bool printVersion; bool printVersion;
char envFile[PATH_MAX]; char envFile[PATH_MAX];
char apolloUrl[PATH_MAX]; char apolloUrl[PATH_MAX];
const char **envCmd;
SArray *pArgs; // SConfigPair SArray *pArgs; // SConfigPair
SDnode *pDnode; SDnode *pDnode;
EDndNodeType ntype; EDndNodeType ntype;
@ -56,6 +57,9 @@ static void dmSetSignalHandle() {
} }
static int32_t dmParseArgs(int32_t argc, char const *argv[]) { static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
int32_t cmdEnvIndex = 0;
global.envCmd = taosMemoryMalloc(argc-1);
memset(global.envCmd, 0, argc-1);
for (int32_t i = 1; i < argc; ++i) { for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) { if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) { if (i < argc - 1) {
@ -70,7 +74,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
} }
} else if (strcmp(argv[i], "-a") == 0) { } else if (strcmp(argv[i], "-a") == 0) {
tstrncpy(global.apolloUrl, argv[++i], PATH_MAX); tstrncpy(global.apolloUrl, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-e") == 0) { } else if (strcmp(argv[i], "-E") == 0) {
tstrncpy(global.envFile, argv[++i], PATH_MAX); tstrncpy(global.envFile, argv[++i], PATH_MAX);
} else if (strcmp(argv[i], "-n") == 0) { } else if (strcmp(argv[i], "-n") == 0) {
global.ntype = atoi(argv[++i]); global.ntype = atoi(argv[++i]);
@ -84,6 +88,9 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
global.dumpConfig = true; global.dumpConfig = true;
} else if (strcmp(argv[i], "-V") == 0) { } else if (strcmp(argv[i], "-V") == 0) {
global.printVersion = true; global.printVersion = true;
} else if (strcmp(argv[i], "-e") == 0) {
global.envCmd[cmdEnvIndex] = argv[++i];
cmdEnvIndex++;
} else { } else {
} }
} }
@ -106,7 +113,7 @@ static void dmPrintVersion() {
static void dmDumpCfg() { static void dmDumpCfg() {
SConfig *pCfg = taosGetCfg(); SConfig *pCfg = taosGetCfg();
cfgDumpCfg(pCfg, 0, 1); cfgDumpCfg(pCfg, 0, true);
} }
static SDnodeOpt dmGetOpt() { static SDnodeOpt dmGetOpt() {
@ -129,7 +136,7 @@ static SDnodeOpt dmGetOpt() {
static int32_t dmInitLog() { static int32_t dmInitLog() {
char logName[12] = {0}; char logName[12] = {0};
snprintf(logName, sizeof(logName), "%slog", dmLogName(global.ntype)); snprintf(logName, sizeof(logName), "%slog", dmLogName(global.ntype));
return taosCreateLog(logName, 1, configDir, global.envFile, global.apolloUrl, global.pArgs, 0); return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0);
} }
static void dmSetProcInfo(int32_t argc, char **argv) { static void dmSetProcInfo(int32_t argc, char **argv) {
@ -168,6 +175,10 @@ static int32_t dmRunDnode() {
return code; return code;
} }
static void taosCleanupArgs() {
if (global.envCmd != NULL) taosMemoryFree(global.envCmd);
}
int main(int argc, char const *argv[]) { int main(int argc, char const *argv[]) {
if (!taosCheckSystemIsSmallEnd()) { if (!taosCheckSystemIsSmallEnd()) {
printf("failed to start since on non-small-end machines\n"); printf("failed to start since on non-small-end machines\n");
@ -176,26 +187,31 @@ int main(int argc, char const *argv[]) {
if (dmParseArgs(argc, argv) != 0) { if (dmParseArgs(argc, argv) != 0) {
printf("failed to start since parse args error\n"); printf("failed to start since parse args error\n");
taosCleanupArgs();
return -1; return -1;
} }
if (global.generateGrant) { if (global.generateGrant) {
dmGenerateGrant(); dmGenerateGrant();
taosCleanupArgs();
return 0; return 0;
} }
if (global.printVersion) { if (global.printVersion) {
dmPrintVersion(); dmPrintVersion();
taosCleanupArgs();
return 0; return 0;
} }
if (dmInitLog() != 0) { if (dmInitLog() != 0) {
printf("failed to start since init log error\n"); printf("failed to start since init log error");
taosCleanupArgs();
return -1; return -1;
} }
if (taosInitCfg(configDir, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) { if (taosInitCfg(configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0) != 0) {
dError("failed to start since read config error"); dError("failed to start since read config error");
taosCleanupArgs();
return -1; return -1;
} }
@ -203,9 +219,11 @@ int main(int argc, char const *argv[]) {
dmDumpCfg(); dmDumpCfg();
taosCleanupCfg(); taosCleanupCfg();
taosCloseLog(); taosCloseLog();
taosCleanupArgs();
return 0; return 0;
} }
dmSetProcInfo(argc, (char **)argv); dmSetProcInfo(argc, (char **)argv);
taosCleanupArgs();
return dmRunDnode(); return dmRunDnode();
} }

1
source/dnode/mgmt/implement/src/dmExec.c
View File

@ -174,6 +174,7 @@ void dmStopNode(SMgmtWrapper *pWrapper) {
void dmCloseNode(SMgmtWrapper *pWrapper) { void dmCloseNode(SMgmtWrapper *pWrapper) {
dInfo("node:%s, start to close", pWrapper->name); dInfo("node:%s, start to close", pWrapper->name);
pWrapper->deployed = false;
while (pWrapper->refCount > 0) { while (pWrapper->refCount > 0) {
taosMsleep(10); taosMsleep(10);

70
source/dnode/mgmt/implement/src/dmHandle.c
View File

@ -17,7 +17,7 @@
#include "dmImp.h" #include "dmImp.h"
static void dmUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { static void dmUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) {
if (pDnode->data.dnodeId == 0) { if (pDnode->data.dnodeId == 0 || pDnode->data.clusterId == 0) {
dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId);
taosWLockLatch(&pDnode->data.latch); taosWLockLatch(&pDnode->data.latch);
pDnode->data.dnodeId = pCfg->dnodeId; pDnode->data.dnodeId = pCfg->dnodeId;
@ -57,6 +57,7 @@ void dmSendStatusReq(SDnode *pDnode) {
req.dnodeVer = pDnode->data.dnodeVer; req.dnodeVer = pDnode->data.dnodeVer;
req.dnodeId = pDnode->data.dnodeId; req.dnodeId = pDnode->data.dnodeId;
req.clusterId = pDnode->data.clusterId; req.clusterId = pDnode->data.clusterId;
if (req.clusterId == 0) req.dnodeId = 0;
req.rebootTime = pDnode->data.rebootTime; req.rebootTime = pDnode->data.rebootTime;
req.updateTime = pDnode->data.updateTime; req.updateTime = pDnode->data.updateTime;
req.numOfCores = tsNumOfCores; req.numOfCores = tsNumOfCores;
@ -145,10 +146,10 @@ int32_t dmProcessCreateNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMs
dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); dError("node:%s, failed to create since %s", pWrapper->name, terrstr());
} else { } else {
dDebug("node:%s, has been created", pWrapper->name); dDebug("node:%s, has been created", pWrapper->name);
(void)dmOpenNode(pWrapper);
pWrapper->required = true; pWrapper->required = true;
pWrapper->deployed = true; pWrapper->deployed = true;
pWrapper->procType = pDnode->ptype; pWrapper->procType = pDnode->ptype;
(void)dmOpenNode(pWrapper);
} }
taosThreadMutexUnlock(&pDnode->mutex); taosThreadMutexUnlock(&pDnode->mutex);
@ -170,13 +171,13 @@ int32_t dmProcessDropNodeReq(SDnode *pDnode, EDndNodeType ntype, SNodeMsg *pMsg)
dError("node:%s, failed to drop since %s", pWrapper->name, terrstr()); dError("node:%s, failed to drop since %s", pWrapper->name, terrstr());
} else { } else {
dDebug("node:%s, has been dropped", pWrapper->name); dDebug("node:%s, has been dropped", pWrapper->name);
pWrapper->required = false;
pWrapper->deployed = false;
} }
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
if (code == 0) { if (code == 0) {
pWrapper->required = false;
pWrapper->deployed = false;
dmCloseNode(pWrapper); dmCloseNode(pWrapper);
taosRemoveDir(pWrapper->path); taosRemoveDir(pWrapper->path);
} }
@ -220,13 +221,11 @@ static int32_t dmSpawnUdfd(SDnode *pDnode);
void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) { void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
uv_close((uv_handle_t*)process, NULL);
SDnode *pDnode = process->data; SDnode *pDnode = process->data;
SUdfdData *pData = &pDnode->udfdData; if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pDnode->udfdData.stopCalled)) {
if (atomic_load_8(&pData->stopping) != 0) { dInfo("udfd process exit due to SIGINT or dnode-mgmt called stop");
dDebug("udfd process exit due to stopping");
} else { } else {
uv_close((uv_handle_t*)&pData->ctrlPipe, NULL); dInfo("udfd process restart");
dmSpawnUdfd(pDnode); dmSpawnUdfd(pDnode);
} }
} }
@ -248,6 +247,7 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) {
options.file = path; options.file = path;
options.exit_cb = dmUdfdExit; options.exit_cb = dmUdfdExit;
SUdfdData *pData = &pDnode->udfdData; SUdfdData *pData = &pDnode->udfdData;
uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1); uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
@ -260,6 +260,8 @@ static int32_t dmSpawnUdfd(SDnode *pDnode) {
options.stdio_count = 3; options.stdio_count = 3;
options.stdio = child_stdio; options.stdio = child_stdio;
options.flags = UV_PROCESS_DETACHED;
char dnodeIdEnvItem[32] = {0}; char dnodeIdEnvItem[32] = {0};
char thrdPoolSizeEnvItem[32] = {0}; char thrdPoolSizeEnvItem[32] = {0};
snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId); snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId);
@ -284,24 +286,34 @@ static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) {
} }
} }
void dmWatchUdfd(void *args) { static void dmUdfdStopAsyncCb(uv_async_t *async) {
SDnode *pDnode = async->data;
SUdfdData *pData = &pDnode->udfdData;
uv_stop(&pData->loop);
}
static void dmWatchUdfd(void *args) {
SDnode *pDnode = args; SDnode *pDnode = args;
SUdfdData *pData = &pDnode->udfdData; SUdfdData *pData = &pDnode->udfdData;
uv_loop_init(&pData->loop); uv_loop_init(&pData->loop);
uv_async_init(&pData->loop, &pData->stopAsync, dmUdfdStopAsyncCb);
pData->stopAsync.data = pDnode;
int32_t err = dmSpawnUdfd(pDnode); int32_t err = dmSpawnUdfd(pDnode);
atomic_store_32(&pData->spawnErr, err); atomic_store_32(&pData->spawnErr, err);
uv_barrier_wait(&pData->barrier); uv_barrier_wait(&pData->barrier);
uv_run(&pData->loop, UV_RUN_DEFAULT); uv_run(&pData->loop, UV_RUN_DEFAULT);
err = uv_loop_close(&pData->loop); uv_loop_close(&pData->loop);
while (err == UV_EBUSY) {
uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL); uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL);
uv_run(&pData->loop, UV_RUN_DEFAULT); uv_run(&pData->loop, UV_RUN_DEFAULT);
err = uv_loop_close(&pData->loop); uv_loop_close(&pData->loop);
}
return; return;
} }
int32_t dmStartUdfd(SDnode *pDnode) { static int32_t dmStartUdfd(SDnode *pDnode) {
char dnodeId[8] = {0};
snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId);
uv_os_setenv("DNODE_ID", dnodeId);
SUdfdData *pData = &pDnode->udfdData; SUdfdData *pData = &pDnode->udfdData;
if (pData->startCalled) { if (pData->startCalled) {
dInfo("dnode-mgmt start udfd already called"); dInfo("dnode-mgmt start udfd already called");
@ -309,30 +321,34 @@ int32_t dmStartUdfd(SDnode *pDnode) {
} }
pData->startCalled = true; pData->startCalled = true;
uv_barrier_init(&pData->barrier, 2); uv_barrier_init(&pData->barrier, 2);
pData->stopping = 0;
uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
uv_barrier_wait(&pData->barrier); uv_barrier_wait(&pData->barrier);
int32_t err = atomic_load_32(&pData->spawnErr);
if (err != 0) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
pData->needCleanUp = false;
dInfo("dnode-mgmt udfd cleaned up after spawn err");
} else {
pData->needCleanUp = true; pData->needCleanUp = true;
return pData->spawnErr; }
return err;
} }
int32_t dmStopUdfd(SDnode *pDnode) { static int32_t dmStopUdfd(SDnode *pDnode) {
dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d", dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d",
pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr); pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr);
SUdfdData *pData = &pDnode->udfdData; SUdfdData *pData = &pDnode->udfdData;
if (!pData->needCleanUp) { if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) {
return 0; return 0;
} }
atomic_store_8(&pData->stopping, 1); atomic_store_32(&pData->stopCalled, 1);
pData->needCleanUp = false;
uv_barrier_destroy(&pData->barrier); uv_barrier_destroy(&pData->barrier);
if (pData->spawnErr == 0) { uv_async_send(&pData->stopAsync);
uv_process_kill(&pData->process, SIGINT);
}
uv_stop(&pData->loop);
uv_thread_join(&pData->thread); uv_thread_join(&pData->thread);
dInfo("dnode-mgmt udfd cleaned up");
atomic_store_8(&pData->stopping, 0);
return 0; return 0;
} }

35
source/dnode/mgmt/implement/src/dmTransport.c
View File

@ -59,6 +59,10 @@ static inline int32_t dmBuildMsg(SNodeMsg *pMsg, SRpcMsg *pRpc) {
pMsg->clientIp = connInfo.clientIp; pMsg->clientIp = connInfo.clientIp;
pMsg->clientPort = connInfo.clientPort; pMsg->clientPort = connInfo.clientPort;
memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg)); memcpy(&pMsg->rpcMsg, pRpc, sizeof(SRpcMsg));
if ((pRpc->msgType & 1u)) {
assert(pRpc->refId != 0);
}
// assert(pRpc->handle != NULL && pRpc->refId != 0 && pMsg->rpcMsg.refId != 0);
return 0; return 0;
} }
@ -67,12 +71,15 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe
SNodeMsg *pMsg = NULL; SNodeMsg *pMsg = NULL;
NodeMsgFp msgFp = NULL; NodeMsgFp msgFp = NULL;
uint16_t msgType = pRpc->msgType; uint16_t msgType = pRpc->msgType;
bool needRelease = false;
if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) { if (pEpSet && pEpSet->numOfEps > 0 && msgType == TDMT_MND_STATUS_RSP) {
dmSetMnodeEpSet(pWrapper->pDnode, pEpSet); dmSetMnodeEpSet(pWrapper->pDnode, pEpSet);
} }
if (dmMarkWrapper(pWrapper) != 0) goto _OVER; if (dmMarkWrapper(pWrapper) != 0) goto _OVER;
needRelease = true;
if ((msgFp = dmGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER; if ((msgFp = dmGetMsgFp(pWrapper, pRpc)) == NULL) goto _OVER;
if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER; if ((pMsg = taosAllocateQitem(sizeof(SNodeMsg))) == NULL) goto _OVER;
if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER; if (dmBuildMsg(pMsg, pRpc) != 0) goto _OVER;
@ -84,7 +91,7 @@ static void dmProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSe
dTrace("msg:%p, is created and put into child queue, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType), dTrace("msg:%p, is created and put into child queue, type:%s handle:%p user:%s", pMsg, TMSG_INFO(msgType),
pRpc->handle, pMsg->user); pRpc->handle, pMsg->user);
code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, pRpc->handle, code = taosProcPutToChildQ(pWrapper->procObj, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, pRpc->handle,
PROC_FUNC_REQ); pRpc->refId, PROC_FUNC_REQ);
} else { } else {
dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user); dTrace("msg:%p, should not processed in child process, handle:%p user:%s", pMsg, pRpc->handle, pMsg->user);
ASSERT(1); ASSERT(1);
@ -107,7 +114,7 @@ _OVER:
} }
} }
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code, .refId = pRpc->refId};
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
} }
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
@ -115,8 +122,10 @@ _OVER:
rpcFreeCont(pRpc->pCont); rpcFreeCont(pRpc->pCont);
} }
if (needRelease) {
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
} }
}
static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SDnodeTrans * pTrans = &pDnode->trans; SDnodeTrans * pTrans = &pDnode->trans;
@ -131,10 +140,17 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
return; return;
} }
if (msgType == TDMT_DND_NET_TEST) {
dTrace("net test req will be processed, handle:%p, app:%p", pMsg->handle, pMsg->ahandle);
dmProcessServerStatusReq(pDnode, pMsg);
return;
}
if (pDnode->status != DND_STAT_RUNNING) { if (pDnode->status != DND_STAT_RUNNING) {
dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); dError("msg:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
if (isReq) { if (isReq) {
SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle}; SRpcMsg rspMsg = {
.handle = pMsg->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pMsg->ahandle, .refId = pMsg->refId};
rpcSendResponse(&rspMsg); rpcSendResponse(&rspMsg);
} }
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
@ -143,7 +159,8 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
if (isReq && pMsg->pCont == NULL) { if (isReq && pMsg->pCont == NULL) {
dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); dError("req:%s not processed since its empty, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_INVALID_MSG_LEN, .ahandle = pMsg->ahandle}; SRpcMsg rspMsg = {
.handle = pMsg->handle, .code = TSDB_CODE_INVALID_MSG_LEN, .ahandle = pMsg->ahandle, .refId = pMsg->refId};
rpcSendResponse(&rspMsg); rpcSendResponse(&rspMsg);
return; return;
} }
@ -151,7 +168,8 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
if (pWrapper == NULL) { if (pWrapper == NULL) {
dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle); dError("msg:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pMsg->handle, pMsg->ahandle);
if (isReq) { if (isReq) {
SRpcMsg rspMsg = {.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle}; SRpcMsg rspMsg = {
.handle = pMsg->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pMsg->ahandle, .refId = pMsg->refId};
rpcSendResponse(&rspMsg); rpcSendResponse(&rspMsg);
} }
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
@ -170,6 +188,9 @@ static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
} }
dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle); dTrace("msg:%s will be processed by %s, app:%p", TMSG_INFO(msgType), pWrapper->name, pMsg->ahandle);
if (isReq) {
assert(pMsg->refId != 0);
}
dmProcessRpcMsg(pWrapper, pMsg, pEpSet); dmProcessRpcMsg(pWrapper, pMsg, pEpSet);
} }
@ -317,7 +338,7 @@ static void dmConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t
if (code != 0) { if (code != 0) {
dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); dError("msg:%p, failed to process since code:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code));
if (pRpc->msgType & 1U) { if (pRpc->msgType & 1U) {
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno}; SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = terrno, .refId = pRpc->refId};
dmSendRsp(pWrapper, &rsp); dmSendRsp(pWrapper, &rsp);
} }
@ -346,7 +367,7 @@ static void dmConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t
dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg); dmSendRpcReq(pWrapper->pDnode, (SEpSet *)((char *)pMsg + sizeof(SRpcMsg)), pMsg);
break; break;
case PROC_FUNC_RSP: case PROC_FUNC_RSP:
taosProcRemoveHandle(pWrapper->procObj, pMsg->handle); pMsg->refId = taosProcRemoveHandle(pWrapper->procObj, pMsg->handle);
dmSendRpcRsp(pWrapper->pDnode, pMsg); dmSendRpcRsp(pWrapper->pDnode, pMsg);
break; break;
default: default:

2
source/dnode/mgmt/implement/src/dmWorker.c
View File

@ -150,7 +150,7 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
if (pRpc->msgType & 1u) { if (pRpc->msgType & 1u) {
if (code != 0) code = terrno; if (code != 0) code = terrno;
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code, .refId = pRpc->refId};
rpcSendResponse(&rsp); rpcSendResponse(&rsp);
} }

3
source/dnode/mgmt/interface/inc/dmDef.h
View File

@ -151,8 +151,9 @@ typedef struct SUdfdData {
uv_barrier_t barrier; uv_barrier_t barrier;
uv_process_t process; uv_process_t process;
int spawnErr; int spawnErr;
int8_t stopping;
uv_pipe_t ctrlPipe; uv_pipe_t ctrlPipe;
uv_async_t stopAsync;
int32_t stopCalled;
} SUdfdData; } SUdfdData;
typedef struct SDnode { typedef struct SDnode {

1
source/dnode/mgmt/interface/inc/dmInt.h
View File

@ -37,6 +37,7 @@ void dmSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgF
void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); void dmReportStartup(SDnode *pDnode, const char *pName, const char *pDesc);
void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc); void dmReportStartupByWrapper(SMgmtWrapper *pWrapper, const char *pName, const char *pDesc);
void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg); void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pMsg);
void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pMsg);
void dmGetMonitorSysInfo(SMonSysInfo *pInfo); void dmGetMonitorSysInfo(SMonSysInfo *pInfo);
// dmFile.c // dmFile.c

1
source/dnode/mgmt/interface/src/dmEnv.c
View File

@ -53,6 +53,7 @@ void dmCleanup() {
} }
monCleanup(); monCleanup();
syncCleanUp();
walCleanUp(); walCleanUp();
taosStopCacheRefreshWorker(); taosStopCacheRefreshWorker();
dInfo("dnode env is cleaned up"); dInfo("dnode env is cleaned up");

2
source/dnode/mgmt/interface/src/dmFile.c
View File

@ -124,7 +124,7 @@ TdFilePtr dmCheckRunning(const char *dataDir) {
TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to lock file:%s since %s", filepath, terrstr()); dError("failed to open file:%s since %s", filepath, terrstr());
return NULL; return NULL;
} }

14
source/dnode/mgmt/interface/src/dmInt.c
View File

@ -171,13 +171,25 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) {
} }
} }
void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) {
dDebug("net test req is received");
SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0};
rsp.pCont = rpcMallocCont(pRpc->contLen);
if (rsp.pCont == NULL) {
rsp.code = TSDB_CODE_OUT_OF_MEMORY;
} else {
rsp.contLen = pRpc->contLen;
}
rpcSendResponse(&rsp);
}
void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) { void dmProcessServerStatusReq(SDnode *pDnode, SRpcMsg *pReq) {
dDebug("server status req is received"); dDebug("server status req is received");
SServerStatusRsp statusRsp = {0}; SServerStatusRsp statusRsp = {0};
dmGetServerStatus(pDnode, &statusRsp); dmGetServerStatus(pDnode, &statusRsp);
SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle}; SRpcMsg rspMsg = {.handle = pReq->handle, .ahandle = pReq->ahandle, .refId = pReq->refId};
int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp); int32_t rspLen = tSerializeSServerStatusRsp(NULL, 0, &statusRsp);
if (rspLen < 0) { if (rspLen < 0) {
rspMsg.code = TSDB_CODE_OUT_OF_MEMORY; rspMsg.code = TSDB_CODE_OUT_OF_MEMORY;

2
source/dnode/mgmt/mgmt_bnode/src/bmHandle.c
View File

@ -53,7 +53,7 @@ int32_t bmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) {
return -1; return -1;
} }
if (createReq.dnodeId != pDnode->data.dnodeId) { if (pDnode->data.dnodeId != 0 && createReq.dnodeId != pDnode->data.dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->data.dnodeId); dError("failed to create bnode since %s, input:%d cur:%d", terrstr(), createReq.dnodeId, pDnode->data.dnodeId);
return -1; return -1;

4
source/dnode/mgmt/mgmt_bnode/src/bmWorker.c
View File

@ -17,7 +17,8 @@
#include "bmInt.h" #include "bmInt.h"
static void bmSendErrorRsp(SNodeMsg *pMsg, int32_t code) { static void bmSendErrorRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rpcRsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; SRpcMsg rpcRsp = {
.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code, .refId = pMsg->rpcMsg.refId};
tmsgSendRsp(&rpcRsp); tmsgSendRsp(&rpcRsp);
dTrace("msg:%p, is freed", pMsg); dTrace("msg:%p, is freed", pMsg);
@ -38,6 +39,7 @@ static void bmSendErrorRsps(STaosQall *qall, int32_t numOfMsgs, int32_t code) {
static inline void bmSendRsp(SNodeMsg *pMsg, int32_t code) { static inline void bmSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle, .ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code, .code = code,
.pCont = pMsg->pRsp, .pCont = pMsg->pRsp,
.contLen = pMsg->rspLen}; .contLen = pMsg->rspLen};

8
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -134,9 +134,9 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) {
return -1; return -1;
} }
if (alterReq.dnodeId != pDnode->data.dnodeId) { if (pDnode->data.dnodeId != 0 && alterReq.dnodeId != pDnode->data.dnodeId) {
terrno = TSDB_CODE_INVALID_OPTION; terrno = TSDB_CODE_INVALID_OPTION;
dError("failed to alter mnode since %s, dnodeId:%d input:%d", terrstr(), pDnode->data.dnodeId, alterReq.dnodeId); dError("failed to alter mnode since %s, input:%d cur:%d", terrstr(), alterReq.dnodeId, pDnode->data.dnodeId);
return -1; return -1;
} else { } else {
return mmAlter(pMgmt, &alterReq); return mmAlter(pMgmt, &alterReq);
@ -177,6 +177,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_QNODE_LIST, mmProcessReadMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
@ -218,9 +219,6 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE);
// Requests handled by VNODE // Requests handled by VNODE
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

4
source/dnode/mgmt/mgmt_mnode/src/mmInt.c
View File

@ -161,9 +161,9 @@ static int32_t mmOpen(SMgmtWrapper *pWrapper) {
SMnodeOpt option = {0}; SMnodeOpt option = {0};
if (!deployed) { if (!deployed) {
dInfo("mnode start to deploy"); dInfo("mnode start to deploy");
if (pWrapper->procType == DND_PROC_CHILD) { // if (pWrapper->procType == DND_PROC_CHILD) {
pWrapper->pDnode->data.dnodeId = 1; pWrapper->pDnode->data.dnodeId = 1;
} // }
mmBuildOptionForDeploy(pMgmt, &option); mmBuildOptionForDeploy(pMgmt, &option);
} else { } else {
dInfo("mnode start to open"); dInfo("mnode start to open");

1
source/dnode/mgmt/mgmt_mnode/src/mmWorker.c
View File

@ -19,6 +19,7 @@
static inline void mmSendRsp(SNodeMsg *pMsg, int32_t code) { static inline void mmSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle, .ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code, .code = code,
.pCont = pMsg->pRsp, .pCont = pMsg->pRsp,
.contLen = pMsg->rspLen}; .contLen = pMsg->rspLen};

1
source/dnode/mgmt/mgmt_qnode/src/qmWorker.c
View File

@ -19,6 +19,7 @@
static inline void qmSendRsp(SNodeMsg *pMsg, int32_t code) { static inline void qmSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle, .ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code, .code = code,
.pCont = pMsg->pRsp, .pCont = pMsg->pRsp,
.contLen = pMsg->rspLen}; .contLen = pMsg->rspLen};

1
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -19,6 +19,7 @@
static inline void smSendRsp(SNodeMsg *pMsg, int32_t code) { static inline void smSendRsp(SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle, .ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code, .code = code,
.pCont = pMsg->pRsp, .pCont = pMsg->pRsp,
.contLen = pMsg->rspLen}; .contLen = pMsg->rspLen};

1
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -105,6 +105,7 @@ void vmStopWorker(SVnodesMgmt *pMgmt);
int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode); void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); // sync integration
int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

38
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -120,6 +120,16 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->hashBegin = pCreate->hashBegin; pCfg->hashBegin = pCreate->hashBegin;
pCfg->hashEnd = pCreate->hashEnd; pCfg->hashEnd = pCreate->hashEnd;
pCfg->hashMethod = pCreate->hashMethod; pCfg->hashMethod = pCreate->hashMethod;
// sync integration
pCfg->syncCfg.myIndex = pCreate->selfIndex;
pCfg->syncCfg.replicaNum = pCreate->replica;
memset(&(pCfg->syncCfg.nodeInfo), 0, sizeof(pCfg->syncCfg.nodeInfo));
for (int i = 0; i < pCreate->replica; ++i) {
(pCfg->syncCfg.nodeInfo)[i].nodePort = (pCreate->replicas)[i].port;
snprintf((pCfg->syncCfg.nodeInfo)[i].nodeFqdn, sizeof((pCfg->syncCfg.nodeInfo)[i].nodeFqdn), "%s",
(pCreate->replicas)[i].fqdn);
}
} }
static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) { static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) {
@ -171,6 +181,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; // sync integration
msgCb.qsizeFp = vmGetQueueSize; msgCb.qsizeFp = vmGetQueueSize;
SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@ -190,6 +201,16 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
return code; return code;
} }
code = vnodeStart(pImpl);
if (code != 0) {
tFreeSCreateVnodeReq(&createReq);
dError("vgId:%d, failed to start sync since %s", createReq.vgId, terrstr());
vnodeClose(pImpl);
vnodeDestroy(path, pMgmt->pTfs);
terrno = code;
return code;
}
code = vmWriteVnodesToFile(pMgmt); code = vmWriteVnodesToFile(pMgmt);
if (code != 0) { if (code != 0) {
tFreeSCreateVnodeReq(&createReq); tFreeSCreateVnodeReq(&createReq);
@ -264,10 +285,6 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);
@ -282,4 +299,17 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
// sync integration
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
} }

58
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "vmInt.h" #include "vmInt.h"
#include "libs/function/function.h"
SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) {
SVnodeObj *pVnode = NULL; SVnodeObj *pVnode = NULL;
@ -137,6 +138,7 @@ static void *vmOpenVnodeFunc(void *param) {
msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; // sync integration
msgCb.qsizeFp = vmGetQueueSize; msgCb.qsizeFp = vmGetQueueSize;
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@ -145,6 +147,7 @@ static void *vmOpenVnodeFunc(void *param) {
pThread->failed++; pThread->failed++;
} else { } else {
vmOpenVnode(pMgmt, pCfg, pImpl); vmOpenVnode(pMgmt, pCfg, pImpl);
//vnodeStart(pImpl);
dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->opened++; pThread->opened++;
} }
@ -266,6 +269,9 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
// walCleanUp(); // walCleanUp();
taosMemoryFree(pMgmt); taosMemoryFree(pMgmt);
pWrapper->pMgmt = NULL; pWrapper->pMgmt = NULL;
// syncCleanUp();
udfcClose();
dInfo("vnode-mgmt is cleaned up"); dInfo("vnode-mgmt is cleaned up");
} }
@ -306,6 +312,12 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
} }
dmReportStartup(pDnode, "vnode-wal", "initialized"); dmReportStartup(pDnode, "vnode-wal", "initialized");
// sync integration
if (syncInit() != 0) {
dError("failed to open sync since %s", terrstr());
return -1;
}
if (vnodeInit(tsNumOfCommitThreads) != 0) { if (vnodeInit(tsNumOfCommitThreads) != 0) {
dError("failed to init vnode since %s", terrstr()); dError("failed to init vnode since %s", terrstr());
goto _OVER; goto _OVER;
@ -323,6 +335,10 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
} }
dmReportStartup(pDnode, "vnode-vnodes", "initialized"); dmReportStartup(pDnode, "vnode-vnodes", "initialized");
if (udfcOpen() != 0) {
dError("failed to open udfc in dnode");
}
code = 0; code = 0;
_OVER: _OVER:
@ -343,10 +359,52 @@ static int32_t vmRequire(SMgmtWrapper *pWrapper, bool *required) {
return 0; return 0;
} }
static int32_t vmStart(SMgmtWrapper *pWrapper) {
dDebug("vnode-mgmt start to run");
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
taosRLockLatch(&pMgmt->latch);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode;
vnodeStart(pVnode->pImpl);
pIter = taosHashIterate(pMgmt->hash, pIter);
}
taosRUnLockLatch(&pMgmt->latch);
return 0;
}
static void vmStop(SMgmtWrapper *pWrapper) {
#if 0
dDebug("vnode-mgmt start to stop");
SVnodesMgmt *pMgmt = pWrapper->pMgmt;
taosRLockLatch(&pMgmt->latch);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode;
vnodeStop(pVnode->pImpl);
pIter = taosHashIterate(pMgmt->hash, pIter);
}
taosRUnLockLatch(&pMgmt->latch);
#endif
}
void vmSetMgmtFp(SMgmtWrapper *pWrapper) { void vmSetMgmtFp(SMgmtWrapper *pWrapper) {
SMgmtFp mgmtFp = {0}; SMgmtFp mgmtFp = {0};
mgmtFp.openFp = vmInit; mgmtFp.openFp = vmInit;
mgmtFp.closeFp = vmCleanup; mgmtFp.closeFp = vmCleanup;
mgmtFp.startFp = vmStart;
mgmtFp.stopFp = vmStop;
mgmtFp.requiredFp = vmRequire; mgmtFp.requiredFp = vmRequire;
vmInitMsgHandle(pWrapper); vmInitMsgHandle(pWrapper);

92
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -14,11 +14,16 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "vmInt.h" #include "vmInt.h"
#include "sync.h"
#include "syncTools.h"
static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle, .ahandle = pMsg->rpcMsg.ahandle,
.refId = pMsg->rpcMsg.refId,
.code = code, .code = code,
.pCont = pMsg->pRsp, .pCont = pMsg->pRsp,
.contLen = pMsg->rspLen}; .contLen = pMsg->rspLen};
@ -88,7 +93,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
int64_t version; SRpcMsg rsp;
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *)); SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
if (pArray == NULL) { if (pArray == NULL) {
@ -107,33 +112,56 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
} }
} }
#if 0
int64_t version;
vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version); vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version);
numOfMsgs = taosArrayGetSize(pArray); numOfMsgs = taosArrayGetSize(pArray);
for (int32_t i = 0; i < numOfMsgs; i++) { for (int32_t i = 0; i < numOfMsgs; i++) {
SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
SRpcMsg *pRpc = &pMsg->rpcMsg; SRpcMsg *pRpc = &pMsg->rpcMsg;
SRpcMsg rsp;
rsp.pCont = NULL; rsp.pCont = NULL;
rsp.contLen = 0; rsp.contLen = 0;
rsp.code = 0; rsp.code = 0;
rsp.handle = pRpc->handle; rsp.handle = pRpc->handle;
rsp.ahandle = pRpc->ahandle; rsp.ahandle = pRpc->ahandle;
rsp.refId = pRpc->refId;
int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp); int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp);
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
}
#else
// sync integration response
for (int i = 0; i < taosArrayGetSize(pArray); i++) {
SNodeMsg *pMsg;
SRpcMsg * pRpc;
#if 0 pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
if (pRsp != NULL) { pRpc = &pMsg->rpcMsg;
pRsp->ahandle = pRpc->ahandle;
taosMemoryFree(pRsp); rsp.ahandle = pRpc->ahandle;
rsp.handle = pRpc->handle;
rsp.refId = pRpc->refId;
rsp.pCont = NULL;
rsp.contLen = 0;
int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false);
if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
rsp.code = TSDB_CODE_SYN_NOT_LEADER;
tmsgSendRsp(&rsp);
} else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) {
rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR;
tmsgSendRsp(&rsp);
} else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) {
// ok
// send response in applyQ
} else { } else {
if (code != 0 && terrno != 0) code = terrno; assert(0);
vmSendRsp(pVnode->pWrapper, pMsg, code); }
} }
#endif #endif
}
for (int32_t i = 0; i < numOfMsgs; i++) { for (int32_t i = 0; i < numOfMsgs; i++) {
SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i); SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
@ -148,13 +176,43 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle; SVnodeObj *pVnode = pInfo->ahandle;
SNodeMsg * pMsg = NULL; SNodeMsg * pMsg = NULL;
SRpcMsg rsp;
for (int32_t i = 0; i < numOfMsgs; ++i) { for (int32_t i = 0; i < numOfMsgs; ++i) {
#if 1
// sync integration
taosGetQitem(qall, (void **)&pMsg); taosGetQitem(qall, (void **)&pMsg);
// todo // init response rpc msg
SRpcMsg *pRsp = NULL; rsp.code = 0;
// (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp); rsp.pCont = NULL;
rsp.contLen = 0;
// get original rpc msg
assert(pMsg->rpcMsg.msgType == TDMT_VND_SYNC_APPLY_MSG);
SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(&pMsg->rpcMsg);
syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
SRpcMsg originalRpcMsg;
syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
// apply data into tsdb
if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
rsp.code = terrno;
dTrace("vnodeProcessWriteReq error, code:%d", terrno);
}
syncApplyMsgDestroy(pSyncApplyMsg);
rpcFreeCont(originalRpcMsg.pCont);
// if leader, send response
if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
rsp.ahandle = pMsg->rpcMsg.ahandle;
rsp.handle = pMsg->rpcMsg.handle;
rsp.refId = pMsg->rpcMsg.refId;
tmsgSendRsp(&rsp);
}
#endif
} }
} }
@ -290,6 +348,7 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT
} else { } else {
dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType));
pMsg->rpcMsg = *pRpc; pMsg->rpcMsg = *pRpc;
// if (pMsg->rpcMsg.handle != NULL) assert(pMsg->rpcMsg.refId != 0);
switch (qtype) { switch (qtype) {
case QUERY_QUEUE: case QUERY_QUEUE:
dTrace("msg:%p, will be put into vnode-query queue", pMsg); dTrace("msg:%p, will be put into vnode-query queue", pMsg);
@ -307,6 +366,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT
dTrace("msg:%p, will be put into vnode-merge queue", pMsg); dTrace("msg:%p, will be put into vnode-merge queue", pMsg);
taosWriteQitem(pVnode->pMergeQ, pMsg); taosWriteQitem(pVnode->pMergeQ, pMsg);
break; break;
case SYNC_QUEUE: // sync integration
dTrace("msg:%p, will be put into vnode-sync queue", pMsg);
taosWriteQitem(pVnode->pSyncQ, pMsg);
break;
default: default:
code = -1; code = -1;
terrno = TSDB_CODE_INVALID_PARA; terrno = TSDB_CODE_INVALID_PARA;
@ -333,6 +396,11 @@ int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE); return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE);
} }
// sync integration
int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
return vmPutRpcMsgToQueue(pWrapper, pRpc, SYNC_QUEUE);
}
int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) { int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
int32_t size = -1; int32_t size = -1;
SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId); SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId);

4
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -135,6 +135,7 @@ typedef struct {
int32_t failedTimes; int32_t failedTimes;
void* rpcHandle; void* rpcHandle;
void* rpcAHandle; void* rpcAHandle;
int64_t rpcRefId;
void* rpcRsp; void* rpcRsp;
int32_t rpcRspLen; int32_t rpcRspLen;
SArray* redoLogs; SArray* redoLogs;
@ -349,7 +350,6 @@ typedef struct {
int32_t version; int32_t version;
int32_t nextColId; int32_t nextColId;
float xFilesFactor; float xFilesFactor;
int32_t aggregationMethod;
int32_t delay; int32_t delay;
int32_t ttl; int32_t ttl;
int32_t numOfColumns; int32_t numOfColumns;
@ -449,7 +449,6 @@ typedef struct {
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
int8_t withTagSchema;
SRWLatch lock; SRWLatch lock;
int32_t sqlLen; int32_t sqlLen;
int32_t astLen; int32_t astLen;
@ -516,7 +515,6 @@ typedef struct {
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
int8_t withTagSchema;
SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub
// TODO put -1 into unassignVgs // TODO put -1 into unassignVgs
// SArray* unassignedVgs; // SArray* unassignedVgs;

1
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -30,6 +30,7 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup); SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
int32_t mndGetGlobalVgroupVersion(int32_t *vgId);
void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

8
source/dnode/mnode/impl/src/mndDb.c
View File

@ -1191,6 +1191,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
char *p = strchr(usedbReq.db, '.'); char *p = strchr(usedbReq.db, '.');
if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) { if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN); memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
//mndGetGlobalVgroupVersion(); TODO
static int32_t vgVersion = 1; static int32_t vgVersion = 1;
if (usedbReq.vgVersion < vgVersion) { if (usedbReq.vgVersion < vgVersion) {
usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
@ -1202,16 +1203,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos);
usedbRsp.vgVersion = vgVersion++; usedbRsp.vgVersion = vgVersion++;
if (taosArrayGetSize(usedbRsp.pVgroupInfos) <= 0) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
} else {
code = 0;
}
} else { } else {
usedbRsp.vgVersion = usedbReq.vgVersion; usedbRsp.vgVersion = usedbReq.vgVersion;
code = 0;
} }
usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
code = 0;
// no jump, need to construct rsp // no jump, need to construct rsp
} else { } else {

3
source/dnode/mnode/impl/src/mndDef.c
View File

@ -237,7 +237,6 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->withTbName = pSub->withTbName; pSubNew->withTbName = pSub->withTbName;
pSubNew->withSchema = pSub->withSchema; pSubNew->withSchema = pSub->withSchema;
pSubNew->withTag = pSub->withTag; pSubNew->withTag = pSub->withTag;
pSubNew->withTagSchema = pSub->withTagSchema;
pSubNew->vgNum = pSub->vgNum; pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
@ -270,7 +269,6 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
tlen += taosEncodeFixedI8(buf, pSub->withTbName); tlen += taosEncodeFixedI8(buf, pSub->withTbName);
tlen += taosEncodeFixedI8(buf, pSub->withSchema); tlen += taosEncodeFixedI8(buf, pSub->withSchema);
tlen += taosEncodeFixedI8(buf, pSub->withTag); tlen += taosEncodeFixedI8(buf, pSub->withTag);
tlen += taosEncodeFixedI8(buf, pSub->withTagSchema);
void *pIter = NULL; void *pIter = NULL;
int32_t sz = taosHashGetSize(pSub->consumerHash); int32_t sz = taosHashGetSize(pSub->consumerHash);
@ -297,7 +295,6 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
buf = taosDecodeFixedI8(buf, &pSub->withTbName); buf = taosDecodeFixedI8(buf, &pSub->withTbName);
buf = taosDecodeFixedI8(buf, &pSub->withSchema); buf = taosDecodeFixedI8(buf, &pSub->withSchema);
buf = taosDecodeFixedI8(buf, &pSub->withTag); buf = taosDecodeFixedI8(buf, &pSub->withTag);
buf = taosDecodeFixedI8(buf, &pSub->withTagSchema);
int32_t sz; int32_t sz;
buf = taosDecodeFixedI32(buf, &sz); buf = taosDecodeFixedI32(buf, &sz);

3
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -335,11 +335,14 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) {
} }
bool roleChanged = false; bool roleChanged = false;
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) { for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
// sync integration
if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
if (pVgroup->vnodeGid[vg].role != pVload->syncState) { if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
roleChanged = true; roleChanged = true;
} }
pVgroup->vnodeGid[vg].role = pVload->syncState; pVgroup->vnodeGid[vg].role = pVload->syncState;
} }
}
if (roleChanged) { if (roleChanged) {
// notify scheduler role has changed // notify scheduler role has changed
} }

2
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -309,7 +309,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) {
goto _OVER; goto _OVER;
} }
if (createReq.pCode[0] == 0) { if (createReq.codeLen <= 1) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER; goto _OVER;
} }

16
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -199,15 +199,6 @@ static const SInfosTableSchema vgroupsSchema[] = {
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
}; };
// TODO put into perf schema
static const SInfosTableSchema topicSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema consumerSchema[] = { static const SInfosTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
@ -291,7 +282,6 @@ static const SInfosTableMeta infosMeta[] = {
{TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
{TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
{TSDB_INS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
{TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)}, {TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
{TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)}, {TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)},
{TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)}, {TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)},
@ -335,7 +325,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
return -1; return -1;
} }
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName) + 1, &meta, sizeof(meta))) { if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -350,10 +340,10 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
return -1; return -1;
} }
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName) + 1); STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName); mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
return -1; return -1;
} }

2
source/dnode/mnode/impl/src/mndOffset.c
View File

@ -172,7 +172,7 @@ static int32_t mndProcessCommitOffsetReq(SNodeMsg *pMsg) {
bool create = false; bool create = false;
SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key); SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, key);
if (pOffsetObj == NULL) { if (pOffsetObj == NULL) {
pOffsetObj = taosMemoryMalloc(sizeof(SMqOffset)); pOffsetObj = taosMemoryMalloc(sizeof(SMqOffsetObj));
memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN); memcpy(pOffsetObj->key, key, TSDB_PARTITION_KEY_LEN);
create = true; create = true;
} }

4
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -49,6 +49,7 @@ static const SPerfsTableSchema topicSchema[] = {
static const SPerfsTableSchema consumerSchema[] = { static const SPerfsTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@ -61,6 +62,7 @@ static const SPerfsTableSchema subscribeSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY}, {.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
}; };
@ -126,7 +128,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == meta) { if (NULL == meta) {
mError("invalid performance schema table name:%s", tbName); mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
return -1; return -1;
} }

3
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -58,7 +58,8 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter);
int32_t mndInitProfile(SMnode *pMnode) { int32_t mndInitProfile(SMnode *pMnode) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt; SProfileMgmt *pMgmt = &pMnode->profileMgmt;
int32_t connCheckTime = tsShellActivityTimer * 2; // in ms
int32_t connCheckTime = tsShellActivityTimer * 2 * 1000;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn");
if (pMgmt->cache == NULL) { if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;

19
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -444,22 +444,25 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
goto _OVER; goto _OVER;
} }
qlistRsp.epSetList = taosArrayInit(5, sizeof(SEpSet)); qlistRsp.addrsList = taosArrayInit(5, sizeof(SQueryNodeAddr));
if (NULL == qlistRsp.epSetList) { if (NULL == qlistRsp.addrsList) {
mError("failed to alloc epSet while process qnode list req"); mError("failed to alloc epSet while process qnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;
} }
void *pIter = NULL;
while (1) { while (1) {
void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj); pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj);
if (pIter == NULL) break; if (pIter == NULL) break;
SEpSet epSet = {.numOfEps = 1}; SQueryNodeAddr nodeAddr = {0};
tstrncpy(epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); nodeAddr.nodeId = QNODE_HANDLE;
epSet.eps[0].port = pObj->pDnode->port; nodeAddr.epSet.numOfEps = 1;
tstrncpy(nodeAddr.epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
nodeAddr.epSet.eps[0].port = pObj->pDnode->port;
(void)taosArrayPush(qlistRsp.epSetList, &epSet); (void)taosArrayPush(qlistRsp.addrsList, &nodeAddr);
numOfRows++; numOfRows++;
sdbRelease(pSdb, pObj); sdbRelease(pSdb, pObj);
@ -470,7 +473,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
} }
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pRsp = taosMemoryMalloc(rspLen); void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) { if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER; goto _OVER;

33
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -476,7 +476,10 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) {
int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) { int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscribeObj* pSub) {
SSdb* pSdb = pMnode->pSdb; SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL; SVgObj* pVgroup = NULL;
SQueryPlan* pPlan = qStringToQueryPlan(pTopic->physicalPlan); SQueryPlan* pPlan = NULL;
SSubplan* plan = NULL;
if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
pPlan = qStringToQueryPlan(pTopic->physicalPlan);
if (pPlan == NULL) { if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1; return -1;
@ -501,7 +504,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC; terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
return -1; return -1;
} }
SSubplan* plan = nodesListGetNode(inner->pNodeList, 0); plan = nodesListGetNode(inner->pNodeList, 0);
}
int64_t unexistKey = -1; int64_t unexistKey = -1;
SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); SMqConsumerEpInSub* pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
@ -519,38 +523,35 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
} }
pSub->vgNum++; pSub->vgNum++;
plan->execNode.nodeId = pVgroup->vgId;
plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp));
pVgEp->epSet = plan->execNode.epSet; pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup);
pVgEp->vgId = plan->execNode.nodeId; pVgEp->vgId = pVgroup->vgId;
taosArrayPush(pEpInSub->vgs, &pVgEp);
#if 0
SMqConsumerEp consumerEp = {0};
consumerEp.status = 0;
consumerEp.consumerId = -1;
consumerEp.epSet = plan->execNode.epSet;
consumerEp.vgId = plan->execNode.nodeId;
#endif
mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId); mDebug("init subscribption %s, assign vg: %d", pSub->key, pVgEp->vgId);
if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) {
int32_t msgLen; int32_t msgLen;
plan->execNode.epSet = pVgEp->epSet;
plan->execNode.nodeId = pVgEp->vgId;
if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) { if (qSubPlanToString(plan, &pVgEp->qmsg, &msgLen) < 0) {
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
terrno = TSDB_CODE_QRY_INVALID_INPUT; terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1; return -1;
} }
taosArrayPush(pEpInSub->vgs, &pVgEp); } else {
pVgEp->qmsg = strdup("");
}
ASSERT(taosHashGetSize(pSub->consumerHash) == 1); ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
/*taosArrayPush(pSub->unassignedVg, &consumerEp);*/ /*taosArrayPush(pSub->unassignedVg, &consumerEp);*/
} }
ASSERT(pEpInSub->vgs->size > 0);
pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t)); pEpInSub = taosHashGet(pSub->consumerHash, &unexistKey, sizeof(int64_t));
ASSERT(pEpInSub->vgs->size > 0); ASSERT(pEpInSub->vgs->size > 0);

13
source/dnode/mnode/impl/src/mndShow.c
View File

@ -28,7 +28,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq);
int32_t mndInitShow(SMnode *pMnode) { int32_t mndInitShow(SMnode *pMnode) {
SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5, true, (__cache_free_fn_t)mndFreeShowObj, "show"); pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
if (pMgmt->cache == NULL) { if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", terrstr()); mError("failed to alloc show cache since %s", terrstr());
@ -84,8 +84,6 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
type = TSDB_MGMT_TABLE_GRANTS; type = TSDB_MGMT_TABLE_GRANTS;
} else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
type = TSDB_MGMT_TABLE_VGROUP; type = TSDB_MGMT_TABLE_VGROUP;
} else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) {
type = TSDB_MGMT_TABLE_TOPICS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) {
type = TSDB_MGMT_TABLE_CONSUMERS; type = TSDB_MGMT_TABLE_CONSUMERS;
} else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) {
@ -102,6 +100,8 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
type = TSDB_MGMT_TABLE_QUERIES; type = TSDB_MGMT_TABLE_QUERIES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) {
type = TSDB_MGMT_TABLE_VNODES; type = TSDB_MGMT_TABLE_VNODES;
} else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) {
type = TSDB_MGMT_TABLE_TOPICS;
} else { } else {
// ASSERT(0); // ASSERT(0);
} }
@ -189,12 +189,15 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
} }
if (retrieveReq.showId == 0) { if (retrieveReq.showId == 0) {
STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1); STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) { if (pMeta == NULL) {
terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; pMeta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) {
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
return -1; return -1;
} }
}
pShow = mndCreateShowObj(pMnode, &retrieveReq); pShow = mndCreateShowObj(pMnode, &retrieveReq);
if (pShow == NULL) { if (pShow == NULL) {

27
source/dnode/mnode/impl/src/mndStb.c
View File

@ -87,7 +87,6 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->version, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER)
SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER) SDB_SET_INT32(pRaw, dataPos, (int32_t)(pStb->xFilesFactor * 10000), _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->aggregationMethod, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->delay, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER)
SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER)
@ -175,7 +174,6 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
int32_t xFilesFactor = 0; int32_t xFilesFactor = 0;
SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER) SDB_GET_INT32(pRaw, dataPos, &xFilesFactor, _OVER)
pStb->xFilesFactor = xFilesFactor / 10000.0f; pStb->xFilesFactor = xFilesFactor / 10000.0f;
SDB_GET_INT32(pRaw, dataPos, &pStb->aggregationMethod, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->delay, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->delay, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER)
@ -406,7 +404,7 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
req.name = (char *)tNameGetTableName(&name); req.name = (char *)tNameGetTableName(&name);
req.suid = pStb->uid; req.suid = pStb->uid;
req.rollup = pStb->aggregationMethod > -1 ? 1 : 0; req.rollup = pStb->ast1Len > 0 ? 1 : 0;
req.schema.nCols = pStb->numOfColumns; req.schema.nCols = pStb->numOfColumns;
req.schema.sver = 0; req.schema.sver = 0;
req.schema.pSchema = pStb->pColumns; req.schema.pSchema = pStb->pColumns;
@ -421,20 +419,19 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
if (req.rollup) { if (req.rollup) {
req.pRSmaParam.xFilesFactor = pStb->xFilesFactor; req.pRSmaParam.xFilesFactor = pStb->xFilesFactor;
req.pRSmaParam.delay = pStb->delay; req.pRSmaParam.delay = pStb->delay;
req.pRSmaParam.nFuncIds = 1; // only 1 aggregation method supported currently if (pStb->ast1Len > 0) {
req.pRSmaParam.pFuncIds = (func_id_t *)taosMemoryCalloc(req.pRSmaParam.nFuncIds, sizeof(func_id_t)); if (mndConvertRSmaTask(pStb->pAst1, 0, 0, &req.pRSmaParam.qmsg1, &req.pRSmaParam.qmsg1Len) != TSDB_CODE_SUCCESS) {
if (req.pRSmaParam.pFuncIds == NULL) { return NULL;
terrno = TSDB_CODE_OUT_OF_MEMORY; }
}
if (pStb->ast2Len > 0) {
if (mndConvertRSmaTask(pStb->pAst2, 0, 0, &req.pRSmaParam.qmsg2, &req.pRSmaParam.qmsg2Len) != TSDB_CODE_SUCCESS) {
return NULL; return NULL;
} }
for (int32_t f = 0; f < req.pRSmaParam.nFuncIds; ++f) {
req.pRSmaParam.pFuncIds[f] = pStb->aggregationMethod;
} }
} }
// get length // get length
if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) { if (tEncodeSize(tEncodeSVCreateStbReq, &req, contLen) < 0) {
taosMemoryFree(req.pRSmaParam.pFuncIds);
return NULL; return NULL;
} }
@ -442,7 +439,8 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
SMsgHead *pHead = taosMemoryMalloc(contLen); SMsgHead *pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) { if (pHead == NULL) {
taosMemoryFree(req.pRSmaParam.pFuncIds); taosMemoryFreeClear(req.pRSmaParam.qmsg1);
taosMemoryFreeClear(req.pRSmaParam.qmsg2);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
@ -453,13 +451,13 @@ static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pSt
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER); tCoderInit(&coder, TD_LITTLE_ENDIAN, pBuf, contLen - sizeof(SMsgHead), TD_ENCODER);
if (tEncodeSVCreateStbReq(&coder, &req) < 0) { if (tEncodeSVCreateStbReq(&coder, &req) < 0) {
taosMemoryFree(req.pRSmaParam.pFuncIds);
return NULL; return NULL;
} }
tCoderClear(&coder); tCoderClear(&coder);
*pContLen = contLen; *pContLen = contLen;
taosMemoryFree(req.pRSmaParam.pFuncIds); taosMemoryFreeClear(req.pRSmaParam.qmsg1);
taosMemoryFreeClear(req.pRSmaParam.qmsg2);
return pHead; return pHead;
} }
@ -672,7 +670,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SNodeMsg *pReq, SMCreateStbReq *pCre
stbObj.version = 1; stbObj.version = 1;
stbObj.nextColId = 1; stbObj.nextColId = 1;
stbObj.xFilesFactor = pCreate->xFilesFactor; stbObj.xFilesFactor = pCreate->xFilesFactor;
stbObj.aggregationMethod = pCreate->aggregationMethod;
stbObj.delay = pCreate->delay; stbObj.delay = pCreate->delay;
stbObj.ttl = pCreate->ttl; stbObj.ttl = pCreate->ttl;
stbObj.numOfColumns = pCreate->numOfColumns; stbObj.numOfColumns = pCreate->numOfColumns;

179
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -35,11 +35,6 @@
#define MND_SUBSCRIBE_REBALANCE_CNT 3 #define MND_SUBSCRIBE_REBALANCE_CNT 3
enum {
MQ_SUBSCRIBE_STATUS__ACTIVE = 1,
MQ_SUBSCRIBE_STATUS__DELETED,
};
static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
@ -89,7 +84,6 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
pSub->withTbName = pTopic->withTbName; pSub->withTbName = pTopic->withTbName;
pSub->withSchema = pTopic->withSchema; pSub->withSchema = pTopic->withSchema;
pSub->withTag = pTopic->withTag; pSub->withTag = pTopic->withTag;
pSub->withTagSchema = pTopic->withTagSchema;
ASSERT(taosHashGetSize(pSub->consumerHash) == 1); ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
@ -115,7 +109,6 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
req.withTbName = pSub->withTbName; req.withTbName = pSub->withTbName;
req.withSchema = pSub->withSchema; req.withSchema = pSub->withSchema;
req.withTag = pSub->withTag; req.withTag = pSub->withTag;
req.withTagSchema = pSub->withTagSchema;
strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req);
@ -514,9 +507,11 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
// TODO replace assert with error check // TODO replace assert with error check
ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
// if add more consumer to balanced subscribe, // if add more consumer to balanced subscribe,
// possibly no vg is changed // possibly no vg is changed
/*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
if (rebInput.pTopic) { if (rebInput.pTopic) {
@ -673,177 +668,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) {
sdbRelease(pSdb, pSub); sdbRelease(pSdb, pSub);
} }
#if 0
static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode;
char *msgStr = pMsg->rpcMsg.pCont;
SCMSubscribeReq subscribe;
tDeserializeSCMSubscribeReq(msgStr, &subscribe);
int64_t consumerId = subscribe.consumerId;
char *cgroup = subscribe.consumerGroup;
SArray *newSub = subscribe.topicNames;
int32_t newTopicNum = subscribe.topicNum;
taosArraySortString(newSub, taosArrayCompareString);
SArray *oldSub = NULL;
int32_t oldTopicNum = 0;
bool createConsumer = false;
// create consumer if not exist
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
// create consumer
pConsumer = mndCreateConsumer(consumerId, cgroup);
createConsumer = true;
} else {
pConsumer->epoch++;
oldSub = pConsumer->currentTopics;
}
pConsumer->currentTopics = newSub;
if (oldSub != NULL) {
oldTopicNum = taosArrayGetSize(oldSub);
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
if (pTrans == NULL) {
// TODO: free memory
return -1;
}
int32_t i = 0, j = 0;
while (i < newTopicNum || j < oldTopicNum) {
char *newTopicName = NULL;
char *oldTopicName = NULL;
if (i >= newTopicNum) {
// encode unset topic msg to all vnodes related to that topic
oldTopicName = taosArrayGetP(oldSub, j);
j++;
} else if (j >= oldTopicNum) {
newTopicName = taosArrayGetP(newSub, i);
i++;
} else {
newTopicName = taosArrayGetP(newSub, i);
oldTopicName = taosArrayGetP(oldSub, j);
int32_t comp = compareLenPrefixedStr(newTopicName, oldTopicName);
if (comp == 0) {
// do nothing
oldTopicName = newTopicName = NULL;
i++;
j++;
continue;
} else if (comp < 0) {
oldTopicName = NULL;
i++;
} else {
newTopicName = NULL;
j++;
}
}
if (oldTopicName != NULL) {
ASSERT(newTopicName == NULL);
// cancel subscribe of old topic
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, oldTopicName);
ASSERT(pSub);
int32_t csz = taosArrayGetSize(pSub->consumers);
for (int32_t ci = 0; ci < csz; ci++) {
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, ci);
if (pSubConsumer->consumerId == consumerId) {
int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
for (int32_t vgi = 0; vgi < vgsz; vgi++) {
SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi);
mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp, oldTopicName);
taosArrayPush(pSub->unassignedVg, pConsumerEp);
}
taosArrayRemove(pSub->consumers, ci);
break;
}
}
char *oldTopicNameDup = strdup(oldTopicName);
taosArrayPush(pConsumer->recentRemovedTopics, &oldTopicNameDup);
atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__MODIFY);
/*pSub->status = MQ_SUBSCRIBE_STATUS__DELETED;*/
} else if (newTopicName != NULL) {
ASSERT(oldTopicName == NULL);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName);
if (pTopic == NULL) {
mError("topic being subscribed not exist: %s", newTopicName);
continue;
}
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, newTopicName);
bool createSub = false;
if (pSub == NULL) {
mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup,
newTopicName);
pSub = mndCreateSubscription(pMnode, pTopic, cgroup);
createSub = true;
mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg);
}
SMqSubConsumer mqSubConsumer;
mqSubConsumer.consumerId = consumerId;
mqSubConsumer.vgInfo = taosArrayInit(0, sizeof(SMqConsumerEp));
taosArrayPush(pSub->consumers, &mqSubConsumer);
// if have un assigned vg, assign one to the consumer
if (taosArrayGetSize(pSub->unassignedVg) > 0) {
SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg);
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = consumerId;
taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp);
if (pConsumerEp->oldConsumerId == -1) {
mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, newTopicName,
pConsumerEp->consumerId);
mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
} else {
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, newTopicName);
}
// to trigger rebalance at once, do not set status active
/*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/
}
SSdbRaw *pRaw = mndSubActionEncode(pSub);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pRaw);
if (!createSub) mndReleaseSubscribe(pMnode, pSub);
mndReleaseTopic(pMnode, pTopic);
}
}
/*if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);*/
// persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("mq-subscribe-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer);
return -1;
}
mndTransDrop(pTrans);
if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
#endif
static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) {
mndTransProcessRsp(pRsp); mndTransProcessRsp(pRsp);
return 0; return 0;
} }
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}

32
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -82,7 +82,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withTagSchema, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER);
@ -146,7 +145,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withTagSchema, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char));
@ -234,6 +232,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
sdbRelease(pSdb, pTopic); sdbRelease(pSdb, pTopic);
} }
#if 0
static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
SName name = {0}; SName name = {0};
tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@ -243,6 +242,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
return mndAcquireDb(pMnode, db); return mndAcquireDb(pMnode, db);
} }
#endif
static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) {
int32_t contLen = sizeof(SDDropTopicReq); int32_t contLen = sizeof(SDDropTopicReq);
@ -262,15 +262,11 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
} }
static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) { if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
return -1; return -1;
} }
if ((pCreate->ast == NULL || pCreate->ast[0] == 0) && pCreate->subscribeDbName[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
return -1;
}
return 0; return 0;
} }
@ -286,10 +282,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
topicObj.version = 1; topicObj.version = 1;
topicObj.sql = strdup(pCreate->sql); topicObj.sql = strdup(pCreate->sql);
topicObj.sqlLen = strlen(pCreate->sql) + 1; topicObj.sqlLen = strlen(pCreate->sql) + 1;
topicObj.ast = strdup(pCreate->ast);
topicObj.astLen = strlen(pCreate->ast) + 1;
if (pCreate->ast && pCreate->ast[0]) { if (pCreate->ast && pCreate->ast[0]) {
topicObj.ast = strdup(pCreate->ast);
topicObj.astLen = strlen(pCreate->ast) + 1;
topicObj.subType = TOPIC_SUB_TYPE__TABLE; topicObj.subType = TOPIC_SUB_TYPE__TABLE;
topicObj.withTbName = 0; topicObj.withTbName = 0;
topicObj.withSchema = 0; topicObj.withSchema = 0;
@ -318,6 +314,9 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
return -1; return -1;
} }
} else { } else {
topicObj.ast = strdup("");
topicObj.astLen = 1;
topicObj.physicalPlan = strdup("");
topicObj.subType = TOPIC_SUB_TYPE__DB; topicObj.subType = TOPIC_SUB_TYPE__DB;
topicObj.withTbName = 1; topicObj.withTbName = 1;
topicObj.withSchema = 1; topicObj.withSchema = 1;
@ -386,7 +385,7 @@ static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq) {
goto CREATE_TOPIC_OVER; goto CREATE_TOPIC_OVER;
} }
pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name); pDb = mndAcquireDb(pMnode, createTopicReq.subscribeDbName);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED; terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
goto CREATE_TOPIC_OVER; goto CREATE_TOPIC_OVER;
@ -524,8 +523,11 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
int32_t cols = 0; int32_t cols = 0;
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN);
varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE])); SName n;
tNameFromString(&n, pTopic->name, T_NAME_ACCT|T_NAME_DB);
tNameGetDbName(&n, varDataVal(topicName));
varDataSetLen(topicName, strlen(varDataVal(topicName)));
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
@ -534,12 +536,12 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false); colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
char *sql = taosMemoryCalloc(1, strlen(pTopic->sql) + 1 + VARSTR_HEADER_SIZE); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
strcpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql); tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN);
varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
colDataAppend(pColInfo, numOfRows, (const char *)sql, false); colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
taosMemoryFree(sql); // taosMemoryFree(sql);
numOfRows++; numOfRows++;
sdbRelease(pSdb, pTopic); sdbRelease(pSdb, pTopic);

3
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -484,6 +484,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S
pTrans->createdTime = taosGetTimestampMs(); pTrans->createdTime = taosGetTimestampMs();
pTrans->rpcHandle = pReq->handle; pTrans->rpcHandle = pReq->handle;
pTrans->rpcAHandle = pReq->ahandle; pTrans->rpcAHandle = pReq->ahandle;
pTrans->rpcRefId = pReq->refId;
pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); pTrans->redoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *));
pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); pTrans->undoLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *));
pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *)); pTrans->commitLogs = taosArrayInit(MND_TRANS_ARRAY_SIZE, sizeof(void *));
@ -703,6 +704,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
pNew->rpcHandle = pTrans->rpcHandle; pNew->rpcHandle = pTrans->rpcHandle;
pNew->rpcAHandle = pTrans->rpcAHandle; pNew->rpcAHandle = pTrans->rpcAHandle;
pNew->rpcRefId = pTrans->rpcRefId;
pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRsp = pTrans->rpcRsp;
pNew->rpcRspLen = pTrans->rpcRspLen; pNew->rpcRspLen = pTrans->rpcRspLen;
pTrans->rpcRsp = NULL; pTrans->rpcRsp = NULL;
@ -767,6 +769,7 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) {
SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, SRpcMsg rspMsg = {.handle = pTrans->rpcHandle,
.code = pTrans->code, .code = pTrans->code,
.ahandle = pTrans->rpcAHandle, .ahandle = pTrans->rpcAHandle,
.refId = pTrans->rpcRefId,
.pCont = rpcCont, .pCont = rpcCont,
.contLen = pTrans->rpcRspLen}; .contLen = pTrans->rpcRspLen};
tmsgSendRsp(&rspMsg); tmsgSendRsp(&rspMsg);

4
source/dnode/mnode/impl/src/mnode.c
View File

@ -385,7 +385,11 @@ int32_t mndProcessMsg(SNodeMsg *pMsg) {
terrno = code; terrno = code;
mTrace("msg:%p, in progress, app:%p", pMsg, ahandle); mTrace("msg:%p, in progress, app:%p", pMsg, ahandle);
} else if (code != 0) { } else if (code != 0) {
if (terrno != TSDB_CODE_OPS_NOT_SUPPORT) {
mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle); mError("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
} else {
mTrace("msg:%p, failed to process since %s, app:%p", pMsg, terrstr(), ahandle);
}
} else { } else {
mTrace("msg:%p, is processed, app:%p", pMsg, ahandle); mTrace("msg:%p, is processed, app:%p", pMsg, ahandle);
} }

1
source/dnode/mnode/impl/test/db/db.cpp
View File

@ -89,6 +89,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
taosMsleep(1000); // Wait for the vnode to become the leader
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);

88
source/dnode/mnode/impl/test/func/func.cpp
View File

@ -22,17 +22,16 @@ class MndTestFunc : public ::testing::Test {
void SetUp() override {} void SetUp() override {}
void TearDown() override {} void TearDown() override {}
void SetCode(SCreateFuncReq* pReq, const char* pCode); void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size);
void SetComment(SCreateFuncReq* pReq, const char* pComment); void SetComment(SCreateFuncReq* pReq, const char* pComment);
}; };
Testbase MndTestFunc::test; Testbase MndTestFunc::test;
void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) { void MndTestFunc::SetCode(SCreateFuncReq *pReq, const char *pCode, int32_t size) {
int32_t len = strlen(pCode); pReq->pCode = (char*)taosMemoryMalloc(size);
pReq->pCode = (char*)taosMemoryCalloc(1, len + 1); memcpy(pReq->pCode, pCode, size);
strcpy(pReq->pCode, pCode); pReq->codeLen = size;
pReq->codeLen = len;
} }
void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) {
@ -79,7 +78,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{ {
SCreateFuncReq createReq = {0}; SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1"); strcpy(createReq.name, "f1");
SetCode(&createReq, ""); SetCode(&createReq, "", 1);
SetComment(&createReq, "comment1"); SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -95,7 +94,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{ {
SCreateFuncReq createReq = {0}; SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1"); strcpy(createReq.name, "f1");
SetCode(&createReq, "code1"); SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1"); SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -111,7 +110,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{ {
SCreateFuncReq createReq = {0}; SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1"); strcpy(createReq.name, "f1");
SetCode(&createReq, "code1"); SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1"); SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
@ -128,7 +127,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
for (int32_t i = 0; i < 3; ++i) { for (int32_t i = 0; i < 3; ++i) {
SCreateFuncReq createReq = {0}; SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1"); strcpy(createReq.name, "f1");
SetCode(&createReq, "code1"); SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1"); SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
createReq.igExists = 0; createReq.igExists = 0;
@ -253,7 +252,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
createReq.outputLen = 24; createReq.outputLen = 24;
createReq.bufSize = 6; createReq.bufSize = 6;
createReq.signature = 18; createReq.signature = 18;
SetCode(&createReq, "code2"); SetCode(&createReq, "code2", 6);
SetComment(&createReq, "comment2"); SetComment(&createReq, "comment2");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -439,3 +438,70 @@ TEST_F(MndTestFunc, 04_Drop_Func) {
test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", "");
EXPECT_EQ(test.GetShowRows(), 1); EXPECT_EQ(test.GetShowRows(), 1);
} }
TEST_F(MndTestFunc, 05_Actual_code) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "udf1");
char code[300] = {0};
for (int32_t i = 0; i < sizeof(code); ++i) {
code[i] = (i) % 20;
}
SetCode(&createReq, code, 300);
SetComment(&createReq, "comment1");
createReq.bufSize = 8;
createReq.igExists = 0;
createReq.funcType = 1;
createReq.scriptType = 2;
createReq.outputType = TSDB_DATA_TYPE_SMALLINT;
createReq.outputLen = 12;
createReq.bufSize = 4;
createReq.signature = 5;
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSCreateFuncReq(pReq, contLen, &createReq);
tFreeSCreateFuncReq(&createReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
}
{
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "udf1");
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
tFreeSRetrieveFuncReq(&retrieveReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp);
EXPECT_EQ(retrieveRsp.numOfFuncs, 1);
EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos));
SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0);
EXPECT_STREQ(pFuncInfo->name, "udf1");
EXPECT_EQ(pFuncInfo->funcType, 1);
EXPECT_EQ(pFuncInfo->scriptType, 2);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT);
EXPECT_EQ(pFuncInfo->outputLen, 12);
EXPECT_EQ(pFuncInfo->bufSize, 4);
EXPECT_EQ(pFuncInfo->signature, 5);
EXPECT_STREQ("comment1", pFuncInfo->pComment);
for (int32_t i = 0; i < 300; ++i) {
EXPECT_EQ(pFuncInfo->pCode[i], (i) % 20);
}
tFreeSRetrieveFuncRsp(&retrieveRsp);
}
}

1
source/dnode/mnode/impl/test/sma/sma.cpp
View File

@ -258,6 +258,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateDbReq(dbname, &contLen); pReq = BuildCreateDbReq(dbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(1000); // Wait for the vnode to become the leader
} }
{ {

8
source/dnode/mnode/impl/test/stb/stb.cpp
View File

@ -304,6 +304,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -438,6 +439,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -497,6 +499,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -538,6 +541,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -602,6 +606,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -655,6 +660,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -715,6 +721,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {
@ -775,6 +782,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen); void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0); ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
} }
{ {

7
source/dnode/qnode/src/qnode.c
View File

@ -17,6 +17,7 @@
#include "qndInt.h" #include "qndInt.h"
#include "query.h" #include "query.h"
#include "qworker.h" #include "qworker.h"
//#include "tudf.h"
SQnode *qndOpen(const SQnodeOpt *pOption) { SQnode *qndOpen(const SQnodeOpt *pOption) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
@ -25,6 +26,8 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
return NULL; return NULL;
} }
//udfcOpen();
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) {
taosMemoryFreeClear(pQnode); taosMemoryFreeClear(pQnode);
return NULL; return NULL;
@ -37,13 +40,15 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
void qndClose(SQnode *pQnode) { void qndClose(SQnode *pQnode) {
qWorkerDestroy((void **)&pQnode->pQuery); qWorkerDestroy((void **)&pQnode->pQuery);
//udfcClose();
taosMemoryFree(pQnode); taosMemoryFree(pQnode);
} }
int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; }
int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) {
qTrace("message in query queue is processing"); qTrace("message in qnode query queue is processing");
SReadHandle handle = {0}; SReadHandle handle = {0};
switch (pMsg->msgType) { switch (pMsg->msgType) {

1
source/dnode/vnode/CMakeLists.txt
View File

@ -13,6 +13,7 @@ target_sources(
"src/vnd/vnodeStateMgr.c" "src/vnd/vnodeStateMgr.c"
"src/vnd/vnodeModule.c" "src/vnd/vnodeModule.c"
"src/vnd/vnodeSvr.c" "src/vnd/vnodeSvr.c"
"src/vnd/vnodeSync.c"
# meta # meta
"src/meta/metaOpen.c" "src/meta/metaOpen.c"

8
source/dnode/vnode/inc/vnode.h
View File

@ -21,6 +21,7 @@
#include "tqueue.h" #include "tqueue.h"
#include "trpc.h" #include "trpc.h"
#include "sync.h"
#include "tarray.h" #include "tarray.h"
#include "tfs.h" #include "tfs.h"
#include "wal.h" #include "wal.h"
@ -62,6 +63,12 @@ int32_t vnodeSync(SVnode *pVnode);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int vnodeValidateTableHash(SVnode *pVnode, char *tableFName); int vnodeValidateTableHash(SVnode *pVnode, char *tableFName);
int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode);
int64_t vnodeGetSyncHandle(SVnode *pVnode);
void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
// meta // meta
typedef struct SMeta SMeta; // todo: remove typedef struct SMeta SMeta; // todo: remove
typedef struct SMetaReader SMetaReader; typedef struct SMetaReader SMetaReader;
@ -159,6 +166,7 @@ struct SVnodeCfg {
bool isWeak; bool isWeak;
STsdbCfg tsdbCfg; STsdbCfg tsdbCfg;
SWalCfg walCfg; SWalCfg walCfg;
SSyncCfg syncCfg; // sync integration
uint32_t hashBegin; uint32_t hashBegin;
uint32_t hashEnd; uint32_t hashEnd;
int8_t hashMethod; int8_t hashMethod;

3
source/dnode/vnode/src/inc/meta.h
View File

@ -110,6 +110,7 @@ typedef struct {
#define META_CHILD_TABLE TD_CHILD_TABLE #define META_CHILD_TABLE TD_CHILD_TABLE
#define META_NORMAL_TABLE TD_NORMAL_TABLE #define META_NORMAL_TABLE TD_NORMAL_TABLE
// int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaDropTable(SMeta* pMeta, tb_uid_t uid); int metaDropTable(SMeta* pMeta, tb_uid_t uid);
int metaCommit(SMeta* pMeta); int metaCommit(SMeta* pMeta);
int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg); int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
@ -131,7 +132,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
// SMetaDB // SMetaDB
int metaOpenDB(SMeta* pMeta); int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta); void metaCloseDB(SMeta* pMeta);
int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg); int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid); int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg); int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid); int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);

56
source/dnode/vnode/src/inc/tq.h
View File

@ -18,8 +18,10 @@
#include "executor.h" #include "executor.h"
#include "os.h" #include "os.h"
#include "tcache.h"
#include "thash.h" #include "thash.h"
#include "tmsg.h" #include "tmsg.h"
#include "trpc.h"
#include "ttimer.h" #include "ttimer.h"
#include "wal.h" #include "wal.h"
@ -81,7 +83,6 @@ typedef struct STqOffsetStore STqOffsetStore;
struct STqReadHandle { struct STqReadHandle {
int64_t ver; int64_t ver;
int64_t tbUid;
SHashObj* tbIdHash; SHashObj* tbIdHash;
const SSubmitReq* pMsg; const SSubmitReq* pMsg;
SSubmitBlk* pBlock; SSubmitBlk* pBlock;
@ -141,6 +142,15 @@ typedef struct {
FTqDelete pDeleter; FTqDelete pDeleter;
} STqMetaStore; } STqMetaStore;
typedef struct {
int64_t consumerId;
int32_t epoch;
int32_t skipLogNum;
int64_t reqOffset;
SRWLatch lock;
SRpcMsg* handle;
} STqPushHandle;
typedef struct { typedef struct {
char subKey[TSDB_SUBSCRIBE_KEY_LEN]; char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId; int64_t consumerId;
@ -149,19 +159,20 @@ typedef struct {
int8_t withTbName; int8_t withTbName;
int8_t withSchema; int8_t withSchema;
int8_t withTag; int8_t withTag;
int8_t withTagSchema;
char* qmsg; char* qmsg;
STqPushHandle pushHandle;
// SRWLatch lock; // SRWLatch lock;
SWalReadHandle* pWalReader; SWalReadHandle* pWalReader;
// number should be identical to fetch thread num // task number should be the same with fetch thread
STqReadHandle* pStreamReader[4]; STqReadHandle* pExecReader[5];
qTaskInfo_t task[4]; qTaskInfo_t task[5];
} STqExec; } STqExec;
struct STQ { struct STQ {
char* path; char* path;
// STqMetaStore* tqMeta; // STqMetaStore* tqMeta;
SHashObj* execs; // subKey -> tqExec SHashObj* pushMgr; // consumerId -> STqExec*
SHashObj* execs; // subKey -> STqExec
SHashObj* pStreamTasks; SHashObj* pStreamTasks;
SVnode* pVnode; SVnode* pVnode;
SWal* pWal; SWal* pWal;
@ -205,20 +216,6 @@ typedef struct {
SArray* topics; // SArray<STqTopic> SArray* topics; // SArray<STqTopic>
} STqConsumer; } STqConsumer;
enum {
TQ_PUSHER_TYPE__CLIENT = 1,
TQ_PUSHER_TYPE__STREAM,
};
typedef struct {
int8_t type;
int8_t reserved[3];
int32_t ttl;
int64_t consumerId;
SRpcMsg* pMsg;
// SMqPollRsp* rsp;
} STqClientPusher;
typedef struct { typedef struct {
int8_t type; int8_t type;
int8_t nodeType; int8_t nodeType;
@ -228,10 +225,6 @@ typedef struct {
// TODO sync function // TODO sync function
} STqStreamPusher; } STqStreamPusher;
typedef struct {
SHashObj* pHash; // <id, STqPush*>
} STqPushMgr;
typedef struct { typedef struct {
int8_t inited; int8_t inited;
tmr_h timer; tmr_h timer;
@ -247,14 +240,11 @@ void tqCleanUp();
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal); STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
void tqClose(STQ*); void tqClose(STQ*);
// required by vnode // required by vnode
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqCommit(STQ*); int tqCommit(STQ*);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
// int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
// int32_t tqProcessRebReq(STQ* pTq, char* msg);
// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId); int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen); int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId); int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
@ -295,16 +285,6 @@ int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t
int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetPersistAll(STqOffsetStore* pStore); int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
// tqPush
int32_t tqPushMgrInit();
void tqPushMgrCleanUp();
STqPushMgr* tqPushMgrOpen();
void tqPushMgrClose(STqPushMgr* pushMgr);
STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/vnode/src/inc/tsdb.h
View File

@ -74,7 +74,6 @@ int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid);
int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg);
void tsdbCleanupReadHandle(tsdbReaderT queryHandle); void tsdbCleanupReadHandle(tsdbReaderT queryHandle);
int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg); int32_t tdScanAndConvertSubmitMsg(SSubmitReq *pMsg);
typedef enum { typedef enum {
TSDB_FILE_HEAD = 0, // .head TSDB_FILE_HEAD = 0, // .head
TSDB_FILE_DATA, // .data TSDB_FILE_DATA, // .data

64
source/dnode/vnode/src/inc/tsdbSma.h Normal file
View File

@ -0,0 +1,64 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_TSDB_SMA_H_
#define _TD_VNODE_TSDB_SMA_H_
#include "os.h"
#include "thash.h"
#include "tmsg.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2);
struct STbDdlH {
void *ahandle;
void *result;
__tb_ddl_fn_t fp;
};
typedef struct {
tb_uid_t suid;
SArray *tbUids;
SHashObj *uidHash;
} STbUidStore;
static FORCE_INLINE int32_t tsdbUidStoreInit(STbUidStore **pStore) {
ASSERT(*pStore == NULL);
*pStore = taosMemoryCalloc(1, sizeof(STbUidStore));
if (*pStore == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
int32_t tsdbUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid);
void tsdbUidStoreDestory(STbUidStore *pStore);
void *tsdbUidStoreFree(STbUidStore *pStore);
int32_t tsdbRegisterRSma(STsdb *pTsdb, SMeta *pMeta, SVCreateTbReq *pReq);
int32_t tsdbFetchTbUidList(void *pTsdb, void **result, void *suid, void *uid);
int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pUidStore);
int32_t tsdbTriggerRSma(STsdb *pTsdb, SMeta *pMeta, void *pMsg, int32_t inputType);
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_TSDB_SMA_H_*/

Some files were not shown because too many files have changed in this diff Show More