xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor

This commit is contained in:
Cary Xu 2022-04-23 19:25:27 +08:00
parent fb45d0d935 d16a2e276a
commit 006b0cd0ac
307 changed files with 10173 additions and 8854 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cmake/cmake.define
View File

@ -1,5 +1,7 @@
cmake_minimum_required(VERSION 3.16)
set(CMAKE_VERBOSE_MAKEFILE ON)
#set output directory
SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
@ -43,11 +45,14 @@ ENDIF ()
IF (TD_WINDOWS)
MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd4999 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
SET(COMMON_FLAGS "/W3 /D_WIN32")
IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
ENDIF ()
# IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
# SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
# ENDIF ()
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")
ELSE ()
IF (${SANITIZER} MATCHES "true")

25
cmake/cmake.options
View File

@ -25,9 +25,28 @@ IF(${TD_WINDOWS})
ON
)
ENDIF ()
MESSAGE("build msvcregex Win32")
option(
BUILD_MSVCREGEX
"If build msvcregex on Windows"
ON
)
IF(${TD_LINUX} MATCHES TRUE)
MESSAGE("build wcwidth Win32")
option(
BUILD_WCWIDTH
"If build wcwidth on Windows"
ON
)
MESSAGE("build wingetopt Win32")
option(
BUILD_WINGETOPT
"If build wingetopt on Windows"
ON
)
ENDIF ()
option(
BUILD_TEST
@ -35,8 +54,6 @@ option(
ON
)
ENDIF ()
option(
BUILD_WITH_LEVELDB
"If build with leveldb"

13
cmake/msvcregex_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,13 @@
# msvcregex
ExternalProject_Add(msvcregex
GIT_REPOSITORY https://gitee.com/l0km/libgnurx-msvc.git
GIT_TAG master
SOURCE_DIR "${TD_CONTRIB_DIR}/msvcregex"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

2
cmake/pthread_CMakeLists.txt.in
View File

@ -3,7 +3,7 @@
ExternalProject_Add(pthread
GIT_REPOSITORY https://github.com/GerHobbelt/pthread-win32
GIT_TAG v3.0.3.1
SOURCE_DIR "${TD_CONTRIB_DIR}/pthread-win32"
SOURCE_DIR "${TD_CONTRIB_DIR}/pthread"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""

13
cmake/wcwidth_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,13 @@
# wcwidth
ExternalProject_Add(wcwidth
GIT_REPOSITORY https://github.com/fumiyas/wcwidth-cjk.git
GIT_TAG master
SOURCE_DIR "${TD_CONTRIB_DIR}/wcwidth"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

13
cmake/wingetopt_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,13 @@
# wingetopt
ExternalProject_Add(wingetopt
GIT_REPOSITORY https://github.com/alex85k/wingetopt.git
GIT_TAG master
SOURCE_DIR "${TD_CONTRIB_DIR}/wingetopt"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

95
contrib/CMakeLists.txt
View File

@ -14,9 +14,24 @@ if(${BUILD_PTHREAD})
cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# gnu regex
if(${BUILD_GNUREGEX})
cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
# iconv
if(${BUILD_WITH_ICONV})
cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# msvc regex
if(${BUILD_MSVCREGEX})
cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wcwidth
if(${BUILD_WCWIDTH})
cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wingetopt
if(${BUILD_WINGETOPT})
cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# googletest
@ -99,8 +114,27 @@ if(${BUILD_TEST})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
if(${TD_WINDOWS})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
)
endif(${TD_WINDOWS})
if(${TD_LINUX})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
endif(${TD_LINUX})
if(${TD_DARWIN})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
)
endif(${TD_DARWIN})
endif(${BUILD_TEST})
# cJson
@ -182,6 +216,53 @@ if(${BUILD_WITH_NURAFT})
add_subdirectory(nuraft)
endif(${BUILD_WITH_NURAFT})
# pthread
if(${BUILD_PTHREAD})
set(CMAKE_BUILD_TYPE release)
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
add_library(pthread STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
endif()
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
endif(${BUILD_WITH_ICONV})
# wingetopt
if(${BUILD_WINGETOPT})
add_subdirectory(wingetopt)
endif(${BUILD_WINGETOPT})
# msvcregex
if(${BUILD_MSVCREGEX})
add_library(msvcregex STATIC "")
target_sources(msvcregex
PRIVATE "msvcregex/regex.c"
)
target_include_directories(msvcregex
PRIVATE "msvcregex"
)
target_link_libraries(msvcregex
INTERFACE Shell32
)
SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
endif(${BUILD_MSVCREGEX})
# msvcregex
if(${BUILD_WCWIDTH})
add_library(wcwidth STATIC "")
target_sources(wcwidth
PRIVATE "wcwidth/wcwidth.c"
)
target_include_directories(wcwidth
PRIVATE "wcwidth"
)
SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
endif(${BUILD_WCWIDTH})
# CRAFT
if(${BUILD_WITH_CRAFT})
add_library(craft STATIC IMPORTED GLOBAL)
@ -238,8 +319,12 @@ if(${BUILD_WITH_SQLITE})
target_link_libraries(sqlite
INTERFACE m
INTERFACE pthread
INTERFACE dl
)
if(NOT TD_WINDOWS)
target_link_libraries(sqlite
INTERFACE dl
)
endif(NOT TD_WINDOWS)
endif(${BUILD_WITH_SQLITE})
# pthread

9
contrib/test/craft/CMakeLists.txt
View File

@ -1,2 +1,9 @@
add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
if(${BUILD_WINGETOPT})
target_link_libraries(simulate_vnode PUBLIC wingetopt)
target_include_directories(
simulate_vnode
PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
)
endif()

72
contrib/test/tdev/src/main.c
View File

@ -6,43 +6,39 @@
#define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
#define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
#define tPutA(buf, val) \
({ \
memcpy(buf, &val, sizeof(val)); \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutA(void **buf, uint64_t val) {
memcpy(buf, &val, sizeof(val));
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutB(buf, val) \
({ \
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \
((uint8_t *)buf)[0] = (val)&0xff; \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutB(void **buf, uint64_t val) {
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
((uint8_t *)buf)[0] = (val)&0xff;
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutC(buf, val) \
({ \
if (buf) { \
((uint64_t *)buf)[0] = (val); \
POINTER_SHIFT(buf, sizeof(val)); \
} \
NULL; \
})
static inline void tPutC(void **buf, uint64_t val) {
if (buf) {
((uint64_t *)buf)[0] = (val);
POINTER_SHIFT(buf, sizeof(val));
}
*buf = NULL;
}
#define tPutD(buf, val) \
({ \
uint64_t tmp = val; \
for (size_t i = 0; i < sizeof(val); i++) { \
((uint8_t *)buf)[i] = tmp & 0xff; \
tmp >>= 8; \
} \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutD(void **buf, uint64_t val) {
uint64_t tmp = val;
for (size_t i = 0; i < sizeof(val); i++) {
((uint8_t *)buf)[i] = tmp & 0xff;
tmp >>= 8;
}
*buf = POINTER_SHIFT(buf, sizeof(val));
}
static inline void tPutE(void **buf, uint64_t val) {
if (buf) {
@ -61,7 +57,7 @@ static void func(T t) {
switch (t) {
case A:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutA(pBuf, val);
tPutA(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -69,7 +65,7 @@ static void func(T t) {
break;
case B:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutB(pBuf, val);
tPutB(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -77,7 +73,7 @@ static void func(T t) {
break;
case C:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutC(pBuf, val);
tPutC(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -85,7 +81,7 @@ static void func(T t) {
break;
case D:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
pBuf = tPutD(pBuf, val);
tPutD(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}

1
example/CMakeLists.txt
View File

@ -39,6 +39,7 @@ target_include_directories(tstream
target_include_directories(demoapi
PUBLIC "${TD_SOURCE_DIR}/include/client"
PUBLIC "${TD_SOURCE_DIR}/include/os"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)

40
example/src/demoapi.c
View File

@ -4,10 +4,12 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
// #include <unistd.h>
#include <inttypes.h>
#ifndef WINDOWS
#include <argp.h>
#endif
#include "osSleep.h"
#include "taos.h"
#define debugPrint(fmt, ...) \
@ -32,6 +34,7 @@
int64_t g_num_of_tb = 2;
int64_t g_num_of_rec = 3;
#ifndef WINDOWS
static struct argp_option options[] = {
{"tables", 't', "NUMBER", 0, "Number of child tables, default is 10000."},
{"records", 'n', "NUMBER", 0,
@ -61,16 +64,16 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
}
static struct argp argp = {options, parse_opt, "", ""};
#endif
static void prepare_data(TAOS* taos) {
TAOS_RES *res;
res = taos_query(taos, "drop database if exists test;");
taos_free_result(res);
usleep(100000);
taosMsleep(100);
res = taos_query(taos, "create database test;");
taos_free_result(res);
usleep(100000);
taosMsleep(100);
taos_select_db(taos, "test");
char command[1024] = {0};
@ -87,18 +90,17 @@ static void prepare_data(TAOS* taos) {
taos_free_result(res);
for (int64_t i = 0; i < g_num_of_tb; i ++) {
// sprintf(command, "create table t%"PRId64" using meters "
// "tags(%"PRId64", '%s', '%s', '%s');",
// i, i, (i%2)?"beijing":"shanghai",
// (i%2)?"朝阳区":"黄浦区",
// (i%2)?"长安街":"中山路");
sprintf(command, "create table t%"PRId64" using meters "
"tags(%"PRId64", '%s', '%s', '%s');",
i, i, (i%2)?"beijing":"shanghai",
(i%2)?"朝阳区":"黄浦区",
(i%2)?"长安街":"中山路");
/* sprintf(command, "create table t%"PRId64" using meters "
"tags(%"PRId64", '%s', '%s', '%s');",
i, i,
(i%2)?"beijing":"shanghai",
(i%2)?"chaoyang":"huangpu",
(i%2?"changan street":"jianguo rd"));
*/
res = taos_query(taos, command);
if ((res) && (0 == taos_errno(res))) {
okPrint("t%" PRId64 " created\n", i);
@ -117,7 +119,8 @@ static void prepare_data(TAOS* taos) {
"'%c%d', '%s%c%d', '%c%d')",
i, 1650000000000+j, (float)j, j,
'a'+(int)j%25, rand(),
"涛思", 'z' - (int)j%25, rand(),
// "涛思", 'z' - (int)j%25, rand(),
"TAOS", 'z' - (int)j%25, rand(),
'b' - (int)j%25, rand()
);
res = taos_query(taos, command);
@ -196,7 +199,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %"PRId64"\n",
f, c,
*(int64_t*)(row[f]+c*sizeof(int64_t)));
*(int64_t*)((char*)(row[f])+c*sizeof(int64_t)));
}
break;
@ -208,7 +211,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %d\n",
f, c,
*(int32_t*)(row[f]+c*sizeof(int32_t)));
*(int32_t*)((char*)(row[f])+c*sizeof(int32_t)));
}
break;
@ -220,7 +223,7 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
printf("col%d, row: %"PRId64", "
"value: %f\n",
f, c,
*(float*)(row[f]+c*sizeof(float)));
*(float*)((char*)(row[f])+c*sizeof(float)));
}
break;
@ -235,9 +238,9 @@ static int print_result(char *tbname, TAOS_RES* res, int block) {
if (offsets) {
for (int c = 0; c < rows; c++) {
if (offsets[c] != -1) {
int length = *(int16_t*)(row[f] + offsets[c]);
int length = *(int16_t*)((char*)(row[f]) + offsets[c]);
char *buf = calloc(1, length + 1);
strncpy(buf, (char *)(row[f] + offsets[c] + 2), length);
strncpy(buf, (char *)((char*)(row[f]) + offsets[c] + 2), length);
printf("row: %d, col: %d, offset: %d, length: %d, content: %s\n",
c, f, offsets[c], length, buf);
free(buf);
@ -314,8 +317,9 @@ int main(int argc, char *argv[]) {
const char* host = "127.0.0.1";
const char* user = "root";
const char* passwd = "taosdata";
#ifndef WINDOWS
argp_parse(&argp, argc, argv, 0, 0, NULL);
#endif
TAOS* taos = taos_connect(host, user, passwd, "", 0);
if (taos == NULL) {
printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos));

3
example/src/tmq.c
View File

@ -101,8 +101,7 @@ int32_t create_topic() {
}
taos_free_result(pRes);
/*const char* sql = "select * from tu1";*/
/*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql));*/
/*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");
if (taos_errno(pRes) != 0) {
printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));

13
include/client/taos.h
View File

@ -216,13 +216,14 @@ typedef struct tmq_topic_vgroup_list_t tmq_topic_vgroup_list_t;
typedef struct tmq_conf_t tmq_conf_t;
typedef struct tmq_list_t tmq_list_t;
// typedef struct tmq_message_t tmq_message_t;
typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *));
DLL_EXPORT tmq_list_t *tmq_list_new();
DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *);
DLL_EXPORT void tmq_list_destroy(tmq_list_t *);
DLL_EXPORT int32_t tmq_list_get_size(const tmq_list_t *);
DLL_EXPORT char **tmq_list_to_c_array(const tmq_list_t *);
#if 0
DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen);
@ -262,12 +263,6 @@ DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const
DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf);
DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb);
#if 0
// temporary used function for demo only
void tmqShowMsg(tmq_message_t *tmq_message);
int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message);
#endif
/* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */
DLL_EXPORT char *tmq_get_topic_name(TAOS_RES *res);
@ -278,12 +273,8 @@ DLL_EXPORT char *tmq_get_block_table_name(TAOS_RES *res);
#endif
#if 0
DLL_EXPORT TAOS_ROW tmq_get_row(tmq_message_t *message);
DLL_EXPORT int64_t tmq_get_request_offset(tmq_message_t *message);
DLL_EXPORT int64_t tmq_get_response_offset(tmq_message_t *message);
DLL_EXPORT TAOS_FIELD *tmq_get_fields(tmq_t *tmq, const char *topic);
DLL_EXPORT int32_t tmq_field_count(tmq_t *tmq, const char *topic);
DLL_EXPORT void tmq_message_destroy(TAOS_RES *res);
#endif
/* --------------------TMPORARY INTERFACE FOR TESTING--------------------- */
#if 0

3
include/common/tglobal.h
View File

@ -94,6 +94,9 @@ extern bool tsRetrieveBlockingModel; // retrieve threads will be blocked
extern bool tsKeepOriginalColumnName;
extern bool tsDeadLockKillQuery;
// query client
extern int32_t tsQueryPolicy;
// client
extern int32_t tsMaxWildCardsLen;
extern int32_t tsMaxRegexStringLen;

20
include/common/tmsg.h
View File

@ -326,6 +326,13 @@ int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp);
int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp);
void* taosDecodeSEpSet(const void* buf, SEpSet* pEp);
typedef struct {
SEpSet epSet;
} SMEpSet;
int32_t tSerializeSMEpSet(void* buf, int32_t bufLen, SMEpSet* pReq);
int32_t tDeserializeSMEpSet(void* buf, int32_t buflen, SMEpSet* pReq);
typedef struct {
int8_t connType;
int32_t pid;
@ -632,13 +639,19 @@ int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq);
typedef struct {
SArray* epSetList; // SArray<SEpSet>
SArray* addrsList; // SArray<SQueryNodeAddr>
} SQnodeListRsp;
int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp);
void tFreeSQnodeListRsp(SQnodeListRsp* pRsp);
typedef struct SQueryNodeAddr {
int32_t nodeId; // vgId or qnodeId
SEpSet epSet;
} SQueryNodeAddr;
typedef struct {
SArray* pArray; // Array of SUseDbRsp
} SUseDbBatchRsp;
@ -1276,15 +1289,13 @@ typedef struct {
} SMVCreateStreamRsp, SMSCreateStreamRsp;
typedef struct {
char name[TSDB_TOPIC_FNAME_LEN];
char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic
int8_t igExists;
int8_t withTbName;
int8_t withSchema;
int8_t withTag;
int8_t withTagSchema;
char* sql;
char* ast;
int64_t subDbUid;
char subscribeDbName[TSDB_DB_NAME_LEN];
} SCMCreateTopicReq;
@ -2725,6 +2736,7 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p
}
return buf;
}
#pragma pack(pop)
#ifdef __cplusplus

19
include/common/tmsgdef.h
View File

@ -175,11 +175,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", SMqSetCVgReq, SMqSetCVgRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_REB, "vnode-mq-mv-rebalance", SMqMVRebReq, SMqMVRebRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CANCEL_CONN, "vnode-mq-mv-cancel-conn", SMqCancelConnReq, SMqCancelConnRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_TASK, "vnode-cancel-task", NULL, NULL)
@ -205,6 +201,21 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
// sync integration
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)

2
include/common/trow.h
View File

@ -1359,7 +1359,7 @@ static void tdSRowPrint(STSRow *row, STSchema *pSchema) {
printf(">>>");
for (int i = 0; i < pSchema->numOfCols; ++i) {
STColumn *stCol = pSchema->columns + i;
SCellVal sVal = {.valType = 255, .val = NULL};
SCellVal sVal = { 255, NULL};
if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) {
break;
}

178
include/common/ttokendef.h
View File

@ -156,87 +156,96 @@
#define TK_INTERVAL 138
#define TK_TOPIC 139
#define TK_AS 140
#define TK_DESC 141
#define TK_DESCRIBE 142
#define TK_RESET 143
#define TK_QUERY 144
#define TK_EXPLAIN 145
#define TK_ANALYZE 146
#define TK_VERBOSE 147
#define TK_NK_BOOL 148
#define TK_RATIO 149
#define TK_COMPACT 150
#define TK_VNODES 151
#define TK_IN 152
#define TK_OUTPUTTYPE 153
#define TK_AGGREGATE 154
#define TK_BUFSIZE 155
#define TK_STREAM 156
#define TK_INTO 157
#define TK_TRIGGER 158
#define TK_AT_ONCE 159
#define TK_WINDOW_CLOSE 160
#define TK_WATERMARK 161
#define TK_KILL 162
#define TK_CONNECTION 163
#define TK_MERGE 164
#define TK_VGROUP 165
#define TK_REDISTRIBUTE 166
#define TK_SPLIT 167
#define TK_SYNCDB 168
#define TK_NULL 169
#define TK_NK_QUESTION 170
#define TK_NK_ARROW 171
#define TK_ROWTS 172
#define TK_TBNAME 173
#define TK_QSTARTTS 174
#define TK_QENDTS 175
#define TK_WSTARTTS 176
#define TK_WENDTS 177
#define TK_WDURATION 178
#define TK_CAST 179
#define TK_NOW 180
#define TK_TODAY 181
#define TK_TIMEZONE 182
#define TK_COUNT 183
#define TK_FIRST 184
#define TK_LAST 185
#define TK_LAST_ROW 186
#define TK_BETWEEN 187
#define TK_IS 188
#define TK_NK_LT 189
#define TK_NK_GT 190
#define TK_NK_LE 191
#define TK_NK_GE 192
#define TK_NK_NE 193
#define TK_MATCH 194
#define TK_NMATCH 195
#define TK_CONTAINS 196
#define TK_JOIN 197
#define TK_INNER 198
#define TK_SELECT 199
#define TK_DISTINCT 200
#define TK_WHERE 201
#define TK_PARTITION 202
#define TK_BY 203
#define TK_SESSION 204
#define TK_STATE_WINDOW 205
#define TK_SLIDING 206
#define TK_FILL 207
#define TK_VALUE 208
#define TK_NONE 209
#define TK_PREV 210
#define TK_LINEAR 211
#define TK_NEXT 212
#define TK_GROUP 213
#define TK_HAVING 214
#define TK_ORDER 215
#define TK_SLIMIT 216
#define TK_SOFFSET 217
#define TK_LIMIT 218
#define TK_OFFSET 219
#define TK_ASC 220
#define TK_NULLS 221
#define TK_WITH 141
#define TK_SCHEMA 142
#define TK_DESC 143
#define TK_DESCRIBE 144
#define TK_RESET 145
#define TK_QUERY 146
#define TK_EXPLAIN 147
#define TK_ANALYZE 148
#define TK_VERBOSE 149
#define TK_NK_BOOL 150
#define TK_RATIO 151
#define TK_COMPACT 152
#define TK_VNODES 153
#define TK_IN 154
#define TK_OUTPUTTYPE 155
#define TK_AGGREGATE 156
#define TK_BUFSIZE 157
#define TK_STREAM 158
#define TK_INTO 159
#define TK_TRIGGER 160
#define TK_AT_ONCE 161
#define TK_WINDOW_CLOSE 162
#define TK_WATERMARK 163
#define TK_KILL 164
#define TK_CONNECTION 165
#define TK_MERGE 166
#define TK_VGROUP 167
#define TK_REDISTRIBUTE 168
#define TK_SPLIT 169
#define TK_SYNCDB 170
#define TK_NULL 171
#define TK_NK_QUESTION 172
#define TK_NK_ARROW 173
#define TK_ROWTS 174
#define TK_TBNAME 175
#define TK_QSTARTTS 176
#define TK_QENDTS 177
#define TK_WSTARTTS 178
#define TK_WENDTS 179
#define TK_WDURATION 180
#define TK_CAST 181
#define TK_NOW 182
#define TK_TODAY 183
#define TK_TIMEZONE 184
#define TK_COUNT 185
#define TK_FIRST 186
#define TK_LAST 187
#define TK_LAST_ROW 188
#define TK_BETWEEN 189
#define TK_IS 190
#define TK_NK_LT 191
#define TK_NK_GT 192
#define TK_NK_LE 193
#define TK_NK_GE 194
#define TK_NK_NE 195
#define TK_MATCH 196
#define TK_NMATCH 197
#define TK_CONTAINS 198
#define TK_JOIN 199
#define TK_INNER 200
#define TK_SELECT 201
#define TK_DISTINCT 202
#define TK_WHERE 203
#define TK_PARTITION 204
#define TK_BY 205
#define TK_SESSION 206
#define TK_STATE_WINDOW 207
#define TK_SLIDING 208
#define TK_FILL 209
#define TK_VALUE 210
#define TK_NONE 211
#define TK_PREV 212
#define TK_LINEAR 213
#define TK_NEXT 214
#define TK_GROUP 215
#define TK_HAVING 216
#define TK_ORDER 217
#define TK_SLIMIT 218
#define TK_SOFFSET 219
#define TK_LIMIT 220
#define TK_OFFSET 221
#define TK_ASC 222
#define TK_NULLS 223
#define TK_ID 224
#define TK_NK_BITNOT 225
#define TK_INSERT 226
#define TK_VALUES 227
#define TK_IMPORT 228
#define TK_NK_SEMI 229
#define TK_FILE 230
#define TK_NK_SPACE 300
#define TK_NK_COMMENT 301
@ -244,13 +253,6 @@
#define TK_NK_HEX 303 // hex number 0x123
#define TK_NK_OCT 304 // oct number
#define TK_NK_BIN 305 // bin format data 0b111
#define TK_NK_FILE 306
#define TK_NK_BITNOT 501
#define TK_INSERT 502
#define TK_VALUES 507
#define TK_IMPORT 509
#define TK_NK_SEMI 508
#define TK_NK_NIL 65535

3
include/dnode/bnode/bnode.h
View File

@ -26,6 +26,9 @@ extern "C" {
typedef struct SBnode SBnode;
typedef struct {
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
} SBnodeLoad;
typedef struct {

1
include/libs/catalog/catalog.h
View File

@ -213,7 +213,6 @@ int32_t catalogGetTableHashVgroup(SCatalog* pCatalog, void * pTransporter, const
*/
int32_t catalogGetAllMeta(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SCatalogReq* pReq, SMetaData* pRsp);
int32_t catalogGetQnodeList(SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, SArray* pQnodeList);
int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num);

4
include/libs/function/function.h
View File

@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef void (*FExecFinalize)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef struct SScalarFuncExecFuncs {
@ -113,7 +113,7 @@ typedef struct SResultRowEntryInfo {
bool initialized:1; // output buffer has been initialized
bool complete:1; // query has completed
uint8_t isNullRes:6; // the result is null
uint8_t numOfRes; // num of output result in current buffer
uint8_t numOfRes; // num of output result in current buffer
} SResultRowEntryInfo;
// determine the real data need to calculated the result

8
include/libs/nodes/cmdnodes.h
View File

@ -241,12 +241,20 @@ typedef struct SDropComponentNodeStmt {
int32_t dnodeId;
} SDropComponentNodeStmt;
typedef struct STopicOptions {
ENodeType type;
bool withTable;
bool withSchema;
bool withTag;
} STopicOptions;
typedef struct SCreateTopicStmt {
ENodeType type;
char topicName[TSDB_TABLE_NAME_LEN];
char subscribeDbName[TSDB_DB_NAME_LEN];
bool ignoreExists;
SNode* pQuery;
STopicOptions* pOptions;
} SCreateTopicStmt;
typedef struct SDropTopicStmt {

1
include/libs/nodes/nodes.h
View File

@ -84,6 +84,7 @@ typedef enum ENodeType {
QUERY_NODE_INDEX_OPTIONS,
QUERY_NODE_EXPLAIN_OPTIONS,
QUERY_NODE_STREAM_OPTIONS,
QUERY_NODE_TOPIC_OPTIONS,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR,

4
include/libs/nodes/querynodes.h
View File

@ -237,8 +237,8 @@ typedef struct SSelectStmt {
SNodeList* pGroupByList; // SGroupingSetNode
SNode* pHaving;
SNodeList* pOrderByList; // SOrderByExprNode
SNode* pLimit;
SNode* pSlimit;
SLimitNode* pLimit;
SLimitNode* pSlimit;
char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision;
bool isEmptyResult;

2
include/libs/planner/planner.h
View File

@ -35,6 +35,8 @@ typedef struct SPlanContext {
int8_t triggerType;
int64_t watermark;
bool isStmtQuery;
void* pTransporter;
struct SCatalog* pCatalog;
} SPlanContext;
// Create the physical plan for the query, according to the AST.

10
include/libs/qcom/query.h
View File

@ -51,6 +51,10 @@ typedef struct STableComInfo {
typedef struct SIndexMeta {
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
} SIndexMeta;
@ -129,12 +133,6 @@ typedef struct SMsgSendInfo {
SDataBuf msgInfo;
} SMsgSendInfo;
typedef struct SQueryNodeAddr {
int32_t nodeId; // vgId or qnodeId
SEpSet epSet;
} SQueryNodeAddr;
typedef struct SQueryNodeStat {
int32_t tableNum; // vg table number, unit is TSDB_TABLE_NUM_UNIT
} SQueryNodeStat;

128
include/libs/sync/sync.h
View File

@ -20,11 +20,14 @@
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <tdatablock.h>
#include "taosdef.h"
#include "trpc.h"
#include "wal.h"
//#include <tdatablock.h>
#include "cJSON.h"
#include "tdef.h"
//#include "taosdef.h"
//#include "trpc.h"
//#include "wal.h"
typedef uint64_t SyncNodeId;
typedef int32_t SyncGroupId;
@ -38,14 +41,9 @@ typedef enum {
TAOS_SYNC_STATE_ERROR = 103,
} ESyncState;
typedef struct SSyncBuffer {
void* data;
size_t len;
} SSyncBuffer;
typedef struct SNodeInfo {
uint16_t nodePort; // node sync Port
char nodeFqdn[TSDB_FQDN_LEN]; // node FQDN
uint16_t nodePort;
char nodeFqdn[TSDB_FQDN_LEN];
} SNodeInfo;
typedef struct SSyncCfg {
@ -54,47 +52,46 @@ typedef struct SSyncCfg {
SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
} SSyncCfg;
typedef struct SNodesRole {
int32_t replicaNum;
SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
ESyncState role[TSDB_MAX_REPLICA];
} SNodesRole;
// abstract definition of snapshot
typedef struct SSnapshot {
void* data;
SyncIndex lastApplyIndex;
SyncTerm lastApplyTerm;
} SSnapshot;
typedef enum {
TAOS_SYNC_FSM_CB_SUCCESS = 0,
TAOS_SYNC_FSM_CB_OTHER_ERROR,
} ESyncFsmCbCode;
typedef struct SFsmCbMeta {
SyncIndex index;
bool isWeak;
int32_t code;
ESyncState state;
uint64_t seqNum;
} SFsmCbMeta;
struct SRpcMsg;
typedef struct SRpcMsg SRpcMsg;
typedef struct SSyncFSM {
void* data;
// when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
// user can do something according to the code and isWeak. for example, write data into tsdb
void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
ESyncState state);
void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
// when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
// user can do something according to the code and isWeak. for example, write data into tsdb
void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
ESyncState state);
// when log entry is updated by a new one, FpRollBackCb is called
// user can do something to roll back. for example, delete data from tsdb, or just ignore it
void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
ESyncState state);
// user should implement this function, use "data" to take snapshot into "snapshot"
int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
// user should implement this function, restore "data" from "snapshot"
int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
} SSyncFSM;
struct SSyncRaftEntry;
typedef struct SSyncRaftEntry SSyncRaftEntry;
#define SYNC_INDEX_BEGIN 0
#define SYNC_INDEX_INVALID -1
// abstract definition of log store in raft
// SWal implements it
typedef struct SSyncLogStore {
@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
} SSyncLogStore;
// raft need to persist two variables in storage: currentTerm, voteFor
typedef struct SStateMgr {
void* data;
struct SWal;
typedef struct SWal SWal;
int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
} SStateMgr;
struct SEpSet;
typedef struct SEpSet SEpSet;
typedef struct SSyncInfo {
SyncGroupId vgId;
@ -152,27 +140,35 @@ typedef struct SSyncInfo {
} SSyncInfo;
struct SSyncNode;
typedef struct SSyncNode SSyncNode;
int32_t syncInit();
void syncCleanUp();
int64_t syncOpen(const SSyncInfo* pSyncInfo);
void syncStart(int64_t rid);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
ESyncState syncGetMyRole(int64_t rid);
const char* syncGetMyRoleStr(int64_t rid);
SyncTerm syncGetMyTerm(int64_t rid);
int32_t syncInit();
void syncCleanUp();
typedef enum {
TAOS_SYNC_PROPOSE_SUCCESS = 0,
TAOS_SYNC_PROPOSE_NOT_LEADER,
TAOS_SYNC_PROPOSE_OTHER_ERROR,
} ESyncProposeCode;
int64_t syncStart(const SSyncInfo* pSyncInfo);
void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
ESyncState syncGetMyRole(int64_t rid);
// propose with sequence number, to implement linearizable semantics
int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
// for compatibility, the same as syncPropose
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
extern int32_t sDebugFlag;
const char *syncStr(ESyncState state);
//-----------------------------------------
struct SSyncNode;
typedef struct SSyncNode SSyncNode;
struct SSyncBuffer;
typedef struct SSyncBuffer SSyncBuffer;
//-----------------------------------------
const char* syncStr(ESyncState state);
#ifdef __cplusplus
}

414
include/libs/sync/syncTools.h Normal file
View File

@ -0,0 +1,414 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_TOOLS_H
#define _TD_LIBS_SYNC_TOOLS_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
//#include <tdatablock.h>
#include "cJSON.h"
//#include "taosdef.h"
#include "trpc.h"
//#include "wal.h"
// ------------------ ds -------------------
typedef struct SRaftId {
SyncNodeId addr;
SyncGroupId vgId;
} SRaftId;
// ------------------ control -------------------
struct SSyncNode;
typedef struct SSyncNode SSyncNode;
SSyncNode* syncNodeAcquire(int64_t rid);
void syncNodeRelease(SSyncNode* pNode);
int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
void syncSetQ(int64_t rid, void* queueHandle);
void syncSetRpc(int64_t rid, void* rpcHandle);
char* sync2SimpleStr(int64_t rid);
// set timer ms
void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
void setElectTimerMS(int64_t rid, int32_t electTimerMS);
void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
// for compatibility, the same as syncPropose
int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
// utils
const char* syncUtilState2String(ESyncState state);
// ------------------ for debug -------------------
void syncRpcMsgPrint(SRpcMsg* pMsg);
void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
void syncRpcMsgLog(SRpcMsg* pMsg);
void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
// ------------------ for compile -------------------
typedef struct SSyncBuffer {
void* data;
size_t len;
} SSyncBuffer;
typedef struct SNodesRole {
int32_t replicaNum;
SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
ESyncState role[TSDB_MAX_REPLICA];
} SNodesRole;
typedef struct SStateMgr {
void* data;
int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
} SStateMgr;
// ------------------ for message process -------------------
// ---------------------------------------------
typedef struct SyncPing {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
uint32_t dataLen;
char data[];
} SyncPing;
SyncPing* syncPingBuild(uint32_t dataLen);
SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
void syncPingDestroy(SyncPing* pMsg);
void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
int32_t syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
void syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncPing2Json(const SyncPing* pMsg);
char* syncPing2Str(const SyncPing* pMsg);
// for debug ----------------------
void syncPingPrint(const SyncPing* pMsg);
void syncPingPrint2(char* s, const SyncPing* pMsg);
void syncPingLog(const SyncPing* pMsg);
void syncPingLog2(char* s, const SyncPing* pMsg);
// ---------------------------------------------
typedef struct SyncPingReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
uint32_t dataLen;
char data[];
} SyncPingReply;
SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
void syncPingReplyDestroy(SyncPingReply* pMsg);
void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
int32_t syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
void syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncPingReply2Json(const SyncPingReply* pMsg);
char* syncPingReply2Str(const SyncPingReply* pMsg);
// for debug ----------------------
void syncPingReplyPrint(const SyncPingReply* pMsg);
void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
void syncPingReplyLog(const SyncPingReply* pMsg);
void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
// ---------------------------------------------
typedef enum ESyncTimeoutType {
SYNC_TIMEOUT_PING = 100,
SYNC_TIMEOUT_ELECTION,
SYNC_TIMEOUT_HEARTBEAT,
} ESyncTimeoutType;
typedef struct SyncTimeout {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
ESyncTimeoutType timeoutType;
uint64_t logicClock;
int32_t timerMS;
void* data; // need optimized
} SyncTimeout;
SyncTimeout* syncTimeoutBuild();
SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
void* data);
void syncTimeoutDestroy(SyncTimeout* pMsg);
void syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
void syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
void syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncTimeout2Json(const SyncTimeout* pMsg);
char* syncTimeout2Str(const SyncTimeout* pMsg);
// for debug ----------------------
void syncTimeoutPrint(const SyncTimeout* pMsg);
void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
void syncTimeoutLog(const SyncTimeout* pMsg);
void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
// ---------------------------------------------
typedef struct SyncClientRequest {
uint32_t bytes;
int32_t vgId;
uint32_t msgType; // SyncClientRequest msgType
uint32_t originalRpcType; // user RpcMsg msgType
uint64_t seqNum;
bool isWeak;
uint32_t dataLen; // user RpcMsg.contLen
char data[]; // user RpcMsg.pCont
} SyncClientRequest;
SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
int32_t vgId); // step 1
void syncClientRequestDestroy(SyncClientRequest* pMsg);
void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
void syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg); // step 2
void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg); // step 3
cJSON* syncClientRequest2Json(const SyncClientRequest* pMsg);
char* syncClientRequest2Str(const SyncClientRequest* pMsg);
// for debug ----------------------
void syncClientRequestPrint(const SyncClientRequest* pMsg);
void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
void syncClientRequestLog(const SyncClientRequest* pMsg);
void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
// ---------------------------------------------
typedef struct SyncClientRequestReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
int32_t errCode;
SRaftId leaderHint;
} SyncClientRequestReply;
// ---------------------------------------------
typedef struct SyncRequestVote {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
SyncIndex lastLogIndex;
SyncTerm lastLogTerm;
} SyncRequestVote;
SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
void syncRequestVoteDestroy(SyncRequestVote* pMsg);
void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
void syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncRequestVote2Json(const SyncRequestVote* pMsg);
char* syncRequestVote2Str(const SyncRequestVote* pMsg);
// for debug ----------------------
void syncRequestVotePrint(const SyncRequestVote* pMsg);
void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
void syncRequestVoteLog(const SyncRequestVote* pMsg);
void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
// ---------------------------------------------
typedef struct SyncRequestVoteReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
bool voteGranted;
} SyncRequestVoteReply;
SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
void syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
char* syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
// for debug ----------------------
void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
// ---------------------------------------------
typedef struct SyncAppendEntries {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
SyncIndex prevLogIndex;
SyncTerm prevLogTerm;
SyncIndex commitIndex;
uint32_t dataLen;
char data[];
} SyncAppendEntries;
SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
void syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
void syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncAppendEntries2Json(const SyncAppendEntries* pMsg);
char* syncAppendEntries2Str(const SyncAppendEntries* pMsg);
// for debug ----------------------
void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
// ---------------------------------------------
typedef struct SyncAppendEntriesReply {
uint32_t bytes;
int32_t vgId;
uint32_t msgType;
SRaftId srcId;
SRaftId destId;
// private data
SyncTerm term;
bool success;
SyncIndex matchIndex;
} SyncAppendEntriesReply;
SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
void syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
char* syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
// for debug ----------------------
void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
// ---------------------------------------------
typedef struct SyncApplyMsg {
uint32_t bytes;
int32_t vgId;
uint32_t msgType; // user SyncApplyMsg msgType
uint32_t originalRpcType; // user RpcMsg msgType
SFsmCbMeta fsmMeta;
uint32_t dataLen; // user RpcMsg.contLen
char data[]; // user RpcMsg.pCont
} SyncApplyMsg;
SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
void syncApplyMsgDestroy(SyncApplyMsg* pMsg);
void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg); // SyncApplyMsg to SRpcMsg, put it into ApplyQ
void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg); // get SRpcMsg from ApplyQ, to SyncApplyMsg
SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg); // SyncApplyMsg to OriginalRpcMsg
cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
char* syncApplyMsg2Str(const SyncApplyMsg* pMsg);
// for debug ----------------------
void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
void syncApplyMsgLog(const SyncApplyMsg* pMsg);
void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
// on message ----------------------
int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
// ---------------------------------------------
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_TOOLS_H*/

32
include/libs/transport/trpc.h
View File

@ -38,11 +38,11 @@ typedef struct SRpcConnInfo {
typedef struct SRpcMsg {
tmsg_t msgType;
void * pCont;
void *pCont;
int contLen;
int32_t code;
void * handle; // rpc handle returned to app
void * ahandle; // app handle set by client
void *handle; // rpc handle returned to app
void *ahandle; // app handle set by client
int noResp; // has response or not(default 0, 0: resp, 1: no resp);
int persistHandle; // persist handle or not
@ -60,10 +60,11 @@ typedef struct {
typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, char *secret, char *ckey);
typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
typedef struct SRpcInit {
uint16_t localPort; // local port
char * label; // for debug purpose
char *label; // for debug purpose
int numOfThreads; // number of threads to handle connections
int sessions; // number of sessions allowed
int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
@ -80,36 +81,39 @@ typedef struct SRpcInit {
RpcCfp cfp;
// call back to retrieve the client auth info, for server app only
RpcAfp afp;;
RpcAfp afp;
// user defined retry func
RpcRfp rfp;
void *parent;
} SRpcInit;
typedef struct {
void *val;
void *val;
int32_t (*clone)(void *src, void **dst);
void (*freeFunc)(const void *arg);
void (*freeFunc)(const void *arg);
} SRpcCtxVal;
typedef struct {
int32_t msgType;
void *val;
int32_t msgType;
void *val;
int32_t (*clone)(void *src, void **dst);
void (*freeFunc)(const void *arg);
void (*freeFunc)(const void *arg);
} SRpcBrokenlinkVal;
typedef struct {
SHashObj * args;
SHashObj *args;
SRpcBrokenlinkVal brokenVal;
} SRpcCtx;
int32_t rpcInit();
void rpcCleanup();
void * rpcOpen(const SRpcInit *pRpc);
void *rpcOpen(const SRpcInit *pRpc);
void rpcClose(void *);
void * rpcMallocCont(int contLen);
void *rpcMallocCont(int contLen);
void rpcFreeCont(void *pCont);
void * rpcReallocCont(void *ptr, int contLen);
void *rpcReallocCont(void *ptr, int contLen);
// Because taosd supports multi-process mode
// These functions should not be used on the server side

10
include/os/os.h
View File

@ -22,7 +22,6 @@ extern "C" {
#include <assert.h>
#include <ctype.h>
#include <semaphore.h>
#include <regex.h>
@ -51,6 +50,15 @@ extern "C" {
#include <sys/prctl.h>
#include <argp.h>
#endif
#else
#include <time.h>
#ifndef TD_USE_WINSOCK
#include <winsock2.h>
#else
#include <winsock.h>
#endif
#endif

8
include/os/osAtomic.h
View File

@ -63,22 +63,22 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_add_fetch_ptr(void *ptr, int32_t val);
void* atomic_add_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_add_ptr(void *ptr, int32_t val);
void* atomic_fetch_add_ptr(void *ptr, void *val);
int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
void* atomic_sub_fetch_ptr(void *ptr, int32_t val);
void* atomic_sub_fetch_ptr(void *ptr, void *val);
int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val);
void* atomic_fetch_sub_ptr(void *ptr, int32_t val);
void* atomic_fetch_sub_ptr(void *ptr, void *val);
int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val);
int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val);
int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val);

2
include/os/osDef.h
View File

@ -191,7 +191,7 @@ extern "C" {
#define threadlocal __declspec( thread )
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#ifdef WINDOWS
#define PRIzu "ld"
#else
#define PRIzu "zu"

6
include/os/osDir.h
View File

@ -36,12 +36,12 @@ typedef struct TdDirEntry *TdDirEntryPtr;
void taosRemoveDir(const char *dirname);
bool taosDirExist(char *dirname);
bool taosDirExist(const char *dirname);
int32_t taosMkDir(const char *dirname);
int32_t taosMulMkDir(const char *dirname);
void taosRemoveOldFiles(const char *dirname, int32_t keepDays);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
int32_t taosRealPath(char *dirname, int32_t maxlen);
int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen);
bool taosIsDir(const char *dirname);
char* taosDirName(char *dirname);
char* taosDirEntryBaseName(char *dirname);
@ -50,7 +50,7 @@ TdDirPtr taosOpenDir(const char *dirname);
TdDirEntryPtr taosReadDir(TdDirPtr pDir);
bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
char* taosGetDirEntryName(TdDirEntryPtr pDirEntry);
int32_t taosCloseDir(TdDirPtr pDir);
int32_t taosCloseDir(TdDirPtr *ppDir);
#ifdef __cplusplus
}

1
include/os/osEnv.h
View File

@ -34,6 +34,7 @@ extern int64_t tsOpenMax;
extern int64_t tsStreamMax;
extern float tsNumOfCores;
extern int64_t tsTotalMemoryKB;
extern char* tsProcPath;
extern char configDir[];
extern char tsDataDir[];

1
include/os/osFile.h
View File

@ -93,7 +93,6 @@ void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix,
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
void *taosMmapReadOnlyFile(TdFilePtr pFile, int64_t length);
bool taosValidFile(TdFilePtr pFile);
int32_t taosGetErrorFile(TdFilePtr pFile);

2
include/os/osLz4.h
View File

@ -20,7 +20,7 @@
extern "C" {
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#ifdef WINDOWS
int32_t BUILDIN_CLZL(uint64_t val);
int32_t BUILDIN_CLZ(uint32_t val);
int32_t BUILDIN_CTZL(uint64_t val);

3
include/os/osMath.h
View File

@ -23,7 +23,7 @@ extern "C" {
#define TPOW2(x) ((x) * (x))
#define TABS(x) ((x) > 0 ? (x) : -(x))
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#ifdef WINDOWS
#define TSWAP(a, b, c) \
do { \
@ -33,6 +33,7 @@ extern "C" {
} while (0)
#define TMAX(a, b) (((a) > (b)) ? (a) : (b))
#define TMIN(a, b) (((a) < (b)) ? (a) : (b))
#define TRANGE(aa, bb, cc) ((aa) = TMAX((aa), (bb)),(aa) = TMIN((aa), (cc)))
#else

45
include/os/osSemaphore.h
View File

@ -22,21 +22,28 @@ extern "C" {
#include <semaphore.h>
#if defined (_TD_DARWIN_64)
typedef struct tsem_s *tsem_t;
int tsem_init(tsem_t *sem, int pshared, unsigned int value);
int tsem_wait(tsem_t *sem);
int tsem_post(tsem_t *sem);
int tsem_destroy(tsem_t *sem);
#if defined(_TD_DARWIN_64)
typedef struct tsem_s *tsem_t;
int tsem_init(tsem_t *sem, int pshared, unsigned int value);
int tsem_wait(tsem_t *sem);
int tsem_timewait(tsem_t *sim, int64_t nanosecs);
int tsem_post(tsem_t *sem);
int tsem_destroy(tsem_t *sem);
#else
#define tsem_t sem_t
#define tsem_init sem_init
int tsem_wait(tsem_t* sem);
#define tsem_post sem_post
#define tsem_destroy sem_destroy
#define tsem_t sem_t
#define tsem_init sem_init
int tsem_wait(tsem_t *sem);
int tsem_timewait(tsem_t *sim, int64_t nanosecs);
#define tsem_post sem_post
#define tsem_destroy sem_destroy
#endif
#if defined (_TD_DARWIN_64)
#if defined(_TD_DARWIN_64)
// #define TdThreadRwlock TdThreadMutex
// #define taosThreadRwlockInit(lock, NULL) taosThreadMutexInit(lock, NULL)
// #define taosThreadRwlockDestroy(lock) taosThreadMutexDestroy(lock)
@ -44,20 +51,20 @@ extern "C" {
// #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
// #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
#define TdThreadSpinlock TdThreadMutex
#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
#define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
#define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
#define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
#define TdThreadSpinlock TdThreadMutex
#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
#define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
#define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
#define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
#endif
bool taosCheckPthreadValid(TdThread thread);
int64_t taosGetSelfPthreadId();
int64_t taosGetPthreadId(TdThread thread);
void taosResetPthread(TdThread* thread);
void taosResetPthread(TdThread *thread);
bool taosComparePthread(TdThread first, TdThread second);
int32_t taosGetPId();
int32_t taosGetAppName(char* name, int32_t* len);
int32_t taosGetAppName(char *name, int32_t *len);
#ifdef __cplusplus
}

114
include/os/osSocket.h
View File

@ -19,32 +19,53 @@
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC
#define socket SOCKET_FUNC_TAOS_FORBID
#define bind BIND_FUNC_TAOS_FORBID
#define listen LISTEN_FUNC_TAOS_FORBID
#define accept ACCEPT_FUNC_TAOS_FORBID
#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
#define inet_addr INET_ADDR_FUNC_TAOS_FORBID
#define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
#define socket SOCKET_FUNC_TAOS_FORBID
#define bind BIND_FUNC_TAOS_FORBID
#define listen LISTEN_FUNC_TAOS_FORBID
#define accept ACCEPT_FUNC_TAOS_FORBID
#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
#define inet_addr INET_ADDR_FUNC_TAOS_FORBID
#define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID
#endif
#if defined(WINDOWS)
#include "winsock2.h"
#include <WS2tcpip.h>
#include <winbase.h>
#include <Winsock2.h>
#else
#include <netinet/in.h>
#include <sys/socket.h>
#if BYTE_ORDER == LITTLE_ENDIAN
#include <stdlib.h>
#define htobe16(x) _byteswap_ushort(x)
#define htole16(x) (x)
#define be16toh(x) _byteswap_ushort(x)
#define le16toh(x) (x)
#define htobe32(x) _byteswap_ulong(x)
#define htole32(x) (x)
#define be32toh(x) _byteswap_ulong(x)
#define le32toh(x) (x)
#define htobe64(x) _byteswap_uint64(x)
#define htole64(x) (x)
#define be64toh(x) _byteswap_uint64(x)
#define le64toh(x) (x)
#else
#error byte order not supported
#endif
#if defined(_TD_DARWIN_64)
#include <osEok.h>
#else
#include <netinet/in.h>
#include <sys/epoll.h>
#endif
#define __BYTE_ORDER BYTE_ORDER
#define __BIG_ENDIAN BIG_ENDIAN
#define __LITTLE_ENDIAN LITTLE_ENDIAN
#define __PDP_ENDIAN PDP_ENDIAN
#else
#include <netinet/in.h>
#include <sys/socket.h>
#if defined(_TD_DARWIN_64)
#include <osEok.h>
#else
#include <netinet/in.h>
#include <sys/epoll.h>
#endif
#endif
#ifdef __cplusplus
@ -52,14 +73,10 @@ extern "C" {
#endif
#if defined(WINDOWS)
#define htobe64 htonll
#endif
#if defined(WINDOWS)
typedef int socklen_t;
#define TAOS_EPOLL_WAIT_TIME 100
typedef SOCKET eventfd_t;
#define eventfd(a, b) -1
typedef SOCKET EpollFd;
#define EpollClose(pollFd) epoll_close(pollFd)
#ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29)
@ -79,31 +96,29 @@ extern "C" {
#if defined(_TD_DARWIN_64)
// #define htobe64 htonll
# include <libkern/OSByteOrder.h>
#include <libkern/OSByteOrder.h>
# define htobe16(x) OSSwapHostToBigInt16(x)
# define htole16(x) OSSwapHostToLittleInt16(x)
# define be16toh(x) OSSwapBigToHostInt16(x)
# define le16toh(x) OSSwapLittleToHostInt16(x)
#define htobe16(x) OSSwapHostToBigInt16(x)
#define htole16(x) OSSwapHostToLittleInt16(x)
#define be16toh(x) OSSwapBigToHostInt16(x)
#define le16toh(x) OSSwapLittleToHostInt16(x)
# define htobe32(x) OSSwapHostToBigInt32(x)
# define htole32(x) OSSwapHostToLittleInt32(x)
# define be32toh(x) OSSwapBigToHostInt32(x)
# define le32toh(x) OSSwapLittleToHostInt32(x)
#define htobe32(x) OSSwapHostToBigInt32(x)
#define htole32(x) OSSwapHostToLittleInt32(x)
#define be32toh(x) OSSwapBigToHostInt32(x)
#define le32toh(x) OSSwapLittleToHostInt32(x)
# define htobe64(x) OSSwapHostToBigInt64(x)
# define htole64(x) OSSwapHostToLittleInt64(x)
# define be64toh(x) OSSwapBigToHostInt64(x)
# define le64toh(x) OSSwapLittleToHostInt64(x)
#define htobe64(x) OSSwapHostToBigInt64(x)
#define htole64(x) OSSwapHostToLittleInt64(x)
#define be64toh(x) OSSwapBigToHostInt64(x)
#define le64toh(x) OSSwapLittleToHostInt64(x)
# define __BYTE_ORDER BYTE_ORDER
# define __BIG_ENDIAN BIG_ENDIAN
# define __LITTLE_ENDIAN LITTLE_ENDIAN
# define __PDP_ENDIAN PDP_ENDIAN
#define __BYTE_ORDER BYTE_ORDER
#define __BIG_ENDIAN BIG_ENDIAN
#define __LITTLE_ENDIAN LITTLE_ENDIAN
#define __PDP_ENDIAN PDP_ENDIAN
#endif
#define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SocketFd;
typedef SocketFd EpollFd;
@ -122,7 +137,7 @@ typedef struct TdEpoll *TdEpollPtr;
int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, int *addrLen);
int32_t taosCloseSocketNoCheck1(SocketFd fd);
int32_t taosCloseSocket(TdSocketPtr *ppSocket);
int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
@ -139,14 +154,15 @@ int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
void taosWinSocketInit();
TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen);
int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, int *addrLen);
void taosBlockSIGPIPE();
uint32_t taosGetIpv4FromFqdn(const char *);

2
include/os/osString.h
View File

@ -39,7 +39,7 @@ typedef int32_t TdUcs4;
#define wchar_t WCHAR_T_TYPE_TAOS_FORBID
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#ifdef WINDOWS
#define tstrdup(str) _strdup(str)
#else
#define tstrdup(str) strdup(str)

13
include/os/osThread.h
View File

@ -22,9 +22,12 @@
extern "C" {
#endif
#ifndef WINDOWS
#ifndef __USE_XOPEN2K
#define TD_USE_SPINLOCK_AS_MUTEX
typedef pthread_mutex_t pthread_spinlock_t;
#endif
#endif
typedef pthread_t TdThread;
typedef pthread_spinlock_t TdThreadSpinlock;
@ -41,6 +44,13 @@ typedef pthread_key_t TdThreadKey;
#define taosThreadCleanupPush pthread_cleanup_push
#define taosThreadCleanupPop pthread_cleanup_pop
#ifdef WINDOWS
#define TD_PTHREAD_MUTEX_INITIALIZER (TdThreadMutex)(-1)
#else
#define TD_PTHREAD_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
#endif
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC
@ -218,8 +228,7 @@ int32_t taosThreadSpinLock(TdThreadSpinlock * lock);
int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock);
int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock);
void taosThreadTestCancel(void);
int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset);
int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig);
void taosThreadClear(TdThread *thread);
#ifdef __cplusplus
}

8
include/os/osTime.h
View File

@ -20,8 +20,6 @@
extern "C" {
#endif
#include <time.h>
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section.
#ifndef ALLOW_FORBID_FUNC
@ -34,7 +32,7 @@ extern "C" {
#define mktime MKTIME_FUNC_TAOS_FORBID
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
#ifdef WINDOWS
#define CLOCK_REALTIME 0
@ -58,6 +56,8 @@ extern "C" {
int32_t taosGetTimeOfDay(struct timeval *tv);
int32_t taosClockGetTime(int clock_id, struct timespec *pTS);
//@return timestamp in second
int32_t taosGetTimestampSec();
@ -78,7 +78,7 @@ static FORCE_INLINE int64_t taosGetTimestampUs() {
//@return timestamp in nanosecond
static FORCE_INLINE int64_t taosGetTimestampNs() {
struct timespec systemTime = {0};
clock_gettime(CLOCK_REALTIME, &systemTime);
taosClockGetTime(CLOCK_REALTIME, &systemTime);
return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec;
}

8
include/util/taoserror.h
View File

@ -411,6 +411,10 @@ int32_t* taosGetErrno();
#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909)
#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A)
// sync integration
#define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x0910)
#define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF)
// tq
#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00)
#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01)
@ -615,6 +619,10 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL TAOS_DEF_ERROR_CODE(0, 0x2632)
#define TSDB_CODE_PAR_ONLY_ONE_JSON_TAG TAOS_DEF_ERROR_CODE(0, 0x2633)
#define TSDB_CODE_PAR_INCORRECT_NUM_OF_COL TAOS_DEF_ERROR_CODE(0, 0x2634)
#define TSDB_CODE_PAR_INCORRECT_TIMESTAMP_VAL TAOS_DEF_ERROR_CODE(0, 0x2635)
#define TSDB_CODE_PAR_INVALID_DAYS_VALUE TAOS_DEF_ERROR_CODE(0, 0x2636)
#define TSDB_CODE_PAR_OFFSET_LESS_ZERO TAOS_DEF_ERROR_CODE(0, 0x2637)
#define TSDB_CODE_PAR_SLIMIT_LEAK_PARTITION_BY TAOS_DEF_ERROR_CODE(0, 0x2638)
//planner
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)

16
include/util/tcache.h
View File

@ -47,13 +47,13 @@ typedef struct STrashElem STrashElem;
/**
* initialize the cache object
* @param keyType key type
* @param refreshTimeInSeconds refresh operation interval time, the maximum survival time when one element is expired
* @param refreshTimeInMs refresh operation interval time, the maximum survival time when one element is expired
* and not referenced by other objects
* @param extendLifespan auto extend lifespan, if accessed
* @param fn free resource callback function
* @return
*/
SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInMs, bool extendLifespan, __cache_free_fn_t fn,
const char *cacheName);
/**
@ -111,7 +111,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
* @param pCacheObj
* @return
*/
size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj);
/**
* move all data node into trash, clear node in trash can if it is not referenced by any clients
@ -145,11 +145,11 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
*/
void taosStopCacheRefreshWorker();
SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
bool taosCacheIterNext(SCacheIter* pIter);
void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
void taosCacheDestroyIter(SCacheIter* pIter);
SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj);
bool taosCacheIterNext(SCacheIter *pIter);
void *taosCacheIterGetData(const SCacheIter *pIter, size_t *dataLen);
void *taosCacheIterGetKey(const SCacheIter *pIter, size_t *keyLen);
void taosCacheDestroyIter(SCacheIter *pIter);
#ifdef __cplusplus
}

1
include/util/tdef.h
View File

@ -118,7 +118,6 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_INS_TABLE_USER_USERS "user_users"
#define TSDB_INS_TABLE_LICENCES "grants"
#define TSDB_INS_TABLE_VGROUPS "vgroups"
#define TSDB_INS_TABLE_TOPICS "topics"
#define TSDB_INS_TABLE_CONSUMERS "consumers"
#define TSDB_INS_TABLE_SUBSCRIBES "subscribes"
#define TSDB_INS_TABLE_TRANS "trans"

6
include/util/tencode.h
View File

@ -236,7 +236,8 @@ static FORCE_INLINE int32_t tEncodeFloat(SCoder* pEncoder, float val) {
union {
uint32_t ui;
float f;
} v = {.f = val};
} v;
v.f = val;
return tEncodeU32(pEncoder, v.ui);
}
@ -245,7 +246,8 @@ static FORCE_INLINE int32_t tEncodeDouble(SCoder* pEncoder, double val) {
union {
uint64_t ui;
double d;
} v = {.d = val};
} v;
v.d = val;
return tEncodeU64(pEncoder, v.ui);
}

4
include/util/tjson.h
View File

@ -22,6 +22,9 @@
extern "C" {
#endif
#ifdef WINDOWS
#define tjsonGetNumberValue(pJson, pName, val) -1
#else
#define tjsonGetNumberValue(pJson, pName, val) \
({ \
uint64_t _tmp = 0; \
@ -29,6 +32,7 @@ extern "C" {
val = _tmp; \
_code; \
})
#endif
typedef void SJson;

42
source/client/src/clientImpl.c
View File

@ -95,16 +95,16 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
if (initEpSetFromCfg(ip, NULL, &epSet) < 0) {
return NULL;
}
if (port) {
epSet.epSet.eps[0].port = port;
}
} else {
if (initEpSetFromCfg(tsFirst, tsSecond, &epSet) < 0) {
return NULL;
}
}
if (port) {
epSet.epSet.eps[0].port = port;
}
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL;
@ -226,14 +226,17 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQuery* pQuery) {
int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArray* pNodeList) {
pRequest->type = pQuery->msgType;
SPlanContext cxt = {.queryId = pRequest->requestId,
.acctId = pRequest->pTscObj->acctId,
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite};
int32_t code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
if (code != 0) {
return code;
SPlanContext cxt = {
.queryId = pRequest->requestId,
.acctId = pRequest->pTscObj->acctId,
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite,
.pTransporter = pRequest->pTscObj->pAppInfo->pTransporter
};
int32_t code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &cxt.pCatalog);
if (TSDB_CODE_SUCCESS == code) {
code = qCreateQueryPlan(&cxt, pPlan, pNodeList);
}
return code;
}
@ -302,8 +305,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList
}
SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery) {
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
if (TSDB_CODE_SUCCESS == code) {
switch (pQuery->execMode) {
case QUERY_EXEC_MODE_LOCAL:
@ -312,12 +313,15 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
case QUERY_EXEC_MODE_RPC:
code = execDdlQuery(pRequest, pQuery);
break;
case QUERY_EXEC_MODE_SCHEDULE:
case QUERY_EXEC_MODE_SCHEDULE: {
SArray* pNodeList = taosArrayInit(4, sizeof(struct SQueryNodeAddr));
code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList);
if (TSDB_CODE_SUCCESS == code) {
code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList);
}
taosArrayDestroy(pNodeList);
break;
}
case QUERY_EXEC_MODE_EMPTY_RESULT:
pRequest->type = TSDB_SQL_RETRIEVE_EMPTY_RESULT;
break;
@ -326,7 +330,6 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code
}
}
taosArrayDestroy(pNodeList);
if (!keepQuery) {
qDestroyQuery(pQuery);
}
@ -436,7 +439,12 @@ int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSe
return -1;
}
taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return terrno;
}
mgmtEpSet->numOfEps++;
}

3
source/client/src/clientMsgHandler.c
View File

@ -151,13 +151,14 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
taosMemoryFreeClear(output.dbVgroup);
tscError("failed to build use db output since %s", terrstr());
} else {
} else if (output.dbVgroup) {
struct SCatalog* pCatalog = NULL;
int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);
if (code1 != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pRequest->pTscObj->pAppInfo->clusterId,
tstrerror(code1));
taosMemoryFreeClear(output.dbVgroup);
} else {
catalogUpdateDBVgInfo(pCatalog, output.db, output.dbId, output.dbVgroup);
}

282
source/client/src/tmq.c
View File

@ -24,16 +24,6 @@
#include "tqueue.h"
#include "tref.h"
#if 0
struct tmq_message_t {
SMqPollRsp msg;
char* topic;
SArray* res; // SArray<SReqResultInfo>
int32_t vgId;
int32_t resIter;
};
#endif
typedef struct {
int8_t tmqRspType;
int32_t epoch;
@ -270,6 +260,16 @@ void tmq_list_destroy(tmq_list_t* list) {
taosArrayDestroy(container);
}
int32_t tmq_list_get_size(const tmq_list_t* list) {
const SArray* container = &list->container;
return taosArrayGetSize(container);
}
char** tmq_list_to_c_array(const tmq_list_t* list) {
const SArray* container = &list->container;
return container->pData;
}
static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) {
return sprintf(dst, "%s:%d", topicName, vg);
}
@ -397,7 +397,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->commit_cb = conf->commit_cb;
pTmq->resetOffsetCfg = conf->resetOffset;
pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1);
pTmq->consumerId = tGenIdPI64();
pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic));
if (pTmq->clientTopics == NULL) {
taosMemoryFree(pTmq);
@ -770,105 +770,12 @@ _return:
}
#endif
static char* formatTimestamp(char* buf, int64_t val, int precision) {
time_t tt;
int32_t ms = 0;
if (precision == TSDB_TIME_PRECISION_NANO) {
tt = (time_t)(val / 1000000000);
ms = val % 1000000000;
} else if (precision == TSDB_TIME_PRECISION_MICRO) {
tt = (time_t)(val / 1000000);
ms = val % 1000000;
} else {
tt = (time_t)(val / 1000);
ms = val % 1000;
}
/* comment out as it make testcases like select_with_tags.sim fail.
but in windows, this may cause the call to localtime crash if tt < 0,
need to find a better solution.
if (tt < 0) {
tt = 0;
}
*/
#ifdef WINDOWS
if (tt < 0) tt = 0;
#endif
if (tt <= 0 && ms < 0) {
tt--;
if (precision == TSDB_TIME_PRECISION_NANO) {
ms += 1000000000;
} else if (precision == TSDB_TIME_PRECISION_MICRO) {
ms += 1000000;
} else {
ms += 1000;
}
}
struct tm* ptm = taosLocalTime(&tt, NULL);
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm);
if (precision == TSDB_TIME_PRECISION_NANO) {
sprintf(buf + pos, ".%09d", ms);
} else if (precision == TSDB_TIME_PRECISION_MICRO) {
sprintf(buf + pos, ".%06d", ms);
} else {
sprintf(buf + pos, ".%03d", ms);
}
return buf;
}
#if 0
int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return 0;
SMqPollRsp* pRsp = &tmq_message->msg;
return pRsp->skipLogNum;
}
void tmqShowMsg(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return;
static bool noPrintSchema;
char pBuf[128];
SMqPollRsp* pRsp = &tmq_message->msg;
int32_t colNum = 2;
if (!noPrintSchema) {
printf("|");
for (int32_t i = 0; i < colNum; i++) {
if (i == 0)
printf(" %25s |", pRsp->schema->pSchema[i].name);
else
printf(" %15s |", pRsp->schema->pSchema[i].name);
}
printf("\n");
printf("===============================================\n");
noPrintSchema = true;
}
int32_t sz = taosArrayGetSize(pRsp->pBlockData);
for (int32_t i = 0; i < sz; i++) {
SSDataBlock* pDataBlock = taosArrayGet(pRsp->pBlockData, i);
int32_t rows = pDataBlock->info.rows;
for (int32_t j = 0; j < rows; j++) {
printf("|");
for (int32_t k = 0; k < colNum; k++) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k);
void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes);
switch (pColInfoData->info.type) {
case TSDB_DATA_TYPE_TIMESTAMP:
formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI);
printf(" %25s |", pBuf);
break;
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_UINT:
printf(" %15u |", *(uint32_t*)var);
break;
}
}
printf("\n");
}
}
}
#endif
int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
@ -1049,7 +956,6 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) {
}
tDeleteSMqCMGetSubEpRsp(&rsp);
} else {
/*SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp));*/
SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper));
if (pWrapper == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -1208,7 +1114,6 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) {
pRspObj->resIter = -1;
memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp));
/*SRetrieveTableRsp* pRetrieve = taosArrayGetP(pWrapper->msg.blockData, 0);*/
pRspObj->resInfo.totalRows = 0;
pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI;
setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols);
@ -1355,31 +1260,6 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) {
}
}
#if 0
tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) {
tmq_message_t* rspMsg = NULL;
int64_t startTime = taosGetTimestampMs();
int64_t status = atomic_load_64(&tmq->status);
tmqAskEp(tmq, status == TMQ_CONSUMER_STATUS__INIT);
while (1) {
rspMsg = tmqSyncPollImpl(tmq, blocking_time);
if (rspMsg && rspMsg->consumeRsp.numOfTopics) {
return rspMsg;
}
if (blocking_time != 0) {
int64_t endTime = taosGetTimestampMs();
if (endTime - startTime > blocking_time) {
return NULL;
}
} else
return NULL;
}
}
#endif
TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
SMqRspObj* rspObj;
int64_t startTime = taosGetTimestampMs();
@ -1417,137 +1297,10 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
}
}
#if 0
if (blocking_time <= 0) blocking_time = 1;
if (blocking_time > 1000) blocking_time = 1000;
/*blocking_time = 1;*/
if (taosArrayGetSize(tmq->clientTopics) == 0) {
tscDebug("consumer:%ld poll but not assigned", tmq->consumerId);
/*printf("over1\n");*/
taosMsleep(blocking_time);
return NULL;
}
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx);
if (taosArrayGetSize(pTopic->vgs) == 0) {
/*printf("over2\n");*/
taosMsleep(blocking_time);
return NULL;
}
tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics);
int32_t beginVgIdx = pTopic->nextVgIdx;
while (1) {
pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs);
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx);
/*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, pTopic, pVg);
if (pReq == NULL) {
ASSERT(false);
taosMsleep(blocking_time);
return NULL;
}
SMqPollCbParam* param = taosMemoryMalloc(sizeof(SMqPollCbParam));
if (param == NULL) {
ASSERT(false);
taosMsleep(blocking_time);
return NULL;
}
param->tmq = tmq;
param->retMsg = &tmq_message;
param->pVg = pVg;
tsem_init(&param->rspSem, 0, 0);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.handle = NULL,
};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = param;
sendInfo->fp = tmqPollCb;
/*printf("req offset: %ld\n", pReq->offset);*/
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
tmq->pollCnt++;
tsem_wait(&param->rspSem);
tsem_destroy(&param->rspSem);
taosMemoryFree(param);
if (tmq_message == NULL) {
if (beginVgIdx == pTopic->nextVgIdx) {
taosMsleep(blocking_time);
} else {
continue;
}
}
return tmq_message;
}
/*tsem_wait(&pRequest->body.rspSem);*/
/*if (body != NULL) {*/
/*destroySendMsgInfo(body);*/
/*}*/
/*if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {*/
/*pRequest->code = terrno;*/
/*}*/
/*return pRequest;*/
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) {
// TODO
return TMQ_RESP_ERR__SUCCESS;
}
#endif
#if 0
tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_vgroup_list, int32_t async) {
if (tmq_topic_vgroup_list != NULL) {
// TODO
}
// TODO: change semaphore to gate
for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
for (int j = 0; j < taosArrayGetSize(pTopic->vgs); j++) {
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, 0, pTopic, pVg);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq)};
SMqCommitCbParam* pParam = taosMemoryMalloc(sizeof(SMqCommitCbParam));
if (pParam == NULL) {
continue;
}
pParam->tmq = tmq;
pParam->pVg = pVg;
pParam->async = async;
if (!async) tsem_init(&pParam->rspSem, 0, 0);
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0;
sendInfo->param = pParam;
sendInfo->fp = tmqCommitCb;
int64_t transporterId = 0;
asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo);
if (!async) tsem_wait(&pParam->rspSem);
}
}
return 0;
}
#endif
tmq_resp_err_t tmq_consumer_close(tmq_t* tmq) { return TMQ_RESP_ERR__SUCCESS; }
const char* tmq_err2str(tmq_resp_err_t err) {
if (err == TMQ_RESP_ERR__SUCCESS) {
@ -1573,10 +1326,3 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
return -1;
}
}
void tmq_message_destroy(TAOS_RES* res) {
if (res == NULL) return;
if (TD_RES_TMQ(res)) {
SMqRspObj* pRspObj = (SMqRspObj*)res;
}
}

4
source/common/CMakeLists.txt
View File

@ -5,8 +5,8 @@ target_include_directories(
PUBLIC "${TD_SOURCE_DIR}/include/common"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
IF(${TD_WINDOWS})
PRIVATE "${TD_SOURCE_DIR}/contrib/pthread-win32"
PRIVATE "${TD_SOURCE_DIR}/contrib/gnuregex"
PRIVATE "${TD_SOURCE_DIR}/contrib/pthread"
PRIVATE "${TD_SOURCE_DIR}/contrib/msvcregex"
ENDIF ()
)
target_link_libraries(

1
source/common/src/tdatablock.c
View File

@ -31,7 +31,6 @@ int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) {
if (pEp->port == 0) {
pEp->port = tsServerPort;
return -1;
}
return 0;

5
source/common/src/tglobal.c
View File

@ -80,6 +80,9 @@ int32_t tsTelemInterval = 86400;
char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.taosdata.com";
uint16_t tsTelemPort = 80;
// query
int32_t tsQueryPolicy = 1;
/*
* denote if the server needs to compress response message at the application layer to client, including query rsp,
* metricmeta rsp, and multi-meter query rsp message body. The client compress the submit message to server.
@ -322,6 +325,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
return -1;
if (cfgAddBool(pCfg, "keepColumnName", tsKeepOriginalColumnName, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "maxBinaryDisplayWidth", tsMaxBinaryDisplayWidth, 1, 65536, 1) != 0) return -1;
if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 3, 1) != 0) return -1;
tsNumOfTaskQueueThreads = tsNumOfCores / 4;
tsNumOfTaskQueueThreads = TRANGE(tsNumOfTaskQueueThreads, 1, 2);
@ -515,6 +519,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsKeepOriginalColumnName = cfgGetItem(pCfg, "keepColumnName")->bval;
tsMaxBinaryDisplayWidth = cfgGetItem(pCfg, "maxBinaryDisplayWidth")->i32;
tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32;
tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32;
return 0;
}

53
source/common/src/tmsg.c
View File

@ -114,6 +114,19 @@ int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) {
return 0;
}
int32_t tEncodeSQueryNodeAddr(SCoder *pEncoder, SQueryNodeAddr *pAddr) {
if (tEncodeI32(pEncoder, pAddr->nodeId) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pAddr->epSet) < 0) return -1;
return 0;
}
int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) {
if (tDecodeI32(pDecoder, &pAddr->nodeId) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pAddr->epSet) < 0) return -1;
return 0;
}
int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pEp->inUse);
@ -844,6 +857,27 @@ void tFreeSMAltertbReq(SMAltertbReq *pReq) {
taosArrayDestroy(pReq->pFields);
pReq->pFields = NULL;
}
int32_t tSerializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
SCoder encoder = {0};
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeSEpSet(&encoder, &pReq->epSet) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tCoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMEpSet(void *buf, int32_t bufLen, SMEpSet *pReq) {
SCoder decoder = {0};
tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeSEpSet(&decoder, &pReq->epSet) < 0) return -1;
tEndDecode(&decoder);
tCoderClear(&decoder);
return 0;
}
int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq) {
SCoder encoder = {0};
@ -2037,11 +2071,11 @@ int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp)
tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER);
if (tStartEncode(&encoder) < 0) return -1;
int32_t num = taosArrayGetSize(pRsp->epSetList);
int32_t num = taosArrayGetSize(pRsp->addrsList);
if (tEncodeI32(&encoder, num) < 0) return -1;
for (int32_t i = 0; i < num; ++i) {
SEpSet *epSet = taosArrayGet(pRsp->epSetList, i);
if (tEncodeSEpSet(&encoder, epSet) < 0) return -1;
SQueryNodeAddr *addr = taosArrayGet(pRsp->addrsList, i);
if (tEncodeSQueryNodeAddr(&encoder, addr) < 0) return -1;
}
tEndEncode(&encoder);
@ -2057,10 +2091,10 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
if (tStartDecode(&decoder) < 0) return -1;
int32_t num = 0;
if (tDecodeI32(&decoder, &num) < 0) return -1;
pRsp->epSetList = taosArrayInit(num, sizeof(SEpSet));
if (NULL == pRsp->epSetList) return -1;
pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
if (NULL == pRsp->addrsList) return -1;
for (int32_t i = 0; i < num; ++i) {
if (tDecodeSEpSet(&decoder, TARRAY_GET_ELEM(pRsp->epSetList, i)) < 0) return -1;
if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1;
}
tEndDecode(&decoder);
@ -2068,7 +2102,7 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
return 0;
}
void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->epSetList); }
void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->addrsList); }
int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) {
SCoder encoder = {0};
@ -2713,11 +2747,11 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withTagSchema) < 0) return -1;
if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
if (tEncodeI32(&encoder, astLen) < 0) return -1;
if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
if (0 == astLen && tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1;
tEndEncode(&encoder);
@ -2739,7 +2773,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withTagSchema) < 0) return -1;
if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
if (tDecodeI32(&decoder, &astLen) < 0) return -1;
@ -2753,6 +2786,8 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
pReq->ast = taosMemoryCalloc(1, astLen + 1);
if (pReq->ast == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
} else {
if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1;
}
tEndDecode(&decoder);

6
source/common/src/ttime.c
View File

@ -412,10 +412,10 @@ int64_t convertTimeFromPrecisionToUnit(int64_t time, int32_t fromPrecision, char
int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec, int64_t *timeVal) {
int32_t charLen = varDataLen(inputData);
char *newColData;
if (type == TSDB_DATA_TYPE_BINARY) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY) {
newColData = taosMemoryCalloc(1, charLen + 1);
memcpy(newColData, varDataVal(inputData), charLen);
bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, 0);
bool ret = taosParseTime(newColData, timeVal, charLen, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData);
return ret;
@ -429,7 +429,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
return TSDB_CODE_FAILED;
}
newColData[len] = 0;
bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, 0);
bool ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
if (ret != TSDB_CODE_SUCCESS) {
taosMemoryFree(newColData);
return ret;

2
source/common/test/commonTests.cpp
View File

@ -167,7 +167,7 @@ TEST(testCase, Datablock_test) {
printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData));
SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo));
SBlockOrderInfo order = {.nullFirst = true, .order = TSDB_ORDER_ASC, .slotId = 0};
SBlockOrderInfo order = { true, TSDB_ORDER_ASC, 0, NULL };
taosArrayPush(pOrderInfo, &order);
blockDataSort(b, pOrderInfo);

4
source/dnode/mgmt/exe/dmMain.c
View File

@ -40,10 +40,12 @@ static void dmSetSignalHandle() {
taosSetSignal(SIGTERM, dmStopDnode);
taosSetSignal(SIGHUP, dmStopDnode);
taosSetSignal(SIGINT, dmStopDnode);
taosSetSignal(SIGTSTP, dmStopDnode);
taosSetSignal(SIGABRT, dmStopDnode);
taosSetSignal(SIGBREAK, dmStopDnode);
#ifndef WINDOWS
taosSetSignal(SIGTSTP, dmStopDnode);
taosSetSignal(SIGQUIT, dmStopDnode);
#endif
if (!tsMultiProcess) {
} else if (global.ntype == DNODE || global.ntype == NODE_END) {

1
source/dnode/mgmt/implement/src/dmExec.c
View File

@ -174,6 +174,7 @@ void dmStopNode(SMgmtWrapper *pWrapper) {
void dmCloseNode(SMgmtWrapper *pWrapper) {
dInfo("node:%s, start to close", pWrapper->name);
pWrapper->deployed = false;
while (pWrapper->refCount > 0) {
taosMsleep(10);

128
source/dnode/mgmt/implement/src/dmHandle.c
View File

@ -216,6 +216,129 @@ static void dmStopMgmt(SMgmtWrapper *pWrapper) {
dmStopStatusThread(pWrapper->pDnode);
}
static int32_t dmSpawnUdfd(SDnode *pDnode);
void dmUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
dInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal);
SDnode *pDnode = process->data;
if (exitStatus == 0 && termSignal == 0 || atomic_load_32(&pDnode->udfdData.stopCalled)) {
dInfo("udfd process exit due to SIGINT or dnode-mgmt called stop");
} else {
dInfo("udfd process restart");
dmSpawnUdfd(pDnode);
}
}
static int32_t dmSpawnUdfd(SDnode *pDnode) {
dInfo("dnode start spawning udfd");
uv_process_options_t options = {0};
char path[PATH_MAX] = {0};
if (tsProcPath == NULL) {
path[0] = '.';
} else {
strncpy(path, tsProcPath, strlen(tsProcPath));
taosDirName(path);
}
strcat(path, "/udfd");
char* argsUdfd[] = {path, "-c", configDir, NULL};
options.args = argsUdfd;
options.file = path;
options.exit_cb = dmUdfdExit;
SUdfdData *pData = &pDnode->udfdData;
uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
uv_stdio_container_t child_stdio[3];
child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
child_stdio[0].data.stream = (uv_stream_t*) &pData->ctrlPipe;
child_stdio[1].flags = UV_IGNORE;
child_stdio[2].flags = UV_INHERIT_FD;
child_stdio[2].data.fd = 2;
options.stdio_count = 3;
options.stdio = child_stdio;
options.flags = UV_PROCESS_DETACHED;
char dnodeIdEnvItem[32] = {0};
char thrdPoolSizeEnvItem[32] = {0};
snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pDnode->data.dnodeId);
float numCpuCores = 4;
taosGetCpuCores(&numCpuCores);
snprintf(thrdPoolSizeEnvItem,32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores*2);
char* envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL};
options.env = envUdfd;
int err = uv_spawn(&pData->loop, &pData->process, &options);
pData->process.data = (void*)pDnode;
if (err != 0) {
dError("can not spawn udfd. path: %s, error: %s", path, uv_strerror(err));
}
return err;
}
static void dmUdfdCloseWalkCb(uv_handle_t* handle, void* arg) {
if (!uv_is_closing(handle)) {
uv_close(handle, NULL);
}
}
static void dmUdfdStopAsyncCb(uv_async_t *async) {
SDnode *pDnode = async->data;
SUdfdData *pData = &pDnode->udfdData;
uv_stop(&pData->loop);
}
static void dmWatchUdfd(void *args) {
SDnode *pDnode = args;
SUdfdData *pData = &pDnode->udfdData;
uv_loop_init(&pData->loop);
uv_async_init(&pData->loop, &pData->stopAsync, dmUdfdStopAsyncCb);
pData->stopAsync.data = pDnode;
int32_t err = dmSpawnUdfd(pDnode);
atomic_store_32(&pData->spawnErr, err);
uv_barrier_wait(&pData->barrier);
uv_run(&pData->loop, UV_RUN_DEFAULT);
uv_loop_close(&pData->loop);
uv_walk(&pData->loop, dmUdfdCloseWalkCb, NULL);
uv_run(&pData->loop, UV_RUN_DEFAULT);
uv_loop_close(&pData->loop);
return;
}
static int32_t dmStartUdfd(SDnode *pDnode) {
SUdfdData *pData = &pDnode->udfdData;
if (pData->startCalled) {
dInfo("dnode-mgmt start udfd already called");
return 0;
}
pData->startCalled = true;
uv_barrier_init(&pData->barrier, 2);
uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
uv_barrier_wait(&pData->barrier);
pData->needCleanUp = true;
return pData->spawnErr;
}
static int32_t dmStopUdfd(SDnode *pDnode) {
dInfo("dnode-mgmt to stop udfd. need cleanup: %d, spawn err: %d",
pDnode->udfdData.needCleanUp, pDnode->udfdData.spawnErr);
SUdfdData *pData = &pDnode->udfdData;
if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) {
return 0;
}
atomic_store_32(&pData->stopCalled, 1);
pData->needCleanUp = false;
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
return 0;
}
static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
dInfo("dnode-mgmt start to init");
SDnode *pDnode = pWrapper->pDnode;
@ -247,6 +370,10 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "dnode-transport", "initialized");
// if (dmStartUdfd(pDnode) != 0) {
// dError("failed to start udfd");
// }
dInfo("dnode-mgmt is initialized");
return 0;
}
@ -254,6 +381,7 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
static void dmCleanupMgmt(SMgmtWrapper *pWrapper) {
dInfo("dnode-mgmt start to clean up");
SDnode *pDnode = pWrapper->pDnode;
dmStopUdfd(pDnode);
dmStopWorker(pDnode);
taosWLockLatch(&pDnode->data.latch);

15
source/dnode/mgmt/interface/inc/dmDef.h
View File

@ -16,6 +16,7 @@
#ifndef _TD_DM_DEF_H_
#define _TD_DM_DEF_H_
#include "uv.h"
#include "dmLog.h"
#include "cJSON.h"
@ -142,6 +143,19 @@ typedef struct {
char desc[TSDB_STEP_DESC_LEN];
} SStartupInfo;
typedef struct SUdfdData {
bool startCalled;
bool needCleanUp;
uv_loop_t loop;
uv_thread_t thread;
uv_barrier_t barrier;
uv_process_t process;
int spawnErr;
uv_pipe_t ctrlPipe;
uv_async_t stopAsync;
int32_t stopCalled;
} SUdfdData;
typedef struct SDnode {
EDndProcType ptype;
EDndNodeType ntype;
@ -150,6 +164,7 @@ typedef struct SDnode {
SStartupInfo startup;
SDnodeTrans trans;
SDnodeData data;
SUdfdData udfdData;
TdThreadMutex mutex;
SMgmtWrapper wrappers[NODE_END];
} SDnode;

1
source/dnode/mgmt/interface/src/dmEnv.c
View File

@ -53,6 +53,7 @@ void dmCleanup() {
}
monCleanup();
syncCleanUp();
walCleanUp();
taosStopCacheRefreshWorker();
dInfo("dnode env is cleaned up");

3
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -218,9 +218,6 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_MND_GET_INDEX, mmProcessReadMsg, DEFAULT_HANDLE);
// Requests handled by VNODE
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, mmProcessWriteMsg, DEFAULT_HANDLE);

1
source/dnode/mgmt/mgmt_vnode/inc/vmInt.h
View File

@ -105,6 +105,7 @@ void vmStopWorker(SVnodesMgmt *pMgmt);
int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc); // sync integration
int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

32
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -121,6 +121,16 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->hashBegin = pCreate->hashBegin;
pCfg->hashEnd = pCreate->hashEnd;
pCfg->hashMethod = pCreate->hashMethod;
// sync integration
pCfg->syncCfg.myIndex = pCreate->selfIndex;
pCfg->syncCfg.replicaNum = pCreate->replica;
memset(&(pCfg->syncCfg.nodeInfo), 0, sizeof(pCfg->syncCfg.nodeInfo));
for (int i = 0; i < pCreate->replica; ++i) {
(pCfg->syncCfg.nodeInfo)[i].nodePort = (pCreate->replicas)[i].port;
snprintf((pCfg->syncCfg.nodeInfo)[i].nodeFqdn, sizeof((pCfg->syncCfg.nodeInfo)[i].nodeFqdn), "%s",
(pCreate->replicas)[i].fqdn);
}
}
static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) {
@ -172,6 +182,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; // sync integration
msgCb.qsizeFp = vmGetQueueSize;
SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@ -252,7 +263,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
//dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE);
@ -265,11 +276,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
//dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
//dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
//dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
//dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, DEFAULT_HANDLE);
@ -283,4 +290,17 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
// dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
// sync integration
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
}

16
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -73,6 +73,12 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
return -1;
}
// sync integration
vnodeSyncSetQ(pImpl, NULL);
vnodeSyncSetRpc(pImpl, NULL);
int32_t ret = vnodeSyncStart(pImpl);
assert(ret == 0);
taosWLockLatch(&pMgmt->latch);
int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
taosWUnLockLatch(&pMgmt->latch);
@ -137,6 +143,7 @@ static void *vmOpenVnodeFunc(void *param) {
msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue; // sync integration
msgCb.qsizeFp = vmGetQueueSize;
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@ -266,6 +273,9 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
// walCleanUp();
taosMemoryFree(pMgmt);
pWrapper->pMgmt = NULL;
// syncCleanUp();
dInfo("vnode-mgmt is cleaned up");
}
@ -306,6 +316,12 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "vnode-wal", "initialized");
// sync integration
if (syncInit() != 0) {
dError("failed to open sync since %s", terrstr());
return -1;
}
if (vnodeInit(tsNumOfCommitThreads) != 0) {
dError("failed to init vnode since %s", terrstr());
goto _OVER;

95
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -14,8 +14,12 @@
*/
#define _DEFAULT_SOURCE
#include "vmInt.h"
#include "sync.h"
#include "syncTools.h"
static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
.ahandle = pMsg->rpcMsg.ahandle,
@ -88,7 +92,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle;
int64_t version;
SRpcMsg rsp;
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
if (pArray == NULL) {
@ -107,13 +111,15 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
}
}
#if 0
int64_t version;
vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version);
numOfMsgs = taosArrayGetSize(pArray);
for (int32_t i = 0; i < numOfMsgs; i++) {
SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
SRpcMsg *pRpc = &pMsg->rpcMsg;
SRpcMsg rsp;
rsp.pCont = NULL;
rsp.contLen = 0;
@ -123,17 +129,36 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp);
tmsgSendRsp(&rsp);
#if 0
if (pRsp != NULL) {
pRsp->ahandle = pRpc->ahandle;
taosMemoryFree(pRsp);
} else {
if (code != 0 && terrno != 0) code = terrno;
vmSendRsp(pVnode->pWrapper, pMsg, code);
}
#endif
}
#else
// sync integration response
for (int i = 0; i < taosArrayGetSize(pArray); i++) {
SNodeMsg *pMsg;
SRpcMsg *pRpc;
pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
pRpc = &pMsg->rpcMsg;
rsp.ahandle = pRpc->ahandle;
rsp.handle = pRpc->handle;
rsp.pCont = NULL;
rsp.contLen = 0;
int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false);
if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
rsp.code = TSDB_CODE_SYN_NOT_LEADER;
tmsgSendRsp(&rsp);
} else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) {
rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR;
tmsgSendRsp(&rsp);
} else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) {
// ok
// send response in applyQ
} else {
assert(0);
}
}
#endif
for (int32_t i = 0; i < numOfMsgs; i++) {
SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
@ -148,13 +173,44 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
SVnodeObj *pVnode = pInfo->ahandle;
SNodeMsg *pMsg = NULL;
SRpcMsg rsp;
// static int64_t version = 0;
for (int32_t i = 0; i < numOfMsgs; ++i) {
#if 1
// sync integration
taosGetQitem(qall, (void **)&pMsg);
// todo
SRpcMsg *pRsp = NULL;
// (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
// init response rpc msg
rsp.code = 0;
rsp.pCont = NULL;
rsp.contLen = 0;
// get original rpc msg
assert(pMsg->rpcMsg.msgType == TDMT_VND_SYNC_APPLY_MSG);
SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(&pMsg->rpcMsg);
syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
SRpcMsg originalRpcMsg;
syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
// apply data into tsdb
if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
rsp.code = terrno;
dTrace("vnodeProcessWriteReq error, code:%d", terrno);
}
syncApplyMsgDestroy(pSyncApplyMsg);
rpcFreeCont(originalRpcMsg.pCont);
// if leader, send response
if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
rsp.ahandle = pMsg->rpcMsg.ahandle;
rsp.handle = pMsg->rpcMsg.handle;
tmsgSendRsp(&rsp);
}
#endif
}
}
@ -307,6 +363,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT
dTrace("msg:%p, will be put into vnode-merge queue", pMsg);
taosWriteQitem(pVnode->pMergeQ, pMsg);
break;
case SYNC_QUEUE: // sync integration
dTrace("msg:%p, will be put into vnode-sync queue", pMsg);
taosWriteQitem(pVnode->pSyncQ, pMsg);
break;
default:
code = -1;
terrno = TSDB_CODE_INVALID_PARA;
@ -333,6 +393,11 @@ int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE);
}
// sync integration
int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
return vmPutRpcMsgToQueue(pWrapper, pRpc, SYNC_QUEUE);
}
int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
int32_t size = -1;
SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId);

1
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -30,6 +30,7 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SEpSet mndGetVgroupEpset(SMnode *pMnode, const SVgObj *pVgroup);
int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId);
int32_t mndGetGlobalVgroupVersion(int32_t *vgId);
void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);

8
source/dnode/mnode/impl/src/mndDb.c
View File

@ -1191,6 +1191,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
char *p = strchr(usedbReq.db, '.');
if (p && 0 == strcmp(p + 1, TSDB_INFORMATION_SCHEMA_DB)) {
memcpy(usedbRsp.db, usedbReq.db, TSDB_DB_FNAME_LEN);
//mndGetGlobalVgroupVersion(); TODO
static int32_t vgVersion = 1;
if (usedbReq.vgVersion < vgVersion) {
usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo));
@ -1202,16 +1203,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) {
mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos);
usedbRsp.vgVersion = vgVersion++;
if (taosArrayGetSize(usedbRsp.pVgroupInfos) <= 0) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
} else {
code = 0;
}
} else {
usedbRsp.vgVersion = usedbReq.vgVersion;
code = 0;
}
usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos);
code = 0;
// no jump, need to construct rsp
} else {

33
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -52,9 +52,9 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq);
static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp);
static int32_t mndProcessStatusReq(SNodeMsg *pReq);
static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter);
int32_t mndInitDnode(SMnode *pMnode) {
@ -335,10 +335,13 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) {
}
bool roleChanged = false;
for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
roleChanged = true;
// sync integration
if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
roleChanged = true;
}
pVgroup->vnodeGid[vg].role = pVload->syncState;
}
pVgroup->vnodeGid[vg].role = pVload->syncState;
}
if (roleChanged) {
// notify scheduler role has changed
@ -634,7 +637,7 @@ static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) {
return TSDB_CODE_SUCCESS;
}
static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode;
int32_t totalRows = 0;
int32_t numOfRows = 0;
@ -667,12 +670,12 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
STR_WITH_MAXSIZE_TO_VARSTR(buf, cfgOpts[i], TSDB_CONFIG_OPTION_LEN);
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char*) buf, false);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)buf, false);
STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONIIG_VALUE_LEN);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char*) bufVal, false);
colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false);
numOfRows++;
}
@ -683,7 +686,7 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {}
static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
@ -698,8 +701,8 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
cols = 0;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->id, false);
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false);
char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->bytes[cols]);
@ -709,19 +712,19 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
int16_t id = mndGetVnodesNum(pMnode, pDnode->id);
colDataAppend(pColInfo, numOfRows, (const char*) &id, false);
colDataAppend(pColInfo, numOfRows, (const char *)&id, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false);
char b1[9] = {0};
STR_TO_VARSTR(b1, online? "ready":"offline");
STR_TO_VARSTR(b1, online ? "ready" : "offline");
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, b1, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->createdTime, false);
colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false);
char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]);

10
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -199,15 +199,6 @@ static const SInfosTableSchema vgroupsSchema[] = {
{.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
// TODO put into perf schema
static const SInfosTableSchema topicSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = 1024 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
};
static const SInfosTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
@ -291,7 +282,6 @@ static const SInfosTableMeta infosMeta[] = {
{TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)},
{TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)},
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)},
{TSDB_INS_TABLE_TOPICS, topicSchema, tListLen(topicSchema)},
{TSDB_INS_TABLE_CONSUMERS, consumerSchema, tListLen(consumerSchema)},
{TSDB_INS_TABLE_SUBSCRIBES, subscribeSchema, tListLen(subscribeSchema)},
{TSDB_INS_TABLE_TRANS, transSchema, tListLen(transSchema)},

6
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -41,14 +41,15 @@ static const SPerfsTableSchema queriesSchema[] = {
static const SPerfsTableSchema topicSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
/*{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},*/
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
{.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY},
{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
/*{.name = "row_len", .bytes = 4, .type = TSDB_DATA_TYPE_INT},*/
};
static const SPerfsTableSchema consumerSchema[] = {
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "app_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "status", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
@ -61,6 +62,7 @@ static const SPerfsTableSchema subscribeSchema[] = {
{.name = "topic_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "group_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
{.name = "offset", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT},
{.name = "client_id", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_BINARY},
};

25
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -58,7 +58,8 @@ static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter);
int32_t mndInitProfile(SMnode *pMnode) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
int32_t connCheckTime = tsShellActivityTimer * 2;
// in ms
int32_t connCheckTime = tsShellActivityTimer * 2 * 1000;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn");
if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -174,10 +175,10 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
}
static int32_t mndProcessConnectReq(SNodeMsg *pReq) {
SMnode * pMnode = pReq->pNode;
SUserObj * pUser = NULL;
SDbObj * pDb = NULL;
SConnObj * pConn = NULL;
SMnode *pMnode = pReq->pNode;
SUserObj *pUser = NULL;
SDbObj *pDb = NULL;
SConnObj *pConn = NULL;
int32_t code = -1;
SConnectReq connReq = {0};
char ip[30] = {0};
@ -464,7 +465,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq);
int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp);
void * buf = rpcMallocCont(tlen);
void *buf = rpcMallocCont(tlen);
tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp);
int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps);
@ -486,7 +487,7 @@ static int32_t mndProcessHeartBeatReq(SNodeMsg *pReq) {
}
static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
SMnode * pMnode = pReq->pNode;
SMnode *pMnode = pReq->pNode;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@ -520,7 +521,7 @@ static int32_t mndProcessKillQueryReq(SNodeMsg *pReq) {
}
static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
SMnode * pMnode = pReq->pNode;
SMnode *pMnode = pReq->pNode;
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
SUserObj *pUser = mndAcquireUser(pMnode, pReq->user);
@ -552,11 +553,11 @@ static int32_t mndProcessKillConnReq(SNodeMsg *pReq) {
}
static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
SMnode * pMnode = pReq->pNode;
SMnode *pMnode = pReq->pNode;
int32_t numOfRows = 0;
SConnObj *pConn = NULL;
int32_t cols = 0;
char * pWrite;
char *pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 6];
if (pShow->pIter == NULL) {
@ -611,8 +612,8 @@ static int32_t mndRetrieveConns(SNodeMsg *pReq, SShowObj *pShow, char *data, int
}
static int32_t mndRetrieveQueries(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
SMnode *pMnode = pReq->pNode;
int32_t numOfRows = 0;
SMnode *pMnode = pReq->pNode;
int32_t numOfRows = 0;
#if 0
SConnObj *pConn = NULL;
int32_t cols = 0;

14
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -444,8 +444,8 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
goto _OVER;
}
qlistRsp.epSetList = taosArrayInit(5, sizeof(SEpSet));
if (NULL == qlistRsp.epSetList) {
qlistRsp.addrsList = taosArrayInit(5, sizeof(SQueryNodeAddr));
if (NULL == qlistRsp.addrsList) {
mError("failed to alloc epSet while process qnode list req");
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
@ -455,11 +455,13 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj);
if (pIter == NULL) break;
SEpSet epSet = {.numOfEps = 1};
tstrncpy(epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
epSet.eps[0].port = pObj->pDnode->port;
SQueryNodeAddr nodeAddr = {0};
nodeAddr.nodeId = QNODE_HANDLE;
nodeAddr.epSet.numOfEps = 1;
tstrncpy(nodeAddr.epSet.eps[0].fqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
nodeAddr.epSet.eps[0].port = pObj->pDnode->port;
(void)taosArrayPush(qlistRsp.epSetList, &epSet);
(void)taosArrayPush(qlistRsp.addrsList, &nodeAddr);
numOfRows++;
sdbRelease(pSdb, pObj);

4
source/dnode/mnode/impl/src/mndShow.c
View File

@ -28,7 +28,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq);
int32_t mndInitShow(SMnode *pMnode) {
SShowMgmt *pMgmt = &pMnode->showMgmt;
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5, true, (__cache_free_fn_t)mndFreeShowObj, "show");
pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, 5000, true, (__cache_free_fn_t)mndFreeShowObj, "show");
if (pMgmt->cache == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to alloc show cache since %s", terrstr());
@ -84,8 +84,6 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
type = TSDB_MGMT_TABLE_GRANTS;
} else if (strncasecmp(name, TSDB_INS_TABLE_VGROUPS, len) == 0) {
type = TSDB_MGMT_TABLE_VGROUP;
} else if (strncasecmp(name, TSDB_INS_TABLE_TOPICS, len) == 0) {
type = TSDB_MGMT_TABLE_TOPICS;
} else if (strncasecmp(name, TSDB_INS_TABLE_CONSUMERS, len) == 0) {
type = TSDB_MGMT_TABLE_CONSUMERS;
} else if (strncasecmp(name, TSDB_INS_TABLE_SUBSCRIBES, len) == 0) {

5
source/dnode/mnode/impl/src/mndStream.c
View File

@ -59,7 +59,7 @@ int32_t mndInitStream(SMnode *pMnode) {
/*mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndProcessDropStreamInRsp);*/
// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveStream);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);
/*mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextStream);*/
return sdbSetTable(pMnode->pSdb, table);
}
@ -247,7 +247,8 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64
return code;
}
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark, STrans *pTrans) {
int32_t mndAddStreamToTrans(SMnode *pMnode, SStreamObj *pStream, const char *ast, int8_t triggerType, int64_t watermark,
STrans *pTrans) {
SNode *pAst = NULL;
if (nodesStringToNode(ast, &pAst) < 0) {

52
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -35,7 +35,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj
static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq);
static int32_t mndProcessDropTopicReq(SNodeMsg *pReq);
static int32_t mndProcessDropTopicInRsp(SNodeMsg *pRsp);
static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows);
static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
int32_t mndInitTopic(SMnode *pMnode) {
@ -51,7 +51,7 @@ int32_t mndInitTopic(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq);
mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp);
// mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic);
return sdbSetTable(pMnode->pSdb, table);
@ -511,56 +511,40 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo
return 0;
}
static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) {
static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) {
SMnode *pMnode = pReq->pNode;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SMqTopicObj *pTopic = NULL;
int32_t cols = 0;
char *pWrite;
char prefix[TSDB_DB_FNAME_LEN] = {0};
SDbObj *pDb = mndAcquireDb(pMnode, pShow->db);
if (pDb == NULL) return 0;
tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = (int32_t)strlen(prefix);
while (numOfRows < rows) {
while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic);
if (pShow->pIter == NULL) break;
if (pTopic->dbUid != pDb->uid) {
if (strncmp(pTopic->name, prefix, prefixLen) != 0) {
mError("Inconsistent topic data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid);
}
int32_t cols = 0;
sdbRelease(pSdb, pTopic);
continue;
}
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN);
varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE]));
cols = 0;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
char topicName[TSDB_TOPIC_NAME_LEN] = {0};
tstrncpy(topicName, pTopic->name + prefixLen, TSDB_TOPIC_NAME_LEN);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, topicName);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)&pTopic->createTime, false);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pTopic->createTime;
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(&sql[VARSTR_HEADER_SIZE], pTopic->sql, TSDB_SHOW_SQL_LEN);
varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pTopic->sql, pShow->bytes[cols]);
cols++;
taosMemoryFree(sql);
numOfRows++;
sdbRelease(pSdb, pTopic);
}
mndReleaseDb(pMnode, pDb);
pShow->numOfRows += numOfRows;
return numOfRows;
}

1
source/dnode/mnode/impl/test/db/db.cpp
View File

@ -89,6 +89,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
void* pReq = rpcMallocCont(contLen);
tSerializeSAlterDbReq(pReq, contLen, &alterdbReq);
taosMsleep(1000); // Wait for the vnode to become the leader
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);

1
source/dnode/mnode/impl/test/sma/sma.cpp
View File

@ -258,6 +258,7 @@ TEST_F(MndTestSma, 02_Create_Show_Meta_Drop_Restart_BSma) {
pReq = BuildCreateDbReq(dbname, &contLen);
pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(1000); // Wait for the vnode to become the leader
}
{

8
source/dnode/mnode/impl/test/stb/stb.cpp
View File

@ -304,6 +304,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{
@ -438,6 +439,7 @@ TEST_F(MndTestStb, 02_Alter_Stb_AddTag) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{
@ -497,6 +499,7 @@ TEST_F(MndTestStb, 03_Alter_Stb_DropTag) {
void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{
@ -538,6 +541,7 @@ TEST_F(MndTestStb, 04_Alter_Stb_AlterTagName) {
void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{
@ -602,6 +606,7 @@ TEST_F(MndTestStb, 05_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{
@ -655,6 +660,7 @@ TEST_F(MndTestStb, 06_Alter_Stb_AddColumn) {
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{
@ -715,6 +721,7 @@ TEST_F(MndTestStb, 07_Alter_Stb_DropColumn) {
void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{
@ -775,6 +782,7 @@ TEST_F(MndTestStb, 08_Alter_Stb_AlterTagBytes) {
void* pReq = BuildCreateDbReq(dbname, &contLen);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen);
ASSERT_EQ(pRsp->code, 0);
taosMsleep(2000); // Wait for the vnode to become the leader
}
{

1
source/dnode/vnode/CMakeLists.txt
View File

@ -15,6 +15,7 @@ target_sources(
"src/vnd/vnodeStateMgr.c"
"src/vnd/vnodeModule.c"
"src/vnd/vnodeSvr.c"
"src/vnd/vnodeSync.c"
# meta
"src/meta/metaOpen.c"

13
source/dnode/vnode/inc/vnode.h
View File

@ -21,6 +21,7 @@
#include "tqueue.h"
#include "trpc.h"
#include "sync.h"
#include "tarray.h"
#include "tfs.h"
#include "wal.h"
@ -60,6 +61,9 @@ int32_t vnodeSync(SVnode *pVnode);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName);
int64_t vnodeGetSyncHandle(SVnode *pVnode);
void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
// meta
typedef struct SMeta SMeta; // todo: remove
typedef struct SMTbCursor SMTbCursor;
@ -110,7 +114,8 @@ int tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList
int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
int32_t tqReadHandleSetMsgEx(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle);
int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows);
int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, int32_t *pNumOfRows,
int16_t *pNumOfCols);
// need to reposition
@ -147,6 +152,7 @@ struct SVnodeCfg {
bool isWeak;
STsdbCfg tsdbCfg;
SWalCfg walCfg;
SSyncCfg syncCfg; // sync integration
uint32_t hashBegin;
uint32_t hashEnd;
int8_t hashMethod;
@ -166,6 +172,11 @@ typedef struct {
uint64_t uid;
} STableKeyInfo;
// sync integration
void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
int32_t vnodeSyncStart(SVnode *pVnode);
#ifdef __cplusplus
}
#endif

4
source/dnode/vnode/src/inc/meta.h
View File

@ -51,7 +51,7 @@ static FORCE_INLINE tb_uid_t metaGenerateUid(SMeta* pMeta) { return tGenIdPI64()
#define META_CHILD_TABLE TD_CHILD_TABLE
#define META_NORMAL_TABLE TD_NORMAL_TABLE
int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlHandle *pHandle);
int metaCreateTable(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaDropTable(SMeta* pMeta, tb_uid_t uid);
int metaCommit(SMeta* pMeta);
int32_t metaCreateTSma(SMeta* pMeta, SSmaCfg* pCfg);
@ -74,7 +74,7 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur);
// SMetaDB
int metaOpenDB(SMeta* pMeta);
void metaCloseDB(SMeta* pMeta);
int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlHandle *pHandle);
int metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);

97
source/dnode/vnode/src/inc/tq.h
View File

@ -18,8 +18,10 @@
#include "executor.h"
#include "os.h"
#include "tcache.h"
#include "thash.h"
#include "tmsg.h"
#include "trpc.h"
#include "ttimer.h"
#include "wal.h"
@ -81,29 +83,18 @@ typedef struct STqOffsetStore STqOffsetStore;
struct STqReadHandle {
int64_t ver;
int64_t tbUid;
SHashObj* tbIdHash;
const SSubmitReq* pMsg;
SSubmitBlk* pBlock;
SSubmitMsgIter msgIter;
SSubmitBlkIter blkIter;
SMeta* pVnodeMeta;
SArray* pColIdList; // SArray<int32_t>
SArray* pColIdList; // SArray<int16_t>
int32_t sver;
SSchemaWrapper* pSchemaWrapper;
STSchema* pSchema;
};
typedef struct {
int8_t type;
int8_t reserved[7];
union {
void* data;
int64_t wmTs;
int64_t checkpointId;
};
} STqStreamToken;
typedef struct {
int16_t ver;
int16_t action;
@ -152,27 +143,40 @@ typedef struct {
} STqMetaStore;
typedef struct {
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId;
int32_t epoch;
char* qmsg;
int64_t consumerId;
int32_t epoch;
int32_t skipLogNum;
int64_t reqOffset;
SRWLatch lock;
SRpcMsg* handle;
} STqPushHandle;
typedef struct {
char subKey[TSDB_SUBSCRIBE_KEY_LEN];
int64_t consumerId;
int32_t epoch;
int8_t subType;
int8_t withTbName;
int8_t withSchema;
int8_t withTag;
int8_t withTagSchema;
char* qmsg;
STqPushHandle pushHandle;
// SRWLatch lock;
SWalReadHandle* pReadHandle;
// number should be identical to fetch thread num
qTaskInfo_t task[4];
SWalReadHandle* pWalReader;
// task number should be the same with fetch thread
STqReadHandle* pExecReader[5];
qTaskInfo_t task[5];
} STqExec;
struct STQ {
// the collection of groups
// the handle of meta kvstore
bool writeTrigger;
char* path;
STqMetaStore* tqMeta;
SHashObj* tqMetaNew; // subKey -> tqExec
SHashObj* pStreamTasks;
SVnode* pVnode;
SWal* pWal;
SMeta* pVnodeMeta;
char* path;
// STqMetaStore* tqMeta;
SHashObj* pushMgr; // consumerId -> STqExec*
SHashObj* execs; // subKey -> STqExec
SHashObj* pStreamTasks;
SVnode* pVnode;
SWal* pWal;
};
typedef struct {
@ -213,20 +217,6 @@ typedef struct {
SArray* topics; // SArray<STqTopic>
} STqConsumer;
enum {
TQ_PUSHER_TYPE__CLIENT = 1,
TQ_PUSHER_TYPE__STREAM,
};
typedef struct {
int8_t type;
int8_t reserved[3];
int32_t ttl;
int64_t consumerId;
SRpcMsg* pMsg;
// SMqPollRsp* rsp;
} STqClientPusher;
typedef struct {
int8_t type;
int8_t nodeType;
@ -236,10 +226,6 @@ typedef struct {
// TODO sync function
} STqStreamPusher;
typedef struct {
SHashObj* pHash; // <id, STqPush*>
} STqPushMgr;
typedef struct {
int8_t inited;
tmr_h timer;
@ -252,17 +238,14 @@ int tqInit();
void tqCleanUp();
// open in each vnode
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pMeta, SMemAllocatorFactory* allocFac);
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal);
void tqClose(STQ*);
// required by vnode
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version);
int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver);
int tqCommit(STQ*);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId);
int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen);
// int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
// int32_t tqProcessRebReq(STQ* pTq, char* msg);
// int32_t tqProcessCancelConnReq(STQ* pTq, char* msg);
int32_t tqProcessTaskExec(STQ* pTq, char* msg, int32_t msgLen, int32_t workerId);
int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen);
int32_t tqProcessStreamTrigger(STQ* pTq, void* data, int32_t dataLen, int32_t workerId);
@ -303,16 +286,6 @@ int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t
int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey);
int32_t tqOffsetPersistAll(STqOffsetStore* pStore);
// tqPush
int32_t tqPushMgrInit();
void tqPushMgrCleanUp();
STqPushMgr* tqPushMgrOpen();
void tqPushMgrClose(STqPushMgr* pushMgr);
STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
#ifdef __cplusplus
}
#endif

2
source/dnode/vnode/src/inc/tsdbSma.h
View File

@ -26,7 +26,7 @@ extern "C" {
typedef int32_t (*__tb_ddl_fn_t)(void *ahandle, void **result, void *p1, void *p2);
struct STbDdlHandle {
struct STbDdlH {
void *ahandle;
void *result;
__tb_ddl_fn_t fp;

31
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -68,13 +68,13 @@ typedef struct {
} SStreamSinkInfo;
typedef struct {
SVnode* pVnode;
SHashObj* pHash; // streamId -> SStreamSinkInfo
SVnode *pVnode;
SHashObj *pHash; // streamId -> SStreamSinkInfo
} SSink;
// SVState
struct SVState {
int64_t processed;
// int64_t processed;
int64_t committed;
int64_t applied;
};
@ -85,27 +85,28 @@ struct SVnodeInfo {
};
struct SVnode {
char* path;
char *path;
SVnodeCfg config;
SVState state;
STfs* pTfs;
STfs *pTfs;
SMsgCb msgCb;
SVBufPool* pBufPool;
SMeta* pMeta;
STsdb* pTsdb;
SWal* pWal;
STQ* pTq;
SSink* pSink;
SVBufPool *pBufPool;
SMeta *pMeta;
STsdb *pTsdb;
SWal *pWal;
STQ *pTq;
SSink *pSink;
int64_t sync; // sync integration
tsem_t canCommit;
SQHandle* pQuery;
SQHandle *pQuery;
};
#define TD_VID(PVNODE) (PVNODE)->config.vgId
typedef struct STbDdlHandle STbDdlHandle;
typedef struct STbDdlH STbDdlH;
// sma
void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
#include "vnd.h"
@ -115,6 +116,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
#include "tq.h"
#include "vnodeSync.h"
#include "tsdbSma.h"
#ifdef __cplusplus

44
source/dnode/vnode/src/inc/vnodeSync.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_VNODE_SYNC_H_
#define _TD_VNODE_SYNC_H_
#ifdef __cplusplus
extern "C" {
#endif
int32_t vnodeSyncOpen(SVnode *pVnode, char *path);
int32_t vnodeSyncStart(SVnode *pVnode);
void vnodeSyncClose(SVnode *pVnode);
void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg);
int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg);
void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot);
SSyncFSM *syncVnodeMakeFsm();
#ifdef __cplusplus
}
#endif
#endif /*_TD_VNODE_SYNC_H_*/

3
source/dnode/vnode/src/meta/metaIdx.c
View File

@ -23,6 +23,9 @@ struct SMetaIdx {
SIndex *pIdx;
#endif
/* data */
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
};
int metaOpenIdx(SMeta *pMeta) {

6
source/dnode/vnode/src/meta/metaTDBImpl.c
View File

@ -45,10 +45,12 @@ struct SMetaDB {
#endif
};
typedef struct __attribute__((__packed__)) {
#pragma pack(push,1)
typedef struct {
tb_uid_t uid;
int32_t sver;
} SSchemaDbKey;
#pragma pack(pop)
typedef struct {
char *name;
@ -248,7 +250,7 @@ void metaCloseDB(SMeta *pMeta) {
}
}
int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlHandle *pHandle) {
int metaSaveTableToDB(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
tb_uid_t uid;
SMetaDB *pMetaDb;
void *pKey;

2
source/dnode/vnode/src/meta/metaTable.c
View File

@ -15,7 +15,7 @@
#include "vnodeInt.h"
int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg, STbDdlHandle *pHandle) {
int metaCreateTable(SMeta *pMeta, STbCfg *pTbCfg, STbDdlH *pHandle) {
// Validate the tbOptions
// if (metaValidateTbCfg(pMeta, pTbCfg) < 0) {
// // TODO: handle error

318
source/dnode/vnode/src/tq/tq.c
View File

@ -15,11 +15,14 @@
#include "vnodeInt.h"
int32_t tqInit() { return tqPushMgrInit(); }
int32_t tqInit() {
//
return 0;
}
void tqCleanUp() { tqPushMgrCleanUp(); }
void tqCleanUp() {}
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMemAllocatorFactory* allocFac) {
STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
STQ* pTq = taosMemoryMalloc(sizeof(STQ));
if (pTq == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@ -28,18 +31,21 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal, SMeta* pVnodeMeta, SMe
pTq->path = strdup(path);
pTq->pVnode = pVnode;
pTq->pWal = pWal;
pTq->pVnodeMeta = pVnodeMeta;
#if 0
pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer,
(FTqDelete)taosMemoryFree, 0);
if (pTq->tqMeta == NULL) {
taosMemoryFree(pTq);
return NULL;
}
#endif
pTq->tqMetaNew = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
pTq->execs = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
pTq->pStreamTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
pTq->pushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK);
return pTq;
}
@ -51,8 +57,139 @@ void tqClose(STQ* pTq) {
// TODO
}
int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t version) {
int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
if (msgType != TDMT_VND_SUBMIT) return 0;
void* pIter = NULL;
STqExec* pExec = NULL;
SSubmitReq* pReq = (SSubmitReq*)msg;
int32_t workerId = 4;
int64_t fetchOffset = ver;
while (1) {
pIter = taosHashIterate(pTq->pushMgr, pIter);
if (pIter == NULL) break;
pExec = *(STqExec**)pIter;
taosWLockLatch(&pExec->pushHandle.lock);
SRpcMsg* pMsg = atomic_load_ptr(&pExec->pushHandle.handle);
ASSERT(pMsg);
SMqDataBlkRsp rsp = {0};
rsp.reqOffset = pExec->pushHandle.reqOffset;
rsp.blockData = taosArrayInit(0, sizeof(int32_t));
rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
qTaskInfo_t task = pExec->task[workerId];
ASSERT(task);
qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
while (1) {
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(0);
}
if (pDataBlock == NULL) break;
ASSERT(pDataBlock->info.rows != 0);
ASSERT(pDataBlock->info.numOfCols != 0);
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
void* buf = taosMemoryCalloc(1, dataStrLen);
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
pRetrieve->useconds = ts;
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
// TODO enable compress
int32_t actualLen = 0;
blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(rsp.blockDataLen, &actualLen);
taosArrayPush(rsp.blockData, &buf);
rsp.blockNum++;
}
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
STqReadHandle* pReader = pExec->pExecReader[workerId];
tqReadHandleSetMsg(pReader, pReq, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
&block.info.numOfCols) < 0) {
ASSERT(0);
}
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block);
void* buf = taosMemoryCalloc(1, dataStrLen);
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
/*pRetrieve->useconds = 0;*/
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->numOfRows = htonl(block.info.rows);
// TODO enable compress
int32_t actualLen = 0;
blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false);
actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(rsp.blockDataLen, &actualLen);
taosArrayPush(rsp.blockData, &buf);
rsp.blockNum++;
}
} else {
ASSERT(0);
}
if (rsp.blockNum == 0) {
taosWUnLockLatch(&pExec->pushHandle.lock);
continue;
}
ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
rsp.rspOffset = fetchOffset;
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
return -1;
}
((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP;
((SMqRspHead*)buf)->epoch = pExec->pushHandle.epoch;
((SMqRspHead*)buf)->consumerId = pExec->pushHandle.consumerId;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqDataBlkRsp(&abuf, &rsp);
pMsg->pCont = buf;
pMsg->contLen = tlen;
pMsg->code = 0;
tmsgSendRsp(pMsg);
atomic_store_ptr(&pExec->pushHandle.handle, NULL);
taosWUnLockLatch(&pExec->pushHandle.lock);
vDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld",
TD_VID(pTq->pVnode), fetchOffset, pExec->pushHandle.consumerId, pExec->pushHandle.epoch, rsp.blockNum,
rsp.reqOffset, rsp.rspOffset);
// TODO destroy
taosArrayDestroy(rsp.blockData);
taosArrayDestroy(rsp.blockDataLen);
}
return 0;
}
int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) {
if (msgType != TDMT_VND_SUBMIT) return 0;
void* data = taosMemoryMalloc(msgLen);
if (data == NULL) {
return -1;
@ -60,7 +197,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
memcpy(data, msg, msgLen);
if (msgType == TDMT_VND_SUBMIT) {
if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, version) != 0) {
if (tsdbUpdateSmaWindow(pTq->pVnode->pTsdb, msg, ver) != 0) {
return -1;
}
}
@ -70,6 +207,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
.pCont = data,
.contLen = msgLen,
};
tmsgPutToQueue(&pTq->pVnode->msgCb, FETCH_QUEUE, &req);
#if 0
@ -104,7 +242,11 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t versi
return 0;
}
int tqCommit(STQ* pTq) { return tqStorePersist(pTq->tqMeta); }
int tqCommit(STQ* pTq) {
// do nothing
/*return tqStorePersist(pTq->tqMeta);*/
return 0;
}
int32_t tqGetTopicHandleSize(const STqTopic* pTopic) {
return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->physicalPlan) + strlen(pTopic->qmsg) +
@ -219,10 +361,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
}
for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
pTopic->buffer.output[j].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
SReadHandle handle = {
.reader = pReadHandle,
.meta = pTq->pVnodeMeta,
.meta = pTq->pVnode->pMeta,
};
pTopic->buffer.output[j].pReadHandle = pReadHandle;
pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
@ -235,9 +377,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
SMqPollReqV2* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
int64_t waitTime = pReq->blockingTime;
int32_t reqEpoch = pReq->epoch;
int64_t fetchOffset;
// get offset to fetch message
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
fetchOffset = walGetFirstVer(pTq->pWal);
} else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
@ -249,7 +393,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
vDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req %ld %ld", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
STqExec* pExec = taosHashGet(pTq->tqMetaNew, pReq->subKey, strlen(pReq->subKey));
STqExec* pExec = taosHashGet(pTq->execs, pReq->subKey, strlen(pReq->subKey));
ASSERT(pExec);
int32_t consumerEpoch = atomic_load_32(&pExec->epoch);
@ -259,8 +403,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
SMqDataBlkRsp rsp = {0};
rsp.reqOffset = pReq->currentOffset;
rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
rsp.blockData = taosArrayInit(0, sizeof(void*));
rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
while (1) {
consumerEpoch = atomic_load_32(&pExec->epoch);
@ -271,12 +415,34 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
}
SWalReadHead* pHead;
if (walReadWithHandle_s(pExec->pReadHandle, fetchOffset, &pHead) < 0) {
if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) {
// TODO: no more log, set timer to wait blocking time
// if data inserted during waiting, launch query and
// response to user
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset);
#if 0
// add to pushMgr
taosWLockLatch(&pExec->pushHandle.lock);
pExec->pushHandle.consumerId = consumerId;
pExec->pushHandle.epoch = reqEpoch;
pExec->pushHandle.reqOffset = rsp.reqOffset;
pExec->pushHandle.skipLogNum = rsp.skipLogNum;
pExec->pushHandle.handle = pMsg;
taosWUnLockLatch(&pExec->pushHandle.lock);
// TODO add timer
// TODO: the pointer will always be valid?
taosHashPut(pTq->pushMgr, &consumerId, sizeof(int64_t), &pExec, sizeof(void*));
taosArrayDestroy(rsp.blockData);
taosArrayDestroy(rsp.blockDataLen);
return 0;
#endif
break;
}
@ -285,41 +451,73 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
if (pHead->msgType == TDMT_VND_SUBMIT) {
SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
qTaskInfo_t task = pExec->task[workerId];
ASSERT(task);
qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
while (1) {
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(0);
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
qTaskInfo_t task = pExec->task[workerId];
ASSERT(task);
qSetStreamInput(task, pCont, STREAM_DATA_TYPE_SUBMIT_BLOCK, false);
while (1) {
SSDataBlock* pDataBlock = NULL;
uint64_t ts = 0;
if (qExecTask(task, &pDataBlock, &ts) < 0) {
ASSERT(0);
}
if (pDataBlock == NULL) break;
ASSERT(pDataBlock->info.rows != 0);
ASSERT(pDataBlock->info.numOfCols != 0);
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
void* buf = taosMemoryCalloc(1, dataStrLen);
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
pRetrieve->useconds = ts;
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
// TODO enable compress
int32_t actualLen = 0;
blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(rsp.blockDataLen, &actualLen);
taosArrayPush(rsp.blockData, &buf);
rsp.blockNum++;
}
if (pDataBlock == NULL) break;
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
STqReadHandle* pReader = pExec->pExecReader[workerId];
tqReadHandleSetMsg(pReader, pCont, 0);
while (tqNextDataBlock(pReader)) {
SSDataBlock block = {0};
if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.rows,
&block.info.numOfCols) < 0) {
ASSERT(0);
}
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(&block);
void* buf = taosMemoryCalloc(1, dataStrLen);
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
/*pRetrieve->useconds = 0;*/
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->numOfRows = htonl(block.info.rows);
ASSERT(pDataBlock->info.rows != 0);
ASSERT(pDataBlock->info.numOfCols != 0);
int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pDataBlock);
void* buf = taosMemoryCalloc(1, dataStrLen);
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf;
pRetrieve->useconds = ts;
pRetrieve->precision = TSDB_DEFAULT_PRECISION;
pRetrieve->compressed = 0;
pRetrieve->completed = 1;
pRetrieve->numOfRows = htonl(pDataBlock->info.rows);
// TODO enable compress
int32_t actualLen = 0;
blockCompressEncode(pDataBlock, pRetrieve->data, &actualLen, pDataBlock->info.numOfCols, false);
actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(rsp.blockDataLen, &actualLen);
taosArrayPush(rsp.blockData, &buf);
rsp.blockNum++;
// TODO enable compress
int32_t actualLen = 0;
blockCompressEncode(&block, pRetrieve->data, &actualLen, block.info.numOfCols, false);
actualLen += sizeof(SRetrieveTableRsp);
ASSERT(actualLen <= dataStrLen);
taosArrayPush(rsp.blockDataLen, &actualLen);
taosArrayPush(rsp.blockData, &buf);
rsp.blockNum++;
}
} else {
ASSERT(0);
}
}
// TODO batch optimization
// TODO batch optimization:
// TODO continue scan until meeting batch requirement
if (rsp.blockNum != 0) break;
rsp.skipLogNum++;
fetchOffset++;
@ -572,10 +770,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
// TODO: persist meta into tdb
int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
SMqRebVgReq req;
SMqRebVgReq req = {0};
tDecodeSMqRebVgReq(msg, &req);
// todo lock
STqExec* pExec = taosHashGet(pTq->tqMetaNew, req.subKey, strlen(req.subKey));
STqExec* pExec = taosHashGet(pTq->execs, req.subKey, strlen(req.subKey));
if (pExec == NULL) {
ASSERT(req.oldConsumerId == -1);
ASSERT(req.newConsumerId != -1);
@ -586,19 +784,27 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
memcpy(pExec->subKey, req.subKey, TSDB_SUBSCRIBE_KEY_LEN);
pExec->consumerId = req.newConsumerId;
pExec->epoch = -1;
pExec->subType = req.subType;
pExec->withTbName = req.withTbName;
pExec->withSchema = req.withSchema;
pExec->withTag = req.withTag;
pExec->withTagSchema = req.withTagSchema;
pExec->qmsg = req.qmsg;
req.qmsg = NULL;
pExec->pReadHandle = walOpenReadHandle(pTq->pVnode->pWal);
for (int32_t i = 0; i < 4; i++) {
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
SReadHandle handle = {
.reader = pReadHandle,
.meta = pTq->pVnodeMeta,
pExec->pWalReader = walOpenReadHandle(pTq->pVnode->pWal);
for (int32_t i = 0; i < 5; i++) {
pExec->pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
SReadHandle handle = {
.reader = pExec->pExecReader[i],
.meta = pTq->pVnode->pMeta,
};
pExec->task[i] = qCreateStreamExecTaskInfo(pExec->qmsg, &handle);
ASSERT(pExec->task[i]);
}
taosHashPut(pTq->tqMetaNew, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
taosHashPut(pTq->execs, req.subKey, strlen(req.subKey), pExec, sizeof(STqExec));
return 0;
} else {
/*if (req.newConsumerId != -1) {*/
@ -627,12 +833,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int32_t parallel) {
return -1;
}
for (int32_t i = 0; i < parallel; i++) {
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta);
SReadHandle handle = {
.reader = pReadHandle,
.meta = pTq->pVnodeMeta,
.reader = pStreamReader,
.meta = pTq->pVnode->pMeta,
};
pTask->exec.runners[i].inputHandle = pReadHandle;
pTask->exec.runners[i].inputHandle = pStreamReader;
pTask->exec.runners[i].executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle);
ASSERT(pTask->exec.runners[i].executor);
}

6
source/dnode/vnode/src/tq/tqMetaStore.c
View File

@ -86,7 +86,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
}
strcpy(pMeta->dirPath, path);
char name[pathLen + 10];
char *name = taosMemoryMalloc(pathLen + 10) ;
strcpy(name, path);
if (!taosDirExist(name) && taosMkDir(name) != 0) {
@ -99,6 +99,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s ", name, terrstr());
// free memory
taosMemoryFree(name);
return NULL;
}
@ -106,6 +107,7 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
pMeta->unpersistHead = taosMemoryCalloc(1, sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
taosMemoryFree(name);
return NULL;
}
pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead;
@ -116,8 +118,10 @@ STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, F
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr());
taosMemoryFree(name);
return NULL;
}
taosMemoryFree(name);
pMeta->pFile = pFile;

70
source/dnode/vnode/src/tq/tqPush.c
View File

@ -12,73 +12,3 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnodeInt.h"
int32_t tqPushMgrInit() {
//
int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1);
if (old == 1) return 0;
tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
return 0;
}
void tqPushMgrCleanUp() {
int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0);
if (old == 0) return;
taosTmrStop(tqPushMgmt.timer);
taosTmrCleanUp(tqPushMgmt.timer);
}
STqPushMgr* tqPushMgrOpen() {
STqPushMgr* mgr = taosMemoryMalloc(sizeof(STqPushMgr));
if (mgr == NULL) {
return NULL;
}
mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
return mgr;
}
void tqPushMgrClose(STqPushMgr* pushMgr) {
taosHashCleanup(pushMgr->pHash);
taosMemoryFree(pushMgr);
}
STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) {
STqClientPusher* clientPusher = taosMemoryMalloc(sizeof(STqClientPusher));
if (clientPusher == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
clientPusher->type = TQ_PUSHER_TYPE__CLIENT;
clientPusher->pMsg = pMsg;
clientPusher->consumerId = consumerId;
clientPusher->ttl = ttl;
if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(clientPusher);
// TODO send rsp back
return NULL;
}
return clientPusher;
}
STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) {
STqStreamPusher* streamPusher = taosMemoryMalloc(sizeof(STqStreamPusher));
if (streamPusher == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
streamPusher->type = TQ_PUSHER_TYPE__STREAM;
streamPusher->nodeType = 0;
streamPusher->streamId = streamId;
/*memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));*/
if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(streamPusher);
return NULL;
}
return streamPusher;
}

5
source/dnode/vnode/src/tq/tqRead.c
View File

@ -98,7 +98,8 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
return false;
}
int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows) {
int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, int32_t* pNumOfRows,
int16_t* pNumOfCols) {
/*int32_t sversion = pHandle->pBlock->sversion;*/
// TODO set to real sversion
int32_t sversion = 0;
@ -120,7 +121,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper;
*pNumOfRows = pHandle->pBlock->numOfRows;
/*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList);
if (colNumNeed > pSchemaWrapper->nCols) {
@ -158,6 +158,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p
}
int32_t colActual = taosArrayGetSize(*ppCols);
*pNumOfCols = colActual;
// TODO in stream shuffle case, fetch groupId
*pGroupId = 0;

27
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -82,6 +82,18 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
// sync integration
if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
SJson *pNodeInfoArr = tjsonCreateArray();
tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", pNodeInfoArr);
for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) {
SJson *pNodeInfo = tjsonCreateObject();
tjsonAddIntegerToObject(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
tjsonAddStringToObject(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
tjsonAddItemToArray(pNodeInfoArr, pNodeInfo);
}
return 0;
}
@ -120,6 +132,21 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
// sync integration
if (tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
if (tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(pNodeInfoArr);
assert(arraySize == pCfg->syncCfg.replicaNum);
for (int i = 0; i < arraySize; ++i) {
SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i);
assert(pNodeInfo != NULL);
tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
}
return 0;
}

15
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -158,12 +158,23 @@ int vnodeSyncCommit(SVnode *pVnode) {
}
static int vnodeCommit(void *arg) {
SVnode *pVnode = (SVnode *)arg;
SVnode *pVnode = (SVnode *)arg;
char dir[TSDB_FILENAME_LEN];
SVnodeInfo info = {0};
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
info.config = pVnode->config;
info.state.committed = pVnode->state.applied;
info.state.applied = pVnode->state.applied;
vnodeSaveInfo(dir, &info);
// metaCommit(pVnode->pMeta);
tqCommit(pVnode->pTq);
tsdbCommit(pVnode->pTsdb);
vnodeCommitInfo(dir, &info);
vnodeBufPoolRecycle(pVnode);
tsem_post(&(pVnode->canCommit));
return 0;
@ -185,6 +196,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {
const SVState *pState = (SVState *)pObj;
if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "applied version", pState->applied) < 0) return -1;
return 0;
}
@ -193,6 +205,7 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
SVState *pState = (SVState *)pObj;
if (tjsonGetNumberValue(pJson, "commit version", pState->committed) < 0) return -1;
if (tjsonGetNumberValue(pJson, "applied version", pState->applied) < 0) return -1;
return 0;
}

21
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -35,6 +35,8 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
info.config = *pCfg;
info.state.committed = -1;
info.state.applied = -1;
if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) {
vError("vgId: %d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno));
@ -75,8 +77,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
pVnode->path = (char *)&pVnode[1];
strcpy(pVnode->path, path);
pVnode->config = info.config;
pVnode->state.committed = info.state.committed;
pVnode->state.processed = pVnode->state.applied = pVnode->state.committed;
pVnode->state = info.state;
pVnode->pTfs = pTfs;
pVnode->msgCb = msgCb;
@ -112,7 +113,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
// open tq
sprintf(tdir, "%s%s%s", dir, TD_DIRSEP, VNODE_TQ_DIR);
pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal, pVnode->pMeta, vBufPoolGetMAF(pVnode));
pVnode->pTq = tqOpen(tdir, pVnode, pVnode->pWal);
if (pVnode->pTq == NULL) {
vError("vgId: %d failed to open vnode tq since %s", TD_VID(pVnode), tstrerror(terrno));
goto _err;
@ -124,6 +125,12 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
goto _err;
}
// sync integration
// open sync
if (vnodeSyncOpen(pVnode, dir)) {
goto _err;
}
#if 0
if (vnodeBegin() < 0) {
goto _err;
@ -149,6 +156,10 @@ void vnodeClose(SVnode *pVnode) {
vnodeSyncCommit(pVnode);
// close vnode
vnodeQueryClose(pVnode);
// sync integration
vnodeSyncClose(pVnode);
walClose(pVnode->pWal);
tqClose(pVnode->pTq);
tsdbClose(pVnode->pTsdb);
@ -159,3 +170,7 @@ void vnodeClose(SVnode *pVnode) {
taosMemoryFree(pVnode);
}
}
int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }

101
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -13,6 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sync.h"
#include "syncTools.h"
#include "vnodeInt.h"
static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq);
@ -21,6 +23,7 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq);
static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp);
int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
#if 0
SNodeMsg *pMsg;
SRpcMsg *pRpc;
@ -38,6 +41,7 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
walFsync(pVnode->pWal, false);
#endif
return 0;
}
@ -87,21 +91,6 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
// TODO: handle error
}
break;
#if 0
case TDMT_VND_MQ_SET_CONN: {
if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
// TODO: handle error
}
} break;
case TDMT_VND_MQ_REB: {
if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
}
} break;
case TDMT_VND_MQ_CANCEL_CONN: {
if (tqProcessCancelConnReq(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead))) < 0) {
}
} break;
#endif
case TDMT_VND_TASK_DEPLOY: {
if (tqProcessTaskDeploy(pVnode->pTq, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)),
pMsg->contLen - sizeof(SMsgHead)) < 0) {
@ -202,8 +191,86 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data);
}
// sync integration
int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
/*vInfo("sync message is processed");*/
SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
assert(pSyncNode != NULL);
ESyncState state = syncGetMyRole(pVnode->sync);
SyncTerm currentTerm = syncGetMyTerm(pVnode->sync);
SMsgHead *pHead = pMsg->pCont;
char logBuf[512];
char *syncNodeStr = sync2SimpleStr(pVnode->sync);
snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr);
syncRpcMsgLog2(logBuf, pMsg);
taosMemoryFree(syncNodeStr);
SRpcMsg *pRpcMsg = pMsg;
if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) {
SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
syncTimeoutDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnPingCb(pSyncNode, pSyncMsg);
syncPingDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
syncPingReplyDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
syncClientRequestDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
syncRequestVoteDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
syncRequestVoteReplyDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
syncAppendEntriesDestroy(pSyncMsg);
} else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
assert(pSyncMsg != NULL);
syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
syncAppendEntriesReplyDestroy(pSyncMsg);
} else {
vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType);
}
syncNodeRelease(pSyncNode);
return 0;
}
@ -236,7 +303,7 @@ static int vnodeProcessCreateTbReq(SVnode *pVnode, SRpcMsg *pMsg, void *pReq, SR
tDeserializeSVCreateTbBatchReq(pReq, &vCreateTbBatchReq);
int reqNum = taosArrayGetSize(vCreateTbBatchReq.pArray);
STbDdlHandle ddlHandle = {
STbDdlH ddlHandle = {
.ahandle = pVnode->pTsdb,
.result = NULL,
.fp = tsdbFetchTbUidList,

178
source/dnode/vnode/src/vnd/vnodeSync.c Normal file
View File

@ -0,0 +1,178 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sync.h"
#include "syncTools.h"
#include "tmsgcb.h"
#include "vnodeInt.h"
// sync integration
int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
SSyncInfo syncInfo;
syncInfo.vgId = pVnode->config.vgId;
SSyncCfg *pCfg = &(syncInfo.syncCfg);
pCfg->replicaNum = pVnode->config.syncCfg.replicaNum;
pCfg->myIndex = pVnode->config.syncCfg.myIndex;
memcpy(pCfg->nodeInfo, pVnode->config.syncCfg.nodeInfo, sizeof(pCfg->nodeInfo));
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s/sync", path);
syncInfo.pWal = pVnode->pWal;
syncInfo.pFsm = syncVnodeMakeFsm(pVnode);
syncInfo.rpcClient = NULL;
syncInfo.FpSendMsg = vnodeSendMsg;
syncInfo.queue = NULL;
syncInfo.FpEqMsg = vnodeSyncEqMsg;
pVnode->sync = syncOpen(&syncInfo);
assert(pVnode->sync > 0);
// for test
setPingTimerMS(pVnode->sync, 3000);
setElectTimerMS(pVnode->sync, 500);
setHeartbeatTimerMS(pVnode->sync, 100);
return 0;
}
int32_t vnodeSyncStart(SVnode *pVnode) {
syncStart(pVnode->sync);
return 0;
}
void vnodeSyncClose(SVnode *pVnode) {
// stop by ref id
syncStop(pVnode->sync);
}
void vnodeSyncSetQ(SVnode *pVnode, void *qHandle) { syncSetQ(pVnode->sync, (void *)(&(pVnode->msgCb))); }
void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle) { syncSetRpc(pVnode->sync, (void *)(&(pVnode->msgCb))); }
int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg) {
int32_t ret = 0;
SMsgCb *pMsgCb = qHandle;
if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) {
tmsgPutToQueue(qHandle, SYNC_QUEUE, pMsg);
} else {
vError("vnodeSyncEqMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE);
}
return ret;
}
int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t ret = 0;
SMsgCb *pMsgCb = rpcHandle;
if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) {
tmsgSendReq(rpcHandle, pEpSet, pMsg);
} else {
vError("vnodeSendMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE);
}
return ret;
}
int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
SVnode *pVnode = (SVnode *)(pFsm->data);
vnodeGetSnapshot(pVnode, pSnapshot);
/*
pSnapshot->data = NULL;
pSnapshot->lastApplyIndex = 0;
pSnapshot->lastApplyTerm = 0;
*/
return 0;
}
void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
SyncIndex beginIndex = SYNC_INDEX_INVALID;
if (pFsm->FpGetSnapshot != NULL) {
SSnapshot snapshot;
pFsm->FpGetSnapshot(pFsm, &snapshot);
beginIndex = snapshot.lastApplyIndex;
}
if (cbMeta.index > beginIndex) {
char logBuf[256];
snprintf(
logBuf, sizeof(logBuf),
"==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
SVnode *pVnode = (SVnode *)(pFsm->data);
SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
SRpcMsg applyMsg;
syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
syncApplyMsgDestroy(pSyncApplyMsg);
/*
SRpcMsg applyMsg;
applyMsg = *pMsg;
applyMsg.pCont = rpcMallocCont(applyMsg.contLen);
assert(applyMsg.contLen == pMsg->contLen);
memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen);
*/
// recover handle for response
SRpcMsg saveRpcMsg;
int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
applyMsg.handle = saveRpcMsg.handle;
applyMsg.ahandle = saveRpcMsg.ahandle;
} else {
applyMsg.handle = NULL;
applyMsg.ahandle = NULL;
}
// put to applyQ
tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
} else {
char logBuf[256];
snprintf(logBuf, sizeof(logBuf),
"==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, "
"beginIndex :%ld\n",
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
beginIndex);
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
}
}
void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
char logBuf[256];
snprintf(logBuf, sizeof(logBuf),
"==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
}
void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
char logBuf[256];
snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
}
SSyncFSM *syncVnodeMakeFsm(SVnode *pVnode) {
SSyncFSM *pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM));
pFsm->data = pVnode;
pFsm->FpCommitCb = vnodeSyncCommitCb;
pFsm->FpPreCommitCb = vnodeSyncPreCommitCb;
pFsm->FpRollBackCb = vnodeSyncRollBackCb;
pFsm->FpGetSnapshot = vnodeSyncGetSnapshotCb;
return pFsm;
}

4
source/libs/catalog/inc/catalogInt.h
View File

@ -104,6 +104,10 @@ typedef struct SCatalog {
typedef struct SCtgApiStat {
#ifdef WINDOWS
size_t avoidCompilationErrors;
#endif
} SCtgApiStat;
typedef struct SCtgRuntimeStat {

117
source/libs/catalog/test/catalogTests.cpp
View File

@ -24,6 +24,9 @@
#pragma GCC diagnostic ignored "-Wformat"
#include <addr_any.h>
#ifdef WINDOWS
#define TD_USE_WINSOCK
#endif
#include "os.h"
#include "tglobal.h"
#include "catalog.h"
@ -155,11 +158,11 @@ int32_t ctgTestGetVgNumFromVgVersion(int32_t vgVersion) {
}
void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) {
SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(cn.dbname, "db1");
strcpy(cn.tname, ctgTestCTablename);
SName sn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName sn = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(sn.dbname, "db1");
strcpy(sn.tname, ctgTestSTablename);
@ -577,9 +580,16 @@ void ctgTestSetRspDbVgroups() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspDbVgroups);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspDbVgroups);
}
@ -590,9 +600,16 @@ void ctgTestSetRspTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspTableMeta);
}
@ -603,9 +620,16 @@ void ctgTestSetRspCTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspCTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspCTableMeta);
}
@ -616,9 +640,16 @@ void ctgTestSetRspSTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspSTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspSTableMeta);
}
@ -629,9 +660,16 @@ void ctgTestSetRspMultiSTableMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspMultiSTableMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspMultiSTableMeta);
}
@ -642,9 +680,16 @@ void ctgTestSetRspByIdx() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspByIdx);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspByIdx);
}
@ -656,9 +701,16 @@ void ctgTestSetRspDbVgroupsAndNormalMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndNormalMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspDbVgroupsAndNormalMeta);
}
@ -669,9 +721,16 @@ void ctgTestSetRspDbVgroupsAndChildMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndChildMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspDbVgroupsAndChildMeta);
}
@ -682,9 +741,16 @@ void ctgTestSetRspDbVgroupsAndSuperMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndSuperMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspDbVgroupsAndSuperMeta);
}
@ -695,9 +761,16 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() {
static Stub stub;
stub.set(rpcSendRecv, ctgTestRspDbVgroupsAndMultiSuperMeta);
{
#ifdef WINDOWS
AddrAny any;
std::map<std::string,void*> result;
any.get_func_addr("rpcSendRecv", result);
#endif
#ifdef LINUX
AddrAny any("libtransport.so");
std::map<std::string, void *> result;
any.get_global_func_addr_dynsym("^rpcSendRecv$", result);
#endif
for (const auto &f : result) {
stub.set(f.second, ctgTestRspDbVgroupsAndMultiSuperMeta);
}
@ -789,7 +862,7 @@ void *ctgTestGetCtableMetaThread(void *param) {
STableMeta *tbMeta = NULL;
bool inCache = false;
SName cn = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName cn = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(cn.dbname, "db1");
strcpy(cn.tname, ctgTestCTablename);
@ -871,7 +944,7 @@ TEST(tableMeta, normalTable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -978,7 +1051,7 @@ TEST(tableMeta, childTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
@ -1085,7 +1158,7 @@ TEST(tableMeta, superTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -1210,7 +1283,7 @@ TEST(tableMeta, rmStbMeta) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -1280,7 +1353,7 @@ TEST(tableMeta, updateStbMeta) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -1378,7 +1451,7 @@ TEST(refreshGetMeta, normal2normal) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -1457,7 +1530,7 @@ TEST(refreshGetMeta, normal2notexist) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -1529,7 +1602,7 @@ TEST(refreshGetMeta, normal2child) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentCTableName = ctgTestTablename;
@ -1613,7 +1686,7 @@ TEST(refreshGetMeta, stable2child) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentSTableName = ctgTestTablename;
@ -1699,7 +1772,7 @@ TEST(refreshGetMeta, stable2stable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentSTableName = ctgTestTablename;
@ -1786,7 +1859,7 @@ TEST(refreshGetMeta, child2stable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
ctgTestCurrentCTableName = ctgTestTablename;
@ -1871,7 +1944,7 @@ TEST(tableDistVgroup, normalTable) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -1913,7 +1986,7 @@ TEST(tableDistVgroup, childTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestCTablename);
@ -1956,7 +2029,7 @@ TEST(tableDistVgroup, superTableCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestSTablename);
@ -2006,7 +2079,7 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2091,7 +2164,7 @@ TEST(multiThread, getSetRmSameDbVgroup) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2143,7 +2216,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2195,7 +2268,7 @@ TEST(multiThread, ctableMeta) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
strcpy(n.tname, ctgTestTablename);
@ -2247,7 +2320,7 @@ TEST(rentTest, allRent) {
code = catalogGetHandle(ctgTestClusterId, &pCtg);
ASSERT_EQ(code, 0);
SName n = {.type = TSDB_TABLE_NAME_T, .acctId = 1};
SName n = { TSDB_TABLE_NAME_T, 1, {0}, {0} };
strcpy(n.dbname, "db1");
for (int32_t i = 1; i <= 10; ++i) {

Some files were not shown because too many files have changed in this diff Show More