xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2021-12-21 18:33:02 +08:00
parent ae5541f536 26a3ff9b0f
commit 7a622fa63a
147 changed files with 7744 additions and 8842 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Jenkinsfile vendored
View File

@ -42,7 +42,6 @@ def pre_test(){
killall -9 gdb || echo "no gdb running"
killall -9 python3.8 || echo "no python program running"
cd ${WKC}
git reset --hard HEAD~10 >/dev/null
'''
script {
if (env.CHANGE_TARGET == 'master') {
@ -75,11 +74,9 @@ def pre_test(){
git pull >/dev/null
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
git clean -dfx
git clean -dfx
export TZ=Asia/Harbin
date
rm -rf debug
mkdir debug
cd debug
cmake .. > /dev/null
@ -109,6 +106,10 @@ pipeline {
abortPreviousBuilds()
}
pre_test()
sh'''
cd ${WKC}/tests
./test-all.sh b1fq
'''
}
}
// stage('Parallel test stage') {

6
cmake/cmake.options
View File

@ -44,6 +44,12 @@ option(
OFF
)
option(
BUILD_WITH_UV
"If build with libuv"
OFF
)
option(
BUILD_WITH_CRAFT
"If build with canonical-raft"

5
cmake/craft_CMakeLists.txt.in
View File

@ -4,9 +4,10 @@ ExternalProject_Add(craft
GIT_REPOSITORY https://github.com/canonical/raft.git
GIT_TAG v0.11.2
SOURCE_DIR "${CMAKE_CONTRIB_DIR}/craft"
BINARY_DIR "${CMAKE_CONTRIB_DIR}/craft/.libs"
BINARY_DIR "${CMAKE_CONTRIB_DIR}/craft"
#BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND "autoreconf -i && ./configure"
# https://answers.ros.org/question/333125/how-to-include-external-automakeautoconf-projects-into-ament_cmake/
CONFIGURE_COMMAND COMMAND autoreconf -i COMMAND ./configure --enable-example
BUILD_COMMAND "$(MAKE)"
INSTALL_COMMAND ""
TEST_COMMAND ""

12
cmake/libuv_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,12 @@
# libuv
ExternalProject_Add(libuv
GIT_REPOSITORY https://github.com/libuv/libuv.git
GIT_TAG v1.42.0
SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv"
BINARY_DIR ""
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

12
cmake/stub_CMakeLists.txt.in Normal file
View File

@ -0,0 +1,12 @@
# stub
ExternalProject_Add(stub
GIT_REPOSITORY https://github.com/coolxv/cpp-stub.git
GIT_SUBMODULES "src"
SOURCE_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub"
BINARY_DIR "${CMAKE_CONTRIB_DIR}/cpp-stub/src"
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)

27
contrib/CMakeLists.txt
View File

@ -12,6 +12,7 @@ configure_file("${CMAKE_SUPPORT_DIR}/deps_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}
# googletest
if(${BUILD_TEST})
cat("${CMAKE_SUPPORT_DIR}/gtest_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
cat("${CMAKE_SUPPORT_DIR}/stub_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_TEST})
# lz4
@ -37,12 +38,18 @@ endif(${BUILD_WITH_ROCKSDB})
# canonical-raft
if(${BUILD_WITH_CRAFT})
cat("${CMAKE_SUPPORT_DIR}/craft_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
SET(BUILD_WITH_UV ON CACHE BOOL "craft need libuv" FORCE)
endif(${BUILD_WITH_CRAFT})
#libuv
if(${BUILD_WITH_UV})
cat("${CMAKE_SUPPORT_DIR}/libuv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_UV})
# bdb
if(${BUILD_WITH_BDB})
cat("${CMAKE_SUPPORT_DIR}/bdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif(${BUILD_WITH_DBD})
endif(${BUILD_WITH_BDB})
# sqlite
if(${BUILD_WITH_SQLITE})
@ -73,6 +80,11 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build .
# googletest
if(${BUILD_TEST})
add_subdirectory(googletest)
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
endif(${BUILD_TEST})
# cJson
@ -154,13 +166,18 @@ if(${BUILD_WITH_CRAFT})
add_library(craft STATIC IMPORTED GLOBAL)
set_target_properties(craft PROPERTIES
IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/craft/.libs/libraft.a"
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/craft"
)
target_link_libraries(craft
INTERFACE pthread
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/craft/include"
)
# target_link_libraries(craft
# INTERFACE pthread
# )
endif(${BUILD_WITH_CRAFT})
# LIBUV
if(${BUILD_WITH_UV})
add_subdirectory(libuv)
endif(${BUILD_WITH_UV})
# BDB
if(${BUILD_WITH_BDB})
add_library(bdb STATIC IMPORTED GLOBAL)

4
contrib/test/CMakeLists.txt
View File

@ -15,4 +15,8 @@ if(${BUILD_WITH_SQLITE})
add_subdirectory(sqlite)
endif(${BUILD_WITH_SQLITE})
if(${BUILD_WITH_CRAFT})
add_subdirectory(craft)
endif(${BUILD_WITH_CRAFT})
add_subdirectory(tdev)

2
contrib/test/craft/CMakeLists.txt Normal file
View File

@ -0,0 +1,2 @@
add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)

130
include/common/taosmsg.h
View File

@ -50,9 +50,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RES_READY, "res-ready" )
// message from client to mnode
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" )
@ -95,22 +94,22 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_NETWORK_TEST, "nettest" )
// message from vnode to dnode
// message from mnode to vnode
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-in" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-in" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-in" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_STB_IN, "create-stb-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STB_IN, "alter-stb-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STB_IN, "drop-stb-internal" )
// message from mnode to mnode
// message from mnode to qnode
// message from mnode to dnode
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_MNODE_IN, "alter-mnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE_IN, "drop-mnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE_IN, "config-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_VNODE_IN, "create-vnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_VNODE_IN, "alter-vnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_VNODE_IN, "drop-vnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_AUTH_VNODE_IN, "auth-vnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SYNC_VNODE_IN, "sync-vnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_COMPACT_VNODE_IN, "compact-vnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_MNODE_IN, "create-mnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_MNODE_IN, "alter-mnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_MNODE_IN, "drop-mnode-internal" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONFIG_DNODE_IN, "config-dnode-internal" )
// message from qnode to vnode
// message from qnode to mnode
@ -225,6 +224,7 @@ typedef struct SBuildUseDBInput {
int32_t vgVersion;
} SBuildUseDBInput;
#pragma pack(push, 1)
// null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta
@ -290,37 +290,6 @@ typedef struct SSchema {
char name[TSDB_COL_NAME_LEN];
} SSchema;
typedef struct {
int32_t contLen;
int32_t vgId;
int8_t tableType;
int16_t numOfColumns;
int16_t numOfTags;
int32_t tid;
int32_t sversion;
int32_t tversion;
int32_t tagDataLen;
int32_t sqlDataLen;
uint64_t uid;
uint64_t superTableUid;
uint64_t createdTime;
char tableFname[TSDB_TABLE_FNAME_LEN];
char stbFname[TSDB_TABLE_FNAME_LEN];
char data[];
} SMDCreateTableMsg;
typedef struct {
int32_t len; // one create table message
char tableName[TSDB_TABLE_FNAME_LEN];
int8_t igExists;
int8_t getMeta;
int16_t numOfTags;
int16_t numOfColumns;
int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string
int8_t reserved[16];
char schema[];
} SCreateTableMsg;
typedef struct {
char name[TSDB_TABLE_FNAME_LEN];
int8_t igExists;
@ -341,16 +310,50 @@ typedef struct {
} SAlterStbMsg;
typedef struct {
char tableFname[TSDB_TABLE_FNAME_LEN];
char db[TSDB_FULL_DB_NAME_LEN];
int16_t type; /* operation type */
int16_t numOfCols; /* number of schema */
int32_t tagValLen;
SSchema schema[];
// tagVal is padded after schema
// char tagVal[];
SMsgHead head;
char name[TSDB_TABLE_FNAME_LEN];
uint64_t suid;
int32_t sverson;
uint32_t ttl;
uint32_t keep;
int32_t numOfTags;
int32_t numOfColumns;
SSchema pSchema[];
} SCreateStbInternalMsg;
typedef struct {
SMsgHead head;
char name[TSDB_TABLE_FNAME_LEN];
uint64_t suid;
} SDropStbInternalMsg;
typedef struct {
SMsgHead head;
char name[TSDB_TABLE_FNAME_LEN];
char stbFname[TSDB_TABLE_FNAME_LEN];
int8_t tableType;
uint64_t suid;
int32_t sversion;
int32_t numOfTags;
int32_t numOfColumns;
int32_t tagDataLen;
char data[];
} SCreateTableMsg;
typedef struct {
SMsgHead head;
char name[TSDB_TABLE_FNAME_LEN];
int8_t type; /* operation type */
int32_t numOfCols; /* number of schema */
int32_t numOfTags;
char data[];
} SAlterTableMsg;
typedef struct {
SMsgHead head;
char name[TSDB_TABLE_FNAME_LEN];
} SDropTableMsg;
typedef struct {
SMsgHead head;
int64_t uid;
@ -545,8 +548,8 @@ typedef struct {
int32_t sqlstrLen; // sql query string
int32_t prevResultLen; // previous result length
int32_t numOfOperator;
int32_t tableScanOperator; // table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function
int32_t tableScanOperator;// table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function
int32_t udfContentOffset;
int32_t udfContentLen;
SColumnInfo tableCols[];
@ -591,8 +594,8 @@ typedef struct {
int32_t daysToKeep0;
int32_t daysToKeep1;
int32_t daysToKeep2;
int32_t minRowsPerFileBlock;
int32_t maxRowsPerFileBlock;
int32_t minRows;
int32_t maxRows;
int32_t commitTime;
int32_t fsyncPeriod;
int8_t walLevel;
@ -706,7 +709,7 @@ typedef struct {
SVnodeLoad data[];
} SVnodeLoads;
typedef struct SStatusMsg {
typedef struct {
int32_t sver;
int32_t dnodeId;
int32_t clusterId;
@ -756,6 +759,7 @@ typedef struct {
int32_t dnodeId;
char db[TSDB_FULL_DB_NAME_LEN];
uint64_t dbUid;
int32_t vgVersion;
int32_t cacheBlockSize;
int32_t totalBlocks;
int32_t daysPerFile;
@ -1005,27 +1009,35 @@ typedef struct {
// mq related
typedef struct {
} SMqConnectReq;
typedef struct {
} SMqConnectRsp;
typedef struct {
} SMqDisconnectReq;
typedef struct {
} SMqDisconnectRsp;
typedef struct {
} SMqAckReq;
typedef struct {
} SMqAckRsp;
typedef struct {
} SMqResetReq;
typedef struct {
} SMqResetRsp;
// mq related end

4
include/common/tmsgtype.h
View File

@ -35,7 +35,7 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAG_VAL, "update-tag-val" )
// the SQL below is for mgmt node
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
@ -54,7 +54,7 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" )
TSDB_DEFINE_SQL_TYPE(TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica")
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica")
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" )

192
include/dnode/vnode/tq/tq.h
View File

@ -16,86 +16,76 @@
#ifndef _TD_TQ_H_
#define _TD_TQ_H_
#include "common.h"
#include "mallocator.h"
#include "os.h"
#include "taoserror.h"
#include "taosmsg.h"
#include "tlist.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct TmqMsgHead {
typedef struct STqMsgHead {
int32_t protoVer;
int32_t msgType;
int64_t cgId;
int64_t clientId;
} TmqMsgHead;
} STqMsgHead;
typedef struct TmqOneAck {
typedef struct STqOneAck {
int64_t topicId;
int64_t consumeOffset;
} TmqOneAck;
} STqOneAck;
typedef struct TmqAcks {
typedef struct STqAcks {
int32_t ackNum;
// should be sorted
TmqOneAck acks[];
} TmqAcks;
STqOneAck acks[];
} STqAcks;
// TODO: put msgs into common
typedef struct TmqConnectReq {
TmqMsgHead head;
TmqAcks acks;
} TmqConnectReq;
typedef struct TmqConnectRsp {
TmqMsgHead head;
int8_t status;
} TmqConnectRsp;
typedef struct TmqDisconnectReq {
TmqMsgHead head;
} TmqDiscconectReq;
typedef struct TmqDisconnectRsp {
TmqMsgHead head;
int8_t status;
} TmqDisconnectRsp;
typedef struct STqSetCurReq {
STqMsgHead head;
int64_t topicId;
int64_t offset;
} STqSetCurReq;
typedef struct STqConsumeReq {
TmqMsgHead head;
TmqAcks acks;
STqMsgHead head;
STqAcks acks;
} STqConsumeReq;
typedef struct TmqMsgContent {
typedef struct STqMsgContent {
int64_t topicId;
int64_t msgLen;
char msg[];
} TmqMsgContent;
} STqMsgContent;
typedef struct STqConsumeRsp {
TmqMsgHead head;
STqMsgHead head;
int64_t bodySize;
TmqMsgContent msgs[];
STqMsgContent msgs[];
} STqConsumeRsp;
typedef struct TmqSubscribeReq {
TmqMsgHead head;
typedef struct STqSubscribeReq {
STqMsgHead head;
int32_t topicNum;
int64_t topic[];
} TmqSubscribeReq;
} STqSubscribeReq;
typedef struct tmqSubscribeRsp {
TmqMsgHead head;
typedef struct STqSubscribeRsp {
STqMsgHead head;
int64_t vgId;
char ep[TSDB_EP_LEN]; // TSDB_EP_LEN
} TmqSubscribeRsp;
} STqSubscribeRsp;
typedef struct TmqHeartbeatReq {
} TmqHeartbeatReq;
typedef struct STqHeartbeatReq {
} STqHeartbeatReq;
typedef struct TmqHeartbeatRsp {
} TmqHeartbeatRsp;
typedef struct STqHeartbeatRsp {
} STqHeartbeatRsp;
typedef struct STqTopicVhandle {
int64_t topicId;
@ -108,48 +98,54 @@ typedef struct STqTopicVhandle {
#define TQ_BUFFER_SIZE 8
typedef struct STqExec {
void* runtimeEnv;
SSDataBlock* (*exec)(void* runtimeEnv);
void* (*assign)(void* runtimeEnv, SSubmitBlk* inputData);
void (*clear)(void* runtimeEnv);
char* (*serialize)(struct STqExec*);
struct STqExec* (*deserialize)(char*);
} STqExec;
typedef struct STqBufferItem {
int64_t offset;
// executors are identical but not concurrent
// so there must be a copy in each item
void* executor;
int64_t size;
void* content;
} STqBufferItem;
STqExec* executor;
int32_t status;
int64_t size;
void* content;
} STqMsgItem;
typedef struct STqBufferHandle {
typedef struct STqTopic {
// char* topic; //c style, end with '\0'
// int64_t cgId;
// void* ahandle;
int64_t nextConsumeOffset;
int64_t floatingCursor;
int64_t topicId;
int32_t head;
int32_t tail;
STqBufferItem buffer[TQ_BUFFER_SIZE];
} STqBufferHandle;
int64_t nextConsumeOffset;
int64_t floatingCursor;
int64_t topicId;
int32_t head;
int32_t tail;
STqMsgItem buffer[TQ_BUFFER_SIZE];
} STqTopic;
typedef struct STqListHandle {
STqBufferHandle bufHandle;
STqTopic topic;
struct STqListHandle* next;
} STqListHandle;
} STqList;
typedef struct STqGroupHandle {
int64_t cId;
int64_t cgId;
void* ahandle;
int32_t topicNum;
STqListHandle* head;
} STqGroupHandle;
typedef struct STqQueryExec {
void* src;
STqBufferItem* dest;
void* executor;
} STqQueryExec;
typedef struct STqGroup {
int64_t clientId;
int64_t cgId;
void* ahandle;
int32_t topicNum;
STqList* head;
SList* topicList; // SList<STqTopic>
void* returnMsg; // SVReadMsg
} STqGroup;
typedef struct STqQueryMsg {
STqQueryExec* exec;
STqMsgItem* item;
struct STqQueryMsg* next;
} STqQueryMsg;
@ -209,15 +205,15 @@ typedef void (*FTqDelete)(void*);
#define TQ_DUP_INTXN_REWRITE 0
#define TQ_DUP_INTXN_REJECT 2
static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; }
static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; }
static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST;
#define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE
typedef struct TqMetaHandle {
typedef struct STqMetaHandle {
int64_t key;
int64_t offset;
int64_t serializedSize;
@ -225,23 +221,25 @@ typedef struct TqMetaHandle {
void* valueInTxn;
} STqMetaHandle;
typedef struct TqMetaList {
STqMetaHandle handle;
struct TqMetaList* next;
// struct TqMetaList* inTxnPrev;
// struct TqMetaList* inTxnNext;
struct TqMetaList* unpersistPrev;
struct TqMetaList* unpersistNext;
typedef struct STqMetaList {
STqMetaHandle handle;
struct STqMetaList* next;
// struct STqMetaList* inTxnPrev;
// struct STqMetaList* inTxnNext;
struct STqMetaList* unpersistPrev;
struct STqMetaList* unpersistNext;
} STqMetaList;
typedef struct TqMetaStore {
typedef struct STqMetaStore {
STqMetaList* bucket[TQ_BUCKET_SIZE];
// a table head
STqMetaList* unpersistHead;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;
// TODO:temporaral use, to be replaced by unified tfile
int idxFd;
int idxFd;
char* dirPath;
int32_t tqConfigFlag;
FTqSerialize pSerializer;
@ -250,8 +248,8 @@ typedef struct TqMetaStore {
} STqMetaStore;
typedef struct STQ {
// the collection of group handle
// the handle of kvstore
// the collection of groups
// the handle of meta kvstore
char* path;
STqCfg* tqConfig;
STqLogReader* tqLogReader;
@ -266,23 +264,25 @@ void tqClose(STQ*);
// void* will be replace by a msg type
int tqPushMsg(STQ*, void* msg, int64_t version);
int tqCommit(STQ*);
int tqSetCursor(STQ*, void* msg);
int tqConsume(STQ*, STqConsumeReq*);
STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId);
int tqSetCursor(STQ*, STqSetCurReq* pMsg);
int tqBufferSetOffset(STqTopic*, int64_t offset);
STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqMoveOffsetToNext(STqGroupHandle*);
int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset);
int tqRegisterContext(STqGroupHandle*, void* ahandle);
int tqLaunchQuery(STqGroupHandle*);
int tqSendLaunchQuery(STqGroupHandle*);
STqTopic* tqFindTopic(STqGroup*, int64_t topicId);
int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead);
STqGroup* tqGetGroup(STQ*, int64_t clientId);
const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle);
STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId);
int tqRegisterContext(STqGroup*, void* ahandle);
int tqSendLaunchQuery(STqMsgItem*, int64_t offset);
int tqSerializeGroup(const STqGroup*, STqSerializedHead**);
const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**);
static int tqQueryExecuting(int32_t status) { return status; }
#ifdef __cplusplus
}

117
include/dnode/vnode/vnode.h
View File

@ -132,6 +132,36 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
*/
int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/**
* @brief Process a query message.
*
* @param pVnode The vnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int 0 for success, -1 for failure
*/
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/**
* @brief Process a fetch message.
*
* @param pVnode The vnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int 0 for success, -1 for failure
*/
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/**
* @brief Process a consume message.
*
* @param pVnode The vnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int 0 for success, -1 for failure
*/
int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/* ------------------------ SVnodeCfg ------------------------ */
/**
* @brief Initialize VNODE options.
@ -170,73 +200,52 @@ typedef struct {
char info[];
} SVnodeRsp;
#define VNODE_INIT_CREATE_STB_REQ(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) \
{ .ver = 0, .ctReq = META_INIT_STB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA, PTAGSCHEMA) }
static FORCE_INLINE void vnodeSetCreateStbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid,
STSchema *pSchema, STSchema *pTagSchema) {
pReq->ver = 0;
#define VNODE_INIT_CREATE_CTB_REQ(NAME, TTL, KEEP, SUID, PTAG) \
{ .ver = 0, .ctReq = META_INIT_CTB_CFG(NAME, TTL, KEEP, SUID, PTAG) }
pReq->ctReq.name = name;
pReq->ctReq.ttl = ttl;
pReq->ctReq.keep = keep;
pReq->ctReq.type = META_SUPER_TABLE;
pReq->ctReq.stbCfg.suid = suid;
pReq->ctReq.stbCfg.pSchema = pSchema;
pReq->ctReq.stbCfg.pTagSchema = pTagSchema;
}
#define VNODE_INIT_CREATE_NTB_REQ(NAME, TTL, KEEP, SUID, PSCHEMA) \
{ .ver = 0, .ctReq = META_INIT_NTB_CFG(NAME, TTL, KEEP, SUID, PSCHEMA) }
static FORCE_INLINE void vnodeSetCreateCtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep, tb_uid_t suid,
SKVRow pTag) {
pReq->ver = 0;
pReq->ctReq.name = name;
pReq->ctReq.ttl = ttl;
pReq->ctReq.keep = keep;
pReq->ctReq.type = META_CHILD_TABLE;
pReq->ctReq.ctbCfg.suid = suid;
pReq->ctReq.ctbCfg.pTag = pTag;
}
static FORCE_INLINE void vnodeSetCreateNtbReq(SVnodeReq *pReq, char *name, uint32_t ttl, uint32_t keep,
STSchema *pSchema) {
pReq->ver = 0;
pReq->ctReq.name = name;
pReq->ctReq.ttl = ttl;
pReq->ctReq.keep = keep;
pReq->ctReq.type = META_NORMAL_TABLE;
pReq->ctReq.ntbCfg.pSchema = pSchema;
}
int vnodeBuildReq(void **buf, const SVnodeReq *pReq, uint8_t type);
void *vnodeParseReq(void *buf, SVnodeReq *pReq, uint8_t type);
/* ------------------------ FOR COMPILE ------------------------ */
#if 1
#include "taosmsg.h"
#include "trpc.h"
// typedef struct {
// char db[TSDB_FULL_DB_NAME_LEN];
// int32_t cacheBlockSize; // MB
// int32_t totalBlocks;
// int32_t daysPerFile;
// int32_t daysToKeep0;
// int32_t daysToKeep1;
// int32_t daysToKeep2;
// int32_t minRowsPerFileBlock;
// int32_t maxRowsPerFileBlock;
// int8_t precision; // time resolution
// int8_t compression;
// int8_t cacheLastRow;
// int8_t update;
// int8_t quorum;
// int8_t replica;
// int8_t selfIndex;
// int8_t walLevel;
// int32_t fsyncPeriod; // millisecond
// SReplica replicas[TSDB_MAX_REPLICA];
// } SVnodeCfg;
typedef enum {
VN_MSG_TYPE_WRITE = 1,
VN_MSG_TYPE_APPLY,
VN_MSG_TYPE_SYNC,
VN_MSG_TYPE_QUERY,
VN_MSG_TYPE_FETCH
} EVnMsgType;
typedef struct {
int32_t curNum;
int32_t allocNum;
SRpcMsg rpcMsg[];
} SVnodeMsg;
int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg);
int32_t vnodeCompact(SVnode *pVnode);
int32_t vnodeSync(SVnode *pVnode);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
SVnodeMsg *vnodeInitMsg(int32_t msgNum);
int32_t vnodeAppendMsg(SVnodeMsg *pMsg, SRpcMsg *pRpcMsg);
void vnodeCleanupMsg(SVnodeMsg *pMsg);
void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType);
#endif
#ifdef __cplusplus
}
#endif

53
include/libs/index/index.h
View File

@ -23,64 +23,63 @@
extern "C" {
#endif
typedef struct SIndex SIndex;
typedef struct SIndexTerm SIndexTerm;
typedef struct SIndexOpts SIndexOpts;
typedef struct SIndex SIndex;
typedef struct SIndexTerm SIndexTerm;
typedef struct SIndexOpts SIndexOpts;
typedef struct SIndexMultiTermQuery SIndexMultiTermQuery;
typedef struct SArray SIndexMultiTerm;
typedef enum {
ADD_VALUE, // add index colume value
DEL_VALUE, // delete index column value
UPDATE_VALUE, // update index column value
ADD_INDEX, // add index on specify column
DROP_INDEX, // drop existed index
DROP_SATBLE // drop stable
typedef enum {
ADD_VALUE, // add index colume value
DEL_VALUE, // delete index column value
UPDATE_VALUE, // update index column value
ADD_INDEX, // add index on specify column
DROP_INDEX, // drop existed index
DROP_SATBLE // drop stable
} SIndexOperOnColumn;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2,QUERY_REGEX = 3} EIndexQueryType;
typedef enum { MUST = 0, SHOULD = 1, NOT = 2 } EIndexOperatorType;
typedef enum { QUERY_TERM = 0, QUERY_PREFIX = 1, QUERY_SUFFIX = 2, QUERY_REGEX = 3 } EIndexQueryType;
/*
* @param: oper
*
*/
*/
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType oper);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type);
void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery);
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType type);
/*
* @param:
* @param:
*/
int indexOpen(SIndexOpts *opt, const char *path, SIndex **index);
void indexClose(SIndex *index);
int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid);
int indexDelete(SIndex *index, SIndexMultiTermQuery *query);
int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result);
int indexRebuild(SIndex *index, SIndexOpts *opt);
int indexOpen(SIndexOpts *opt, const char *path, SIndex **index);
void indexClose(SIndex *index);
int indexPut(SIndex *index, SIndexMultiTerm *terms, uint64_t uid);
int indexDelete(SIndex *index, SIndexMultiTermQuery *query);
int indexSearch(SIndex *index, SIndexMultiTermQuery *query, SArray *result);
int indexRebuild(SIndex *index, SIndexOpts *opt);
/*
* @param
* @param
*/
SIndexMultiTerm *indexMultiTermCreate();
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term);
void indexMultiTermDestroy(SIndexMultiTerm *terms);
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term);
void indexMultiTermDestroy(SIndexMultiTerm *terms);
/*
* @param:
* @param:
*/
SIndexOpts *indexOptsCreate();
void indexOptsDestroy(SIndexOpts *opts);
void indexOptsDestroy(SIndexOpts *opts);
/*
* @param:
* @param:
*/
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType,
const char *colName, int32_t nColName, const char *colVal, int32_t nColVal);
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn operType, uint8_t colType, const char *colName,
int32_t nColName, const char *colVal, int32_t nColVal);
void indexTermDestroy(SIndexTerm *p);
#ifdef __cplusplus
}
#endif

1
include/libs/parser/parser.h
View File

@ -209,6 +209,7 @@ typedef struct SSourceParam {
SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, const char* funcName, SSourceParam* pSource, SSchema* pResSchema, int16_t interSize);
int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy);
int32_t getExprFunctionLevel(SQueryStmtInfo* pQueryInfo);
STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex);

8
include/libs/scheduler/scheduler.h
View File

@ -51,13 +51,15 @@ typedef struct SQueryProfileSummary {
uint64_t resultSize; // generated result size in Kb.
} SQueryProfileSummary;
int32_t schedulerInit(SSchedulerCfg *cfg);
/**
* Process the query job, generated according to the query physical plan.
* This is a synchronized API, and is also thread-safety.
* @param pJob
* @param qnodeList Qnode address list, element is SEpAddr
* @return
*/
int32_t scheduleQueryJob(struct SCatalog *pCatalog, void *pRpc, const SEpSet* pMgmtEps, SQueryDag* pDag, void** pJob);
int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob);
int32_t scheduleFetchRows(void *pJob, void **data);
@ -71,6 +73,8 @@ int32_t scheduleCancelJob(void *pJob);
void scheduleFreeJob(void *pJob);
void schedulerDestroy(void);
#ifdef __cplusplus
}
#endif

18
include/util/taoserror.h
View File

@ -230,6 +230,10 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5)
#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6)
// mnode-trans
#define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0)
#define TSDB_CODE_MND_TRANS_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D1)
// dnode
#define TSDB_CODE_DND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0400)
#define TSDB_CODE_DND_EXITING TAOS_DEF_ERROR_CODE(0, 0x0401)
@ -339,6 +343,20 @@ int32_t* taosGetErrno();
#define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length")
#define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type")
// tq
#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) //"Invalid configuration")
#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) //"Tq init failed")
#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) //"No diskspace for tq")
#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) //"No permission for disk files")
#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) //"Data file(s) corrupted")
#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) //"Out of memory")
#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) //"File already exists")
#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) //"Failed to create dir")
#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) //"Target key not found")
#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) //"Target key not in transaction")
#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) //"Target key duplicated in transaction")
#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) //"Group of corresponding client is not set by mnode")
// wal
#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal")
#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted")

2
include/util/tlog.h
View File

@ -42,11 +42,11 @@ extern int32_t qDebugFlag;
extern int32_t wDebugFlag;
extern int32_t sDebugFlag;
extern int32_t tsdbDebugFlag;
extern int32_t tqDebugFlag;
extern int32_t cqDebugFlag;
extern int32_t debugFlag;
extern int32_t ctgDebugFlag;
#define DEBUG_FATAL 1U
#define DEBUG_ERROR DEBUG_FATAL
#define DEBUG_WARN 2U

2
include/util/tthread.h
View File

@ -24,7 +24,7 @@ extern "C" {
#include "tdef.h"
// create new thread
pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param);
pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param);
// destory thread
bool taosDestoryThread(pthread_t* pthread);
// thread running return true

47
source/client/inc/clientInt.h
View File

@ -86,26 +86,32 @@ typedef struct STscObj {
SAppInstInfo *pAppInfo;
} STscObj;
typedef struct SClientResultInfo {
const char *pMsg;
typedef struct SReqResultInfo {
const char *pRspMsg;
const char *pData;
TAOS_FIELD *fields;
int32_t numOfCols;
int32_t numOfRows;
int32_t current;
uint32_t numOfCols;
int32_t *length;
TAOS_ROW row;
char **pCol;
} SClientResultInfo;
typedef struct SReqBody {
tsem_t rspSem; // not used now
void* fp;
void* param;
int32_t paramLen;
int64_t execId; // showId/queryId
SClientResultInfo* pResInfo;
} SRequestBody;
uint32_t numOfRows;
uint32_t current;
} SReqResultInfo;
typedef struct SReqMsg {
void *pMsg;
uint32_t len;
} SReqMsgInfo;
typedef struct SRequestSendRecvBody {
tsem_t rspSem; // not used now
void* fp;
int64_t execId; // showId/queryId
SReqMsgInfo requestMsg;
SReqResultInfo resInfo;
} SRequestSendRecvBody;
#define ERROR_MSG_BUF_DEFAULT_SIZE 512
@ -115,7 +121,7 @@ typedef struct SRequestObj {
STscObj *pTscObj;
SQueryExecMetric metric;
char *sqlstr; // sql string
SRequestBody body;
SRequestSendRecvBody body;
int64_t self;
char *msgBuf;
int32_t code;
@ -123,11 +129,10 @@ typedef struct SRequestObj {
} SRequestObj;
typedef struct SRequestMsgBody {
int32_t msgType;
void *pData;
int32_t msgLen;
uint64_t requestId;
uint64_t requestObjRefId;
int32_t msgType;
SReqMsgInfo msgInfo;
uint64_t requestId;
uint64_t requestObjRefId;
} SRequestMsgBody;
extern SAppInfo appInfo;
@ -158,7 +163,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass,
TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen);
void* doFetchRow(SRequestObj* pRequest);
void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows);
#ifdef __cplusplus
}

23
source/client/src/clientImpl.c
View File

@ -153,10 +153,9 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
void* output = NULL;
int32_t outputLen = 0;
code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE);
if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) {
if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT) {
pRequest->type = type;
pRequest->body.param = output;
pRequest->body.paramLen = outputLen;
pRequest->body.requestMsg = (SReqMsgInfo){.pMsg = output, .len = outputLen};
SRequestMsgBody body = {0};
buildRequestMsgFp[type](pRequest, &body);
@ -165,6 +164,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
sendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
tsem_wait(&pRequest->body.rspSem);
destroyRequestMsgBody(&body);
} else {
assert(0);
@ -255,7 +256,7 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con
static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT;
pMsgBody->msgLen = sizeof(SConnectMsg);
pMsgBody->msgInfo.len = sizeof(SConnectMsg);
pMsgBody->requestObjRefId = pRequest->self;
SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));
@ -279,28 +280,28 @@ static int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody)
pConnect->startTime = htobe64(appInfo.startTime);
tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app));
pMsgBody->pData = pConnect;
pMsgBody->msgInfo.pMsg = pConnect;
return 0;
}
static void destroyRequestMsgBody(SRequestMsgBody* pMsgBody) {
assert(pMsgBody != NULL);
tfree(pMsgBody->pData);
tfree(pMsgBody->msgInfo.pMsg);
}
int32_t sendMsgToServer(void *pTransporter, SEpSet* epSet, const SRequestMsgBody *pBody, int64_t* pTransporterId) {
char *pMsg = rpcMallocCont(pBody->msgLen);
char *pMsg = rpcMallocCont(pBody->msgInfo.len);
if (NULL == pMsg) {
tscError("0x%"PRIx64" msg:%s malloc failed", pBody->requestId, taosMsg[pBody->msgType]);
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return -1;
}
memcpy(pMsg, pBody->pData, pBody->msgLen);
memcpy(pMsg, pBody->msgInfo.pMsg, pBody->msgInfo.len);
SRpcMsg rpcMsg = {
.msgType = pBody->msgType,
.pCont = pMsg,
.contLen = pBody->msgLen,
.contLen = pBody->msgInfo.len,
.ahandle = (void*) pBody->requestObjRefId,
.handle = NULL,
.code = 0
@ -388,7 +389,7 @@ TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, c
void* doFetchRow(SRequestObj* pRequest) {
assert(pRequest != NULL);
SClientResultInfo* pResultInfo = pRequest->body.pResInfo;
SReqResultInfo* pResultInfo = &pRequest->body.resInfo;
if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) {
pRequest->type = TSDB_SQL_RETRIEVE_MNODE;
@ -421,7 +422,7 @@ void* doFetchRow(SRequestObj* pRequest) {
return pResultInfo->row;
}
void setResultDataPtr(SClientResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) {
assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL);
if (numOfRows == 0) {
return;

11
source/client/src/clientMain.c
View File

@ -115,12 +115,7 @@ int taos_field_count(TAOS_RES *res) {
}
SRequestObj* pRequest = (SRequestObj*) res;
SClientResultInfo* pResInfo = pRequest->body.pResInfo;
if (pResInfo == NULL) {
return 0;
}
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
return pResInfo->numOfCols;
}
@ -133,7 +128,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) {
return NULL;
}
SClientResultInfo* pResInfo = ((SRequestObj*) res)->body.pResInfo;
SReqResultInfo* pResInfo = &(((SRequestObj*) res)->body.resInfo);
return pResInfo->fields;
}
@ -248,7 +243,7 @@ int* taos_fetch_lengths(TAOS_RES *res) {
return NULL;
}
return ((SRequestObj*) res)->body.pResInfo->length;
return ((SRequestObj*) res)->body.resInfo.length;
}
const char *taos_data_type(int type) {

384
source/client/src/clientMsgHandler.c
View File

@ -13,11 +13,11 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <catalog.h>
#include <tname.h>
#include "os.h"
#include "catalog.h"
#include "tname.h"
#include "clientInt.h"
#include "clientLog.h"
#include "os.h"
#include "tmsgtype.h"
#include "trpc.h"
@ -29,16 +29,6 @@ void tscProcessActivityTimer(void *handle, void *tmrId);
static int32_t extractSTableQueryVgroupId(STableMetaInfo* pTableMetaInfo);
static int32_t minMsgSize() { return tsRpcHeadSize + 100; }
static int32_t getWaitingTimeInterval(int32_t count) {
int32_t initial = 100; // 100 ms by default
if (count <= 1) {
return 0;
}
return initial * ((2u)<<(count - 2));
}
static int32_t vgIdCompare(const void *lhs, const void *rhs) {
int32_t left = *(int32_t *)lhs;
int32_t right = *(int32_t *)rhs;
@ -298,36 +288,6 @@ void tscProcessActivityTimer(void *handle, void *tmrId) {
taosReleaseRef(tscRefId, rid);
}
int tscSendMsgToServer(SSqlObj *pSql) {
STscObj* pObj = pSql->pTscObj;
SSqlCmd* pCmd = &pSql->cmd;
char *pMsg = rpcMallocCont(pCmd->payloadLen);
if (NULL == pMsg) {
tscError("0x%"PRIx64" msg:%s malloc failed", pSql->self, taosMsg[pSql->cmd.msgType]);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
// set the mgmt ip list
if (pSql->cmd.command >= TSDB_SQL_MGMT) {
tscDumpMgmtEpSet(pSql);
}
memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen);
SRpcMsg rpcMsg = {
.msgType = pSql->cmd.msgType,
.pCont = pMsg,
.contLen = pSql->cmd.payloadLen,
.ahandle = (void*)pSql->self,
.handle = NULL,
.code = 0
};
rpcSendRequest(pObj->pRpcObj->pDnodeConn, &pSql->epSet, &rpcMsg, &pSql->rpcRid);
return TSDB_CODE_SUCCESS;
}
// handle three situation
// 1. epset retry, only return last failure ep
// 2. no epset retry, like 'taos -h invalidFqdn', return invalidFqdn
@ -354,176 +314,6 @@ void tscSetFqdnErrorMsg(SSqlObj* pSql, SRpcEpSet* pEpSet) {
}
}
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle;
SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle);
if (pSql == NULL) {
rpcFreeCont(rpcMsg->pCont);
return;
}
assert(pSql->self == handle);
STscObj *pObj = pSql->pTscObj;
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
pSql->rpcRid = -1;
if (pObj->signature != pObj) {
tscDebug("0x%"PRIx64" DB connection is closed, cmd:%d pObj:%p signature:%p", pSql->self, pCmd->command, pObj, pObj->signature);
taosRemoveRef(tscObjRef, handle);
taosReleaseRef(tscObjRef, handle);
rpcFreeCont(rpcMsg->pCont);
return;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
if (pQueryInfo != NULL && pQueryInfo->type == TSDB_QUERY_TYPE_FREE_RESOURCE) {
tscDebug("0x%"PRIx64" sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
pSql->self, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
taosRemoveRef(tscObjRef, handle);
taosReleaseRef(tscObjRef, handle);
rpcFreeCont(rpcMsg->pCont);
return;
}
if (pEpSet) {
if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) {
if (pCmd->command < TSDB_SQL_MGMT) {
tscUpdateVgroupInfo(pSql, pEpSet);
} else {
tscUpdateMgmtEpSet(pSql, pEpSet);
}
}
}
int32_t cmd = pCmd->command;
// set the flag to denote that sql string needs to be re-parsed and build submit block with table schema
if (cmd == TSDB_SQL_INSERT && rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE) {
pSql->cmd.insertParam.schemaAttached = 1;
}
// single table query error need to be handled here.
if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) &&
(((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) ||
rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) {
// 1. super table subquery
// 2. nest queries are all not updated the tablemeta and retry parse the sql after cleanup local tablemeta/vgroup id buffer
if ((TSDB_QUERY_HAS_TYPE(pQueryInfo->type, (TSDB_QUERY_TYPE_STABLE_SUBQUERY | TSDB_QUERY_TYPE_SUBQUERY |
TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) &&
!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY)) ||
(TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_NEST_SUBQUERY)) || (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->distinct)) {
// do nothing in case of super table subquery
} else {
pSql->retry += 1;
tscWarn("0x%" PRIx64 " it shall renew table meta, code:%s, retry:%d", pSql->self, tstrerror(rpcMsg->code), pSql->retry);
pSql->res.code = rpcMsg->code; // keep the previous error code
if (pSql->retry > pSql->maxRetry) {
tscError("0x%" PRIx64 " max retry %d reached, give up", pSql->self, pSql->maxRetry);
} else {
// wait for a little bit moment and then retry
// todo do not sleep in rpc callback thread, add this process into queue to process
if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
int32_t duration = getWaitingTimeInterval(pSql->retry);
taosMsleep(duration);
}
pSql->retryReason = rpcMsg->code;
rpcMsg->code = tscRenewTableMeta(pSql, 0);
// if there is an error occurring, proceed to the following error handling procedure.
if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
taosReleaseRef(tscObjRef, handle);
rpcFreeCont(rpcMsg->pCont);
return;
}
}
}
}
pRes->rspLen = 0;
if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) {
tscDebug("0x%"PRIx64" query is cancelled, code:%s", pSql->self, tstrerror(pRes->code));
} else {
pRes->code = rpcMsg->code;
}
if (pRes->code == TSDB_CODE_SUCCESS) {
tscDebug("0x%"PRIx64" reset retry counter to be 0 due to success rsp, old:%d", pSql->self, pSql->retry);
pSql->retry = 0;
}
if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) {
assert(rpcMsg->msgType == pCmd->msgType + 1);
pRes->code = rpcMsg->code;
pRes->rspType = rpcMsg->msgType;
pRes->rspLen = rpcMsg->contLen;
if (pRes->rspLen > 0 && rpcMsg->pCont) {
char *tmp = (char *)realloc(pRes->pRsp, pRes->rspLen);
if (tmp == NULL) {
pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY;
} else {
pRes->pRsp = tmp;
memcpy(pRes->pRsp, rpcMsg->pCont, pRes->rspLen);
}
} else {
tfree(pRes->pRsp);
}
/*
* There is not response callback function for submit response.
* The actual inserted number of points is the first number.
*/
if (rpcMsg->msgType == TSDB_MSG_TYPE_SUBMIT_RSP && pRes->pRsp != NULL) {
SShellSubmitRspMsg *pMsg = (SShellSubmitRspMsg*)pRes->pRsp;
pMsg->code = htonl(pMsg->code);
pMsg->numOfRows = htonl(pMsg->numOfRows);
pMsg->affectedRows = htonl(pMsg->affectedRows);
pMsg->failedRows = htonl(pMsg->failedRows);
pMsg->numOfFailedBlocks = htonl(pMsg->numOfFailedBlocks);
pRes->numOfRows += pMsg->affectedRows;
tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s inserted rows:%d rspLen:%d", pSql->self, sqlCmd[pCmd->command],
tstrerror(pRes->code), pMsg->affectedRows, pRes->rspLen);
} else {
tscDebug("0x%"PRIx64" SQL cmd:%s, code:%s rspLen:%d", pSql->self, sqlCmd[pCmd->command], tstrerror(pRes->code), pRes->rspLen);
}
}
if (pRes->code == TSDB_CODE_SUCCESS && tscProcessMsgRsp[pCmd->command]) {
rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql);
}
bool shouldFree = tscShouldBeFreed(pSql);
if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
if (rpcMsg->code != TSDB_CODE_SUCCESS) {
pRes->code = rpcMsg->code;
}
rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code;
if (rpcMsg->code == TSDB_CODE_RPC_FQDN_ERROR) {
tscAllocPayload(pCmd, TSDB_FQDN_LEN + 64);
tscSetFqdnErrorMsg(pSql, pEpSet);
}
(*pSql->fp)(pSql->param, pSql, rpcMsg->code);
}
if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it
tscDebug("0x%"PRIx64" sqlObj is automatically freed", pSql->self);
taosRemoveRef(tscObjRef, handle);
}
taosReleaseRef(tscObjRef, handle);
rpcFreeCont(rpcMsg->pCont);
}
int doBuildAndSendMsg(SSqlObj *pSql) {
SSqlCmd *pCmd = &pSql->cmd;
SSqlRes *pRes = &pSql->res;
@ -2987,51 +2777,6 @@ int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
return code;
}
static void freeElem(void* p) {
tfree(*(char**)p);
}
/**
* retrieve table meta from mnode, and then update the local table meta hashmap.
* @param pSql sql object
* @param tableIndex table index
* @return status code
*/
int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) {
SSqlCmd* pCmd = &pSql->cmd;
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
char name[TSDB_TABLE_FNAME_LEN] = {0};
int32_t code = tNameExtractFullName(&pTableMetaInfo->name, name);
if (code != TSDB_CODE_SUCCESS) {
tscError("0x%"PRIx64" failed to generate the table full name", pSql->self);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
if (pTableMeta) {
tscDebug("0x%"PRIx64" update table meta:%s, old meta numOfTags:%d, numOfCols:%d, uid:%" PRIu64, pSql->self, name,
tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid);
}
// remove stored tableMeta info in hash table
tscResetSqlCmd(pCmd, true, pSql->self);
SArray* pNameList = taosArrayInit(1, POINTER_BYTES);
SArray* vgroupList = taosArrayInit(1, POINTER_BYTES);
char* n = strdup(name);
taosArrayPush(pNameList, &n);
code = getMultiTableMetaFromMnode(pSql, pNameList, vgroupList, NULL, tscTableMetaCallBack, true);
taosArrayDestroyEx(pNameList, freeElem);
taosArrayDestroyEx(vgroupList, freeElem);
return code;
}
static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i);
@ -3044,58 +2789,11 @@ static bool allVgroupInfoRetrieved(SQueryInfo* pQueryInfo) {
return true;
}
int tscGetSTableVgroupInfo(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
int32_t code = TSDB_CODE_RPC_NETWORK_UNAVAIL;
if (allVgroupInfoRetrieved(pQueryInfo)) {
return TSDB_CODE_SUCCESS;
}
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
pNew->pTscObj = pSql->pTscObj;
pNew->signature = pNew;
pNew->cmd.command = TSDB_SQL_STABLEVGROUP;
// TODO TEST IT
SQueryInfo *pNewQueryInfo = tscGetQueryInfoS(&pNew->cmd);
if (pNewQueryInfo == NULL) {
tscFreeSqlObj(pNew);
return code;
}
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i);
STableMeta* pTableMeta = tscTableMetaDup(pMInfo->pTableMeta);
tscAddTableMetaInfo(pNewQueryInfo, &pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables);
}
if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pNew);
return code;
}
pNewQueryInfo->numOfTables = pQueryInfo->numOfTables;
registerSqlObj(pNew);
tscDebug("0x%"PRIx64" svgroupRid from %" PRId64 " to %" PRId64 , pSql->self, pSql->svgroupRid, pNew->self);
pSql->svgroupRid = pNew->self;
tscDebug("0x%"PRIx64" new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql->self, pNew, pNewQueryInfo->numOfTables);
pNew->fp = tscTableMetaCallBack;
pNew->param = (void *)pSql->self;
code = tscBuildAndSendRequest(pNew, NULL);
if (code == TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_ACTION_IN_PROGRESS;
}
return code;
}
#endif
int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_CONNECT;
pMsgBody->msgLen = sizeof(SConnectMsg);
pMsgBody->msgInfo.len = sizeof(SConnectMsg);
pMsgBody->requestObjRefId = pRequest->self;
SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));
@ -3119,7 +2817,7 @@ int32_t buildConnectMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pConnect->startTime = htobe64(appInfo.startTime);
tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app));
pMsgBody->pData = pConnect;
pMsgBody->msgInfo.pMsg = pConnect;
return 0;
}
@ -3160,26 +2858,32 @@ int processConnectRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
pTscObj->pAppInfo->clusterId = pConnect->clusterId;
atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1);
pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo));
pRequest->body.pResInfo->pMsg = pMsg;
pRequest->body.resInfo.pRspMsg = pMsg;
tscDebug("0x%" PRIx64 " clusterId:%d, totalConn:%"PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns);
return 0;
}
int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->requestObjRefId = pRequest->self;
pMsgBody->msgLen = pRequest->body.paramLen;
pMsgBody->pData = pRequest->body.param;
pMsgBody->msgInfo = pRequest->body.requestMsg;
switch(pRequest->type) {
case TSDB_SQL_CREATE_USER:
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER;
break;
case TSDB_SQL_DROP_USER:
pMsgBody->msgType = TSDB_MSG_TYPE_DROP_USER;
break;
case TSDB_SQL_CREATE_ACCT:
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_ACCT;
break;
case TSDB_SQL_DROP_ACCT:
pMsgBody->msgType = TSDB_MSG_TYPE_DROP_ACCT;
break;
case TSDB_SQL_CREATE_DB: {
pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB;
SCreateDbMsg* pCreateMsg = pRequest->body.param;
SCreateDbMsg* pCreateMsg = pRequest->body.requestMsg.pMsg;
SName name = {0};
int32_t ret = tNameSetDbName(&name, pRequest->pTscObj->acctId, pCreateMsg->db, strnlen(pCreateMsg->db, tListLen(pCreateMsg->db)));
if (ret != TSDB_CODE_SUCCESS) {
@ -3196,36 +2900,6 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
}
}
STableMeta* createTableMetaFromMsg(STableMetaMsg* pTableMetaMsg) {
assert(pTableMetaMsg != NULL && pTableMetaMsg->numOfColumns >= 2);
size_t schemaSize = (pTableMetaMsg->numOfColumns + pTableMetaMsg->numOfTags) * sizeof(SSchema);
STableMeta* pTableMeta = calloc(1, sizeof(STableMeta) + schemaSize);
pTableMeta->tableType = pTableMetaMsg->tableType;
pTableMeta->vgId = pTableMetaMsg->vgId;
pTableMeta->suid = pTableMetaMsg->suid;
pTableMeta->uid = pTableMetaMsg->tuid;
pTableMeta->tableInfo = (STableComInfo) {
.numOfTags = pTableMetaMsg->numOfTags,
.precision = pTableMetaMsg->precision,
.numOfColumns = pTableMetaMsg->numOfColumns,
};
pTableMeta->sversion = pTableMetaMsg->sversion;
pTableMeta->tversion = pTableMetaMsg->tversion;
memcpy(pTableMeta->schema, pTableMetaMsg->pSchema, schemaSize);
int32_t numOfTotalCols = pTableMeta->tableInfo.numOfColumns;
for(int32_t i = 0; i < numOfTotalCols; ++i) {
pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes;
}
return pTableMeta;
}
int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
SShowRsp* pShow = (SShowRsp *)pMsg;
pShow->showId = htonl(pShow->showId);
@ -3248,12 +2922,8 @@ int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen)
pFields[i].bytes = pSchema[i].bytes;
}
if (pRequest->body.pResInfo == NULL) {
pRequest->body.pResInfo = calloc(1, sizeof(SClientResultInfo));
}
pRequest->body.pResInfo->pMsg = pMsg;
SClientResultInfo* pResInfo = pRequest->body.pResInfo;
pRequest->body.resInfo.pRspMsg = pMsg;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
pResInfo->fields = pFields;
pResInfo->numOfCols = pMetaMsg->numOfColumns;
@ -3267,27 +2937,27 @@ int32_t processShowRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen)
int buildRetrieveMnodeMsg(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) {
pMsgBody->msgType = TSDB_MSG_TYPE_SHOW_RETRIEVE;
pMsgBody->msgLen = sizeof(SRetrieveTableMsg);
pMsgBody->msgInfo.len = sizeof(SRetrieveTableMsg);
pMsgBody->requestObjRefId = pRequest->self;
SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg));
pRetrieveMsg->showId = htonl(pRequest->body.execId);
pMsgBody->pData = pRetrieveMsg;
pMsgBody->msgInfo.pMsg = pRetrieveMsg;
return TSDB_CODE_SUCCESS;
}
int32_t processRetrieveMnodeRsp(SRequestObj *pRequest, const char* pMsg, int32_t msgLen) {
assert(msgLen >= sizeof(SRetrieveTableRsp));
tfree(pRequest->body.pResInfo->pMsg);
pRequest->body.pResInfo->pMsg = pMsg;
tfree(pRequest->body.resInfo.pRspMsg);
pRequest->body.resInfo.pRspMsg = pMsg;
SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *) pMsg;
pRetrieve->numOfRows = htonl(pRetrieve->numOfRows);
pRetrieve->precision = htons(pRetrieve->precision);
SClientResultInfo* pResInfo = pRequest->body.pResInfo;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
pResInfo->numOfRows = pRetrieve->numOfRows;
pResInfo->pData = pRetrieve->data; // todo fix this in async model
@ -3328,7 +2998,7 @@ void initMsgHandleFp() {
tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg;
tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg;
tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg;
tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg;
tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg;
tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg;
tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg;
@ -3383,6 +3053,10 @@ void initMsgHandleFp() {
handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp;
buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_DROP_USER] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_CREATE_ACCT] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_DROP_ACCT] = doBuildMsgSupp;
buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp;
handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp;

8
source/client/src/tscEnv.c
View File

@ -170,7 +170,7 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
pRequest->type = type;
pRequest->pTscObj = pObj;
pRequest->body.fp = fp;
pRequest->body.param = param;
// pRequest->body.requestMsg. = param;
pRequest->msgBuf = calloc(1, ERROR_MSG_BUF_DEFAULT_SIZE);
tsem_init(&pRequest->body.rspSem, 0, 0);
@ -188,11 +188,7 @@ static void doDestroyRequest(void* p) {
tfree(pRequest->sqlstr);
tfree(pRequest->pInfo);
if (pRequest->body.pResInfo != NULL) {
tfree(pRequest->body.pResInfo->pData);
tfree(pRequest->body.pResInfo->pMsg);
tfree(pRequest->body.pResInfo);
}
tfree(pRequest->body.resInfo.pRspMsg);
deregisterRequest(pRequest);
tfree(pRequest);

60
source/client/test/clientTests.cpp
View File

@ -40,13 +40,13 @@ TEST(testCase, driverInit_Test) {
TEST(testCase, connect_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
// assert(pConn != NULL);
taos_close(pConn);
}
TEST(testCase, create_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
// assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
@ -57,22 +57,35 @@ TEST(testCase, create_user_Test) {
taos_close(pConn);
}
//TEST(testCase, drop_user_Test) {
// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "drop user abc");
// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
// printf("failed to create user, reason:%s\n", taos_errstr(pRes));
// }
//
// taos_free_result(pRes);
// taos_close(pConn);
//}
TEST(testCase, create_account_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, drop_account_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "drop account aabc");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, show_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
// assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show users");
TAOS_ROW pRow = NULL;
@ -89,10 +102,23 @@ TEST(testCase, show_user_Test) {
taos_close(pConn);
}
TEST(testCase, show_db_Test) {
TEST(testCase, drop_user_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "drop user abc");
if (taos_errno(pRes) != TSDB_CODE_SUCCESS) {
printf("failed to create user, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
TEST(testCase, show_db_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show databases");
TAOS_ROW pRow = NULL;
@ -112,7 +138,7 @@ TEST(testCase, create_db_Test) {
TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create database abc");
TAOS_RES* pRes = taos_query(pConn, "create database abc1");
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL);

2
source/dnode/mgmt/daemon/src/daemon.c
View File

@ -31,7 +31,7 @@ static struct {
} global = {0};
void dmnSigintHandle(int signum, void *info, void *ctx) {
uError("singal:%d is received", signum);
uInfo("singal:%d is received", signum);
global.stop = true;
}

2
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -44,7 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) {
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg;
pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET_CUR] = dndProcessVnodeWriteMsg;
// msg from client to mnode
pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg;

470
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -17,11 +17,23 @@
#include "dndVnodes.h"
#include "dndTransport.h"
typedef struct {
int32_t vgId;
int32_t vgVersion;
int8_t dropped;
uint64_t dbUid;
char db[TSDB_FULL_DB_NAME_LEN];
char path[PATH_MAX + 20];
} SWrapperCfg;
typedef struct {
int32_t vgId;
int32_t refCount;
int32_t vgVersion;
int8_t dropped;
int8_t accessState;
uint64_t dbUid;
char *db;
char *path;
SVnode *pImpl;
taos_queue pWriteQ;
@ -32,13 +44,13 @@ typedef struct {
} SVnodeObj;
typedef struct {
int32_t vnodeNum;
int32_t opened;
int32_t failed;
int32_t threadIndex;
pthread_t *pThreadId;
SVnodeObj *pVnodes;
SDnode * pDnode;
int32_t vnodeNum;
int32_t opened;
int32_t failed;
int32_t threadIndex;
pthread_t thread;
SDnode *pDnode;
SWrapperCfg *pCfgs;
} SVnodeThread;
static int32_t dndInitVnodeReadWorker(SDnode *pDnode);
@ -60,8 +72,8 @@ static void dndFreeVnodeWriteQueue(SDnode *pDnode, SVnodeObj *pVnode);
static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode);
static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode);
static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg);
static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg);
static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg);
static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg);
static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs);
@ -71,18 +83,16 @@ void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pE
void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg);
static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg);
static SVnodeObj * dndAcquireVnode(SDnode *pDnode, int32_t vgId);
static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId);
static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode);
static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl);
static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode);
static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl);
static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode);
static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes);
static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes);
static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes);
static int32_t dndWriteVnodesToFile(SDnode *pDnode);
static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg);
static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode);
static int32_t dndOpenVnodes(SDnode *pDnode);
static void dndCloseVnodes(SDnode *pDnode);
@ -126,22 +136,25 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount);
}
static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) {
static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
SVnodeObj * pVnode = calloc(1, sizeof(SVnodeObj));
SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj));
if (pVnode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pVnode->vgId = vgId;
pVnode->refCount = 0;
pVnode->vgId = pCfg->vgId;
pVnode->refCount = 1;
pVnode->dropped = 0;
pVnode->accessState = TSDB_VN_ALL_ACCCESS;
pVnode->pImpl = pImpl;
pVnode->vgVersion = pCfg->vgVersion;
pVnode->dbUid = pCfg->dbUid;
pVnode->db = tstrdup(pCfg->db);
pVnode->path = tstrdup(pCfg->path);
pVnode->path = tstrdup(path);
if (pVnode->path == NULL) {
if (pVnode->path == NULL || pVnode->db == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
@ -167,7 +180,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S
}
taosWLockLatch(&pMgmt->latch);
int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
taosWUnLockLatch(&pMgmt->latch);
if (code != 0) {
@ -176,7 +189,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S
return code;
}
static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) {
static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
taosWLockLatch(&pMgmt->latch);
taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t));
@ -195,6 +208,9 @@ static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) {
dndFreeVnodeWriteQueue(pDnode, pVnode);
dndFreeVnodeApplyQueue(pDnode, pVnode);
dndFreeVnodeSyncQueue(pDnode, pVnode);
free(pVnode->path);
free(pVnode->db);
free(pVnode);
}
static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
@ -208,16 +224,16 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
SVnodeObj * pVnode = *ppVnode;
if (pVnode) {
SVnodeObj *pVnode = *ppVnode;
if (pVnode && num < size) {
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
pVnodes[num] = (*ppVnode);
num++;
if (num < size) {
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount);
pVnodes[num] = (*ppVnode);
}
pIter = taosHashIterate(pMgmt->hash, pIter);
} else {
taosHashCancelIterate(pMgmt->hash, pIter);
}
pIter = taosHashIterate(pMgmt->hash, pIter);
}
taosRUnLockLatch(&pMgmt->latch);
@ -226,15 +242,15 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) {
return pVnodes;
}
static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) {
int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR;
int32_t len = 0;
int32_t maxLen = 30000;
char * content = calloc(1, maxLen + 1);
cJSON * root = NULL;
FILE * fp = NULL;
char file[PATH_MAX + 20] = {0};
SVnodeObj *pVnodes = NULL;
static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) {
int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR;
int32_t len = 0;
int32_t maxLen = 30000;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
FILE *fp = NULL;
char file[PATH_MAX + 20] = {0};
SWrapperCfg *pCfgs = NULL;
snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
@ -270,31 +286,55 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_
goto PRASE_VNODE_OVER;
}
pVnodes = calloc(vnodesNum, sizeof(SVnodeObj));
if (pVnodes == NULL) {
pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg));
if (pCfgs == NULL) {
dError("failed to read %s since out of memory", file);
goto PRASE_VNODE_OVER;
}
for (int32_t i = 0; i < vnodesNum; ++i) {
cJSON * vnode = cJSON_GetArrayItem(vnodes, i);
SVnodeObj *pVnode = &pVnodes[i];
cJSON *vnode = cJSON_GetArrayItem(vnodes, i);
SWrapperCfg *pCfg = &pCfgs[i];
cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId");
if (!vgId || vgId->type != cJSON_String) {
if (!vgId || vgId->type != cJSON_Number) {
dError("failed to read %s since vgId not found", file);
goto PRASE_VNODE_OVER;
}
pVnode->vgId = atoi(vgId->valuestring);
pCfg->vgId = vgId->valueint;
snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId);
cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped");
if (!dropped || dropped->type != cJSON_String) {
if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", file);
goto PRASE_VNODE_OVER;
}
pVnode->dropped = atoi(vnode->valuestring);
pCfg->dropped = dropped->valueint;
cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion");
if (!vgVersion || vgVersion->type != cJSON_Number) {
dError("failed to read %s since vgVersion not found", file);
goto PRASE_VNODE_OVER;
}
pCfg->vgVersion = vgVersion->valueint;
cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid");
if (!dbUid || dbUid->type != cJSON_String) {
dError("failed to read %s since dbUid not found", file);
goto PRASE_VNODE_OVER;
}
pCfg->dbUid = atoll(dbUid->valuestring);
cJSON *db = cJSON_GetObjectItem(vnode, "db");
if (!db || db->type != cJSON_String) {
dError("failed to read %s since db not found", file);
goto PRASE_VNODE_OVER;
}
tstrncpy(pCfg->db, db->valuestring, TSDB_FULL_DB_NAME_LEN);
}
*ppCfgs = pCfgs;
*numOfVnodes = vnodesNum;
code = 0;
dInfo("succcessed to read file %s", file);
@ -313,30 +353,35 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes);
FILE *fp = fopen(file, "w");
if (fp != NULL) {
if (fp == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to write %s since %s", file, terrstr());
return -1;
}
int32_t len = 0;
int32_t maxLen = 30000;
char * content = calloc(1, maxLen + 1);
int32_t numOfVnodes = 0;
SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
int32_t len = 0;
int32_t maxLen = 65536;
char *content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n");
len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n");
for (int32_t i = 0; i < numOfVnodes; ++i) {
SVnodeObj *pVnode = pVnodes[i];
len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId);
len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped);
len += snprintf(content + len, maxLen - len, " {\n");
len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped);
len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion);
len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid);
len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db);
if (i < numOfVnodes - 1) {
len += snprintf(content + len, maxLen - len, " },{\n");
len += snprintf(content + len, maxLen - len, " },\n");
} else {
len += snprintf(content + len, maxLen - len, " }]\n");
len += snprintf(content + len, maxLen - len, " }\n");
}
}
len += snprintf(content + len, maxLen - len, " ]\n");
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
@ -358,74 +403,29 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) {
return taosRenameFile(file, realfile);
}
static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) {
char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId);
// SVnode *pImpl = vnodeCreate(vgId, path, pCfg);
SVnode *pImpl = vnodeOpen(path, NULL);
if (pImpl == NULL) {
return -1;
}
int32_t code = dndCreateVnodeWrapper(pDnode, vgId, path, pImpl);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(path);
terrno = code;
return code;
}
code = dndWriteVnodesToFile(pDnode);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(path);
terrno = code;
return code;
}
return 0;
}
static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) {
pVnode->dropped = 1;
if (dndWriteVnodesToFile(pDnode) != 0) {
pVnode->dropped = 0;
return -1;
}
dndDropVnodeWrapper(pDnode, pVnode);
vnodeClose(pVnode->pImpl);
vnodeDestroy(pVnode->path);
dndWriteVnodesToFile(pDnode);
return 0;
}
static void *dnodeOpenVnodeFunc(void *param) {
SVnodeThread *pThread = param;
SDnode * pDnode = pThread->pDnode;
SVnodesMgmt * pMgmt = &pDnode->vmgmt;
SDnode *pDnode = pThread->pDnode;
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
setThreadName("open-vnodes");
for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
SVnodeObj *pVnode = &pThread->pVnodes[v];
SWrapperCfg *pCfg = &pThread->pCfgs[v];
char stepDesc[TSDB_STEP_DESC_LEN] = {0};
snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId,
snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId,
pMgmt->openVnodes, pMgmt->totalVnodes);
dndReportStartup(pDnode, "open-vnodes", stepDesc);
char path[PATH_MAX + 20] = {0};
snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId);
SVnode *pImpl = vnodeOpen(path, NULL);
SVnode *pImpl = vnodeOpen(pCfg->path, NULL);
if (pImpl == NULL) {
dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->failed++;
} else {
dndCreateVnodeWrapper(pDnode, pVnode->vgId, path, pImpl);
dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex);
dndOpenVnode(pDnode, pCfg, pImpl);
dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->opened++;
}
@ -448,9 +448,9 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
return -1;
}
SVnodeObj *pVnodes = NULL;
int32_t numOfVnodes = 0;
if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) {
SWrapperCfg *pCfgs = NULL;
int32_t numOfVnodes = 0;
if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) {
dInfo("failed to get vnode list from disk since %s", terrstr());
return -1;
}
@ -463,13 +463,14 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread));
for (int32_t t = 0; t < threadNum; ++t) {
threads[t].threadIndex = t;
threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj));
threads[t].pDnode = pDnode;
threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg));
}
for (int32_t v = 0; v < numOfVnodes; ++v) {
int32_t t = v % threadNum;
SVnodeThread *pThread = &threads[t];
pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v];
pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v];
}
dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes);
@ -478,19 +479,25 @@ static int32_t dndOpenVnodes(SDnode *pDnode) {
SVnodeThread *pThread = &threads[t];
if (pThread->vnodeNum == 0) continue;
pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread);
if (pThread->pThreadId == NULL) {
pthread_attr_t thAttr;
pthread_attr_init(&thAttr);
pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE);
if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) {
dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno));
}
pthread_attr_destroy(&thAttr);
}
for (int32_t t = 0; t < threadNum; ++t) {
SVnodeThread *pThread = &threads[t];
taosDestoryThread(pThread->pThreadId);
pThread->pThreadId = NULL;
free(pThread->pVnodes);
if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
pthread_join(pThread->thread, NULL);
}
free(pThread->pCfgs);
}
free(threads);
free(pCfgs);
if (pMgmt->openVnodes != pMgmt->totalVnodes) {
dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes);
@ -508,7 +515,8 @@ static void dndCloseVnodes(SDnode *pDnode) {
SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes);
for (int32_t i = 0; i < numOfVnodes; ++i) {
dndDropVnodeWrapper(pDnode, pVnodes[i]);
dndReleaseVnode(pDnode, pVnodes[i]);
dndCloseVnode(pDnode, pVnodes[i]);
}
if (pVnodes != NULL) {
@ -523,11 +531,12 @@ static void dndCloseVnodes(SDnode *pDnode) {
dInfo("total vnodes:%d are all closed", numOfVnodes);
}
static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) {
static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = rpcMsg->pCont;
pCreate->vgId = htonl(pCreate->vgId);
pCreate->dnodeId = htonl(pCreate->dnodeId);
pCreate->dbUid = htobe64(pCreate->dbUid);
pCreate->vgVersion = htonl(pCreate->vgVersion);
pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize);
pCreate->totalBlocks = htonl(pCreate->totalBlocks);
pCreate->daysPerFile = htonl(pCreate->daysPerFile);
@ -544,9 +553,10 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg
pReplica->port = htons(pReplica->port);
}
*vgId = pCreate->vgId;
return pCreate;
}
#if 0
static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) {
pCfg->wsize = pCreate->cacheBlockSize;
pCfg->ssize = pCreate->cacheBlockSize;
pCfg->wsize = pCreate->cacheBlockSize;
@ -567,8 +577,15 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg
pCfg->walCfg.rollPeriod = 128;
pCfg->walCfg.segSize = 128;
pCfg->walCfg.vgId = pCreate->vgId;
#endif
return 0;
}
static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) {
memcpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
pCfg->dbUid = pCreate->dbUid;
pCfg->dropped = 0;
snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId);
pCfg->vgId = pCreate->vgId;
pCfg->vgVersion = pCreate->vgVersion;
}
static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) {
@ -584,48 +601,83 @@ static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) {
}
static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
SCreateVnodeMsg *pCreate = dndParseCreateVnodeReq(rpcMsg);
dDebug("vgId:%d, create vnode req is received", pCreate->vgId);
SVnodeCfg vnodeCfg = {0};
int32_t vgId = 0;
dndGenerateVnodeCfg(pCreate, &vnodeCfg);
dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
dDebug("vgId:%d, create vnode req is received", vgId);
SWrapperCfg wrapperCfg = {0};
dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg);
SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, already exist, return success", vgId);
dDebug("vgId:%d, already exist, return success", pCreate->vgId);
dndReleaseVnode(pDnode, pVnode);
return 0;
}
if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) {
dError("vgId:%d, failed to create vnode since %s", vgId, terrstr());
return terrno;
SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/);
if (pImpl == NULL) {
return -1;
}
int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(wrapperCfg.path);
terrno = code;
return code;
}
code = dndWriteVnodesToFile(pDnode);
if (code != 0) {
vnodeClose(pImpl);
vnodeDestroy(wrapperCfg.path);
terrno = code;
return code;
}
return 0;
}
static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
SAlterVnodeMsg *pAlter = (SAlterVnodeMsg *)dndParseCreateVnodeReq(rpcMsg);
dDebug("vgId:%d, alter vnode req is received", pAlter->vgId);
SVnodeCfg vnodeCfg = {0};
int32_t vgId = 0;
dndGenerateVnodeCfg(pAlter, &vnodeCfg);
dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg);
dDebug("vgId:%d, alter vnode req is received", vgId);
SWrapperCfg wrapperCfg = {0};
dndGenerateWrapperCfg(pDnode, pAlter, &wrapperCfg);
SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId);
if (pVnode == NULL) {
dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr());
dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr());
return terrno;
}
if (wrapperCfg.vgVersion == pVnode->vgVersion) {
dndReleaseVnode(pDnode, pVnode);
dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId);
return 0;
}
if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) {
dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr());
dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr());
dndReleaseVnode(pDnode, pVnode);
return terrno;
}
int32_t oldVersion = pVnode->vgVersion;
pVnode->vgVersion = wrapperCfg.vgVersion;
int32_t code = dndWriteVnodesToFile(pDnode);
if (code != 0) {
pVnode->vgVersion = oldVersion;
}
dndReleaseVnode(pDnode, pVnode);
return 0;
return code;
}
static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
@ -637,14 +689,20 @@ static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) {
SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
if (pVnode == NULL) {
dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
return 0;
}
pVnode->dropped = 1;
if (dndWriteVnodesToFile(pDnode) != 0) {
pVnode->dropped = 0;
return terrno;
}
if (dndDropVnode(pDnode, pVnode) != 0) {
dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr());
dndReleaseVnode(pDnode, pVnode);
return terrno;
}
dndReleaseVnode(pDnode, pVnode);
dndCloseVnode(pDnode, pVnode);
vnodeClose(pVnode->pImpl);
vnodeDestroy(pVnode->path);
dndWriteVnodesToFile(pDnode);
return 0;
}
@ -738,48 +796,76 @@ static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
break;
}
if (code != 0) {
SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle};
rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_QUERY);
static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) {
SRpcMsg *pRsp = NULL;
vnodeProcessQueryReq(pVnode->pImpl, pMsg, &pRsp);
}
static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) {
vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_FETCH);
static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) {
SRpcMsg *pRsp = NULL;
vnodeProcessFetchReq(pVnode->pImpl, pMsg, &pRsp);
}
static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs);
SRpcMsg * pRpcMsg = NULL;
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pRpcMsg);
vnodeAppendMsg(pMsg, pRpcMsg);
taosFreeQitem(pRpcMsg);
SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg);
void *ptr = taosArrayPush(pArray, &pMsg);
assert(ptr != NULL);
}
vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_WRITE);
vnodeProcessWMsgs(pVnode->pImpl, pArray);
for (size_t i = 0; i < numOfMsgs; i++) {
SRpcMsg *pRsp = NULL;
SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
int32_t code = vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp);
if (pRsp != NULL) {
rpcSendResponse(pRsp);
free(pRsp);
} else {
if (code != 0) code = terrno;
SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rpcRsp);
}
}
for (size_t i = 0; i < numOfMsgs; i++) {
SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
taosArrayDestroy(pArray);
}
static void dndProcessVnodeApplyQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
SVnodeMsg *pMsg = NULL;
SRpcMsg *pMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg);
vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_APPLY);
SRpcMsg *pRsp = NULL;
(void)vnodeApplyWMsg(pVnode->pImpl, pMsg, &pRsp);
}
}
static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) {
SVnodeMsg *pMsg = NULL;
SRpcMsg *pMsg = NULL;
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(qall, (void **)&pMsg);
vnodeProcessMsg(pVnode->pImpl, pMsg, VN_MSG_TYPE_SYNC);
SRpcMsg *pRsp = NULL;
(void)vnodeProcessSyncReq(pVnode->pImpl, pMsg, &pRsp);
}
}
@ -807,40 +893,14 @@ static int32_t dndWriteRpcMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
}
}
static int32_t dndWriteVnodeMsgToVnodeQueue(taos_queue pQueue, SRpcMsg *pRpcMsg) {
int32_t code = 0;
if (pQueue == NULL) {
code = TSDB_CODE_MSG_NOT_PROCESSED;
} else {
SVnodeMsg *pMsg = vnodeInitMsg(1);
if (pMsg == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
if (vnodeAppendMsg(pMsg, pRpcMsg) != 0) {
code = terrno;
} else {
if (taosWriteQitem(pQueue, pMsg) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
}
}
if (code != TSDB_CODE_SUCCESS) {
SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
rpcSendResponse(&rsp);
rpcFreeCont(pRpcMsg->pCont);
}
}
static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) {
SMsgHead *pHead = (SMsgHead *)pMsg->pCont;
pHead->contLen = htonl(pHead->contLen);
pHead->vgId = htonl(pHead->vgId);
SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId);
if (pVnode == NULL) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
rpcSendResponse(&rsp);
rpcFreeCont(pMsg->pCont);
}
@ -864,7 +924,7 @@ void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
if (pVnode != NULL) {
dndWriteVnodeMsgToVnodeQueue(pVnode->pSyncQ, pMsg);
dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg);
dndReleaseVnode(pDnode, pVnode);
}
}
@ -872,7 +932,7 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
if (pVnode != NULL) {
dndWriteVnodeMsgToVnodeQueue(pVnode->pQueryQ, pMsg);
dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg);
dndReleaseVnode(pDnode, pVnode);
}
}
@ -880,12 +940,12 @@ void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg);
if (pVnode != NULL) {
dndWriteVnodeMsgToVnodeQueue(pVnode->pFetchQ, pMsg);
dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg);
dndReleaseVnode(pDnode, pVnode);
}
}
static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg) {
static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) {
SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId);
if (pVnode == NULL) {
return -1;
@ -1029,7 +1089,7 @@ static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) {
}
static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) {
SVnodesMgmt * pMgmt = &pDnode->vmgmt;
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
SMWorkerPool *pPool = &pMgmt->writePool;
pPool->name = "vnode-write";
pPool->max = pDnode->opt.numOfCores;
@ -1050,7 +1110,7 @@ static void dndCleanupVnodeWriteWorker(SDnode *pDnode) {
static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue);
pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue);
if (pVnode->pSyncQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -1061,7 +1121,7 @@ static int32_t dndAllocVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
static void dndFreeVnodeSyncQueue(SDnode *pDnode, SVnodeObj *pVnode) {
SVnodesMgmt *pMgmt = &pDnode->vmgmt;
tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pSyncQ);
tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ);
pVnode->pSyncQ = NULL;
}
@ -1137,12 +1197,12 @@ void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) {
pLoads->num = taosHashGetSize(pMgmt->hash);
int32_t v = 0;
void * pIter = taosHashIterate(pMgmt->hash, NULL);
void *pIter = taosHashIterate(pMgmt->hash, NULL);
while (pIter) {
SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj * pVnode = *ppVnode;
SVnodeObj *pVnode = *ppVnode;
SVnodeLoad *pLoad = &pLoads->data[v++];
vnodeGetLoad(pVnode->pImpl, pLoad);

8
source/dnode/mgmt/impl/src/dnode.c
View File

@ -176,6 +176,12 @@ SDnode *dndInit(SDnodeOpt *pOption) {
return NULL;
}
if (vnodeInit(1) != 0) {
dError("failed to init vnode env");
dndCleanup(pDnode);
return NULL;
}
if (dndInitDnode(pDnode) != 0) {
dError("failed to init dnode");
dndCleanup(pDnode);
@ -222,8 +228,10 @@ void dndCleanup(SDnode *pDnode) {
dndCleanupMnode(pDnode);
dndCleanupVnodes(pDnode);
dndCleanupDnode(pDnode);
vnodeClear();
walCleanUp();
rpcCleanup();
dndCleanupEnv(pDnode);
free(pDnode);
dInfo("TDengine is cleaned up successfully");

8
source/dnode/mgmt/impl/test/db/db.cpp
View File

@ -210,8 +210,8 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) {
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096);
pReq->minRows = htonl(100);
pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
@ -375,8 +375,8 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) {
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096);
pReq->minRows = htonl(100);
pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;

4
source/dnode/mgmt/impl/test/stb/stb.cpp
View File

@ -187,8 +187,8 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096);
pReq->minRows = htonl(100);
pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;

2
source/dnode/mgmt/impl/test/sut/deploy.cpp
View File

@ -16,7 +16,7 @@
#include "deploy.h"
void initLog(const char* path) {
dDebugFlag = 143;
dDebugFlag = 207;
vDebugFlag = 0;
mDebugFlag = 207;
cDebugFlag = 0;

1
source/dnode/mgmt/impl/test/sut/deploy.h
View File

@ -46,3 +46,4 @@ void stopServer(SServer* pServer);
SClient* createClient(const char* user, const char* pass, const char* fqdn, uint16_t port);
void dropClient(SClient* pClient);
void sendMsg(SClient* pClient, SRpcMsg* pMsg);

143
source/dnode/mgmt/impl/test/vgroup/vgroup.cpp
View File

@ -176,49 +176,112 @@ SServer* DndTestVgroup::pServer;
SClient* DndTestVgroup::pClient;
int32_t DndTestVgroup::connId;
TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) {
{
SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg));
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(1);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRows = htonl(100);
pReq->minRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replica = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->selfIndex = 0;
for (int r = 0; r < pReq->replica; ++r) {
SReplica* pReplica = &pReq->replicas[r];
pReplica->id = htonl(1);
pReplica->port = htons(9150);
for (int i = 0; i < 3; ++i) {
SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg));
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(1);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
pReq->vgVersion = htonl(1);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRows = htonl(100);
pReq->minRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replica = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->selfIndex = 0;
for (int r = 0; r < pReq->replica; ++r) {
SReplica* pReplica = &pReq->replicas[r];
pReplica->id = htonl(1);
pReplica->port = htons(9150);
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SCreateVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
taosMsleep(1000000);
}
}
{
for (int i = 0; i < 3; ++i) {
SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(sizeof(SAlterVnodeMsg));
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(1);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
pReq->vgVersion = htonl(2);
pReq->cacheBlockSize = htonl(16);
pReq->totalBlocks = htonl(10);
pReq->daysPerFile = htonl(10);
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRows = htonl(100);
pReq->minRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;
pReq->precision = 0;
pReq->compression = 2;
pReq->replica = 1;
pReq->quorum = 1;
pReq->update = 0;
pReq->cacheLastRow = 0;
pReq->selfIndex = 0;
for (int r = 0; r < pReq->replica; ++r) {
SReplica* pReplica = &pReq->replicas[r];
pReplica->id = htonl(1);
pReplica->port = htons(9150);
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SAlterVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
}
{
for (int i = 0; i < 3; ++i) {
SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(sizeof(SDropVnodeMsg));
pReq->vgId = htonl(2);
pReq->dnodeId = htonl(1);
strcpy(pReq->db, "1.d1");
pReq->dbUid = htobe64(9527);
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = sizeof(SDropVnodeMsg);
rpcMsg.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN;
sendMsg(pClient, &rpcMsg);
SRpcMsg* pMsg = pClient->pRsp;
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
}
}
}

11
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -61,15 +61,14 @@ typedef enum {
} EAuthOp;
typedef enum {
TRN_STAGE_PREPARE = 1,
TRN_STAGE_EXECUTE = 2,
TRN_STAGE_PREPARE = 0,
TRN_STAGE_EXECUTE = 1,
TRN_STAGE_ROLLBACK = 2,
TRN_STAGE_COMMIT = 3,
TRN_STAGE_ROLLBACK = 4,
TRN_STAGE_RETRY = 5,
TRN_STAGE_OVER = 6,
TRN_STAGE_OVER = 4,
} ETrnStage;
typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy;
typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy;
typedef enum {
DND_STATUS_OFFLINE = 0,

1
source/dnode/mnode/impl/inc/mndInt.h
View File

@ -70,6 +70,7 @@ typedef struct SMnode {
tmr_h timer;
char *path;
SMnodeCfg cfg;
int64_t checkTime;
SSdb *pSdb;
SDnode *pDnode;
SArray *pSteps;

21
source/dnode/mnode/impl/inc/mndTrans.h
View File

@ -22,6 +22,16 @@
extern "C" {
#endif
typedef struct {
SEpSet epSet;
int8_t msgType;
int8_t msgSent;
int8_t msgReceived;
int32_t errCode;
int32_t contLen;
void *pCont;
} STransAction;
int32_t mndInitTrans(SMnode *pMnode);
void mndCleanupTrans(SMnode *pMnode);
@ -30,12 +40,15 @@ void mndTransDrop(STrans *pTrans);
int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw);
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction);
int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction);
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans);
void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code);
char *mndTransStageStr(ETrnStage stage);
char *mndTransPolicyStr(ETrnPolicy policy);
void mndTransHandleActionRsp(SMnodeMsg *pMsg);
char *mndTransStageStr(ETrnStage stage);
char *mndTransPolicyStr(ETrnPolicy policy);
#ifdef __cplusplus
}

3
source/dnode/mnode/impl/inc/mndVgroup.h
View File

@ -26,8 +26,9 @@ int32_t mndInitVgroup(SMnode *pMnode);
void mndCleanupVgroup(SMnode *pMnode);
SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId);
void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup);
int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups);
SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup);
SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);
SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup);

15
source/dnode/mnode/impl/src/mndAcct.c
View File

@ -17,7 +17,8 @@
#include "mndAcct.h"
#include "mndShow.h"
#define SDB_ACCT_VER 1
#define TSDB_ACCT_VER_NUMBER 1
#define TSDB_ACCT_RESERVE_SIZE 64
static int32_t mndCreateDefaultAcct(SMnode *pMnode);
static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct);
@ -70,7 +71,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) {
}
static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj));
SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj) + TSDB_ACCT_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -85,6 +86,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) {
SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxStreams)
SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage)
SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE)
SDB_SET_DATALEN(pRaw, dataPos);
return pRaw;
@ -94,7 +96,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != SDB_ACCT_VER) {
if (sver != TSDB_ACCT_VER_NUMBER) {
mError("failed to decode acct since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL;
@ -116,13 +118,13 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.maxStreams)
SDB_GET_INT64(pRaw, pRow, dataPos, &pAcct->cfg.maxStorage)
SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.accessState)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_ACCT_RESERVE_SIZE)
return pRow;
}
static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) {
mTrace("acct:%s, perform insert action", pAcct->acct);
memset(&pAcct->info, 0, sizeof(SAcctInfo));
return 0;
}
@ -134,12 +136,9 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) {
static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) {
mTrace("acct:%s, perform update action", pOldAcct->acct);
memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN);
pOldAcct->createdTime = pNewAcct->createdTime;
pOldAcct->updateTime = pNewAcct->updateTime;
pOldAcct->acctId = pNewAcct->acctId;
pOldAcct->status = pNewAcct->status;
pOldAcct->cfg = pNewAcct->cfg;
memcpy(&pOldAcct->cfg, &pNewAcct->cfg, sizeof(SAcctInfo));
return 0;
}

10
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -18,7 +18,8 @@
#include "mndShow.h"
#include "mndTrans.h"
#define SDB_CLUSTER_VER 1
#define TSDB_CLUSTER_VER_NUMBE 1
#define TSDB_CLUSTER_RESERVE_SIZE 64
static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster);
static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw);
@ -62,7 +63,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) {
}
static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, SDB_CLUSTER_VER, sizeof(SClusterObj));
SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, TSDB_CLUSTER_VER_NUMBE, sizeof(SClusterObj) + TSDB_CLUSTER_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -70,6 +71,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime)
SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime)
SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE)
return pRaw;
}
@ -78,7 +80,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != SDB_CLUSTER_VER) {
if (sver != TSDB_CLUSTER_VER_NUMBE) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to decode cluster since %s", terrstr());
return NULL;
@ -93,6 +95,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->updateTime)
SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_CLUSTER_RESERVE_SIZE)
return pRow;
}
@ -169,6 +172,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg
pShow->numOfRows = 1;
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbFname, mndShowStr(pShow->type));
return 0;
}

321
source/dnode/mnode/impl/src/mndDb.c
View File

@ -16,13 +16,12 @@
#define _DEFAULT_SOURCE
#include "mndDb.h"
#include "mndDnode.h"
#include "mndMnode.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#define TSDB_DB_VER_NUM 1
#define TSDB_DB_VER_NUMBER 1
#define TSDB_DB_RESERVE_SIZE 64
static SSdbRaw *mndDbActionEncode(SDbObj *pDb);
@ -66,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) {
void mndCleanupDb(SMnode *pMnode) {}
static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE);
SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -106,7 +105,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_DB_VER_NUM) {
if (sver != TSDB_DB_VER_NUMBER) {
mError("failed to decode db since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL;
@ -160,7 +159,7 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) {
static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
mTrace("db:%s, perform update action", pOldDb->name);
pOldDb->updateTime = pNewDb->createdTime;
pOldDb->updateTime = pNewDb->updateTime;
pOldDb->cfgVersion = pNewDb->cfgVersion;
pOldDb->vgVersion = pNewDb->vgVersion;
memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg));
@ -168,8 +167,12 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) {
}
SDbObj *mndAcquireDb(SMnode *pMnode, char *db) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_DB, db);
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
}
return pDb;
}
void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) {
@ -242,68 +245,74 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW;
}
static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING);
if (pDbRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1;
if (sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING) != 0) return -1;
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING);
if (pVgRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1;
}
return 0;
}
static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED);
if (pDbRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1;
if (sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED) != 0) return -1;
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED);
if (pVgRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED) != 0) return -1;
}
return 0;
}
static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
SSdbRaw *pDbRaw = mndDbActionEncode(pDb);
if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1;
sdbSetRawStatus(pDbRaw, SDB_STATUS_READY);
if (pDbRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1;
if (sdbSetRawStatus(pDbRaw, SDB_STATUS_READY) != 0) return -1;
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v);
if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
sdbSetRawStatus(pVgRaw, SDB_STATUS_READY);
if (pVgRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1;
if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1;
}
return 0;
}
static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SVgObj *pVgroup = pVgroups + v;
static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) {
SVgObj *pVgroup = pVgroups + vg;
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) {
return -1;
}
STransAction action = {0};
SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
SEpSet epset = mndGetDnodeEpset(pDnode);
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup);
if (pMsg == NULL) {
return -1;
}
if (pMsg == NULL) return -1;
if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) {
action.pCont = pMsg;
action.contLen = sizeof(SCreateVnodeMsg);
action.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMsg);
return -1;
}
@ -313,26 +322,26 @@ static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV
return 0;
}
static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) {
SVgObj *pVgroup = pVgroups + v;
static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) {
for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) {
SVgObj *pVgroup = pVgroups + vg;
for (int32_t vn = 0; vn < pVgroup->replica; ++vn) {
SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) {
return -1;
}
STransAction action = {0};
SVnodeGid *pVgid = pVgroup->vnodeGid + vn;
SEpSet epset = mndGetDnodeEpset(pDnode);
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) return -1;
action.epSet = mndGetDnodeEpset(pDnode);
mndReleaseDnode(pMnode, pDnode);
SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup);
if (pMsg == NULL) {
return -1;
}
if (pMsg == NULL) return -1;
if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) {
action.pCont = pMsg;
action.contLen = sizeof(SDropVnodeMsg);
action.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
free(pMsg);
return -1;
}
@ -344,14 +353,14 @@ static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV
static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) {
SDbObj dbObj = {0};
tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN);
tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN);
memcpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN);
memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN);
dbObj.createdTime = taosGetTimestampMs();
dbObj.updateTime = dbObj.createdTime;
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN);
dbObj.hashMethod = 1;
dbObj.cfgVersion = 1;
dbObj.vgVersion = 1;
dbObj.hashMethod = 1;
dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups,
.cacheBlockSize = pCreate->cacheBlockSize,
.totalBlocks = pCreate->totalBlocks,
@ -359,8 +368,8 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
.daysToKeep0 = pCreate->daysToKeep0,
.daysToKeep1 = pCreate->daysToKeep1,
.daysToKeep2 = pCreate->daysToKeep2,
.minRows = pCreate->minRowsPerFileBlock,
.maxRows = pCreate->maxRowsPerFileBlock,
.minRows = pCreate->minRows,
.maxRows = pCreate->maxRows,
.fsyncPeriod = pCreate->fsyncPeriod,
.commitTime = pCreate->commitTime,
.precision = pCreate->precision,
@ -396,29 +405,30 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat
mError("db:%s, failed to create since %s", pCreate->db, terrstr());
goto CREATE_DB_OVER;
}
mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db);
if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_DB_OVER;
}
@ -447,8 +457,8 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0);
pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1);
pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2);
pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock);
pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock);
pCreate->minRows = htonl(pCreate->minRows);
pCreate->maxRows = htonl(pCreate->maxRows);
pCreate->commitTime = htonl(pCreate->commitTime);
pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod);
@ -456,7 +466,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db);
if (pDb != NULL) {
sdbRelease(pMnode->pSdb, pDb);
mndReleaseDb(pMnode, pDb);
if (pCreate->ignoreExist) {
mDebug("db:%s, already exist, ignore exist is set", pCreate->db);
return 0;
@ -485,57 +495,77 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) {
}
static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) {
bool changed = false;
terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) {
pDb->cfg.totalBlocks = pAlter->totalBlocks;
changed = true;
terrno = 0;
}
if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) {
pDb->cfg.daysToKeep0 = pAlter->daysToKeep0;
changed = true;
terrno = 0;
}
if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) {
pDb->cfg.daysToKeep1 = pAlter->daysToKeep1;
changed = true;
terrno = 0;
}
if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) {
pDb->cfg.daysToKeep2 = pAlter->daysToKeep2;
changed = true;
terrno = 0;
}
if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) {
pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod;
changed = true;
terrno = 0;
}
if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) {
pDb->cfg.walLevel = pAlter->walLevel;
changed = true;
terrno = 0;
}
if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) {
pDb->cfg.quorum = pAlter->quorum;
changed = true;
terrno = 0;
}
if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) {
pDb->cfg.cacheLastRow = pAlter->cacheLastRow;
changed = true;
terrno = 0;
}
if (!changed) {
terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
return -1;
}
return terrno;
}
static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) {
SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetUpdateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) {
SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; }
static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; }
static int32_t mndSetUpdateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; }
static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("db:%s, failed to update since %s", pOldDb->name, terrstr());
@ -544,30 +574,41 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO
mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name);
SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY);
SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetUpdateDbUndoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
}
if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
}
if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
}
if (mndSetUpdateDbUndoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto UPDATE_DB_OVER;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
goto UPDATE_DB_OVER;
}
code = 0;
UPDATE_DB_OVER:
mndTransDrop(pTrans);
return 0;
return code;
}
static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
@ -610,46 +651,82 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
}
static int32_t mndSetDropDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdbRaw *pUndoRaw = mndDbActionEncode(pDb);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdbRaw *pCommitRaw = mndDbActionEncode(pDb);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; }
static int32_t mndSetDropDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; }
static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("db:%s, failed to drop since %s", pDb->name, terrstr());
return -1;
}
mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name);
SSdbRaw *pRedoRaw = mndDbActionEncode(pDb);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto DROP_DB_OVER;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndDbActionEncode(pDb);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetDropDbUndoLogs(pMnode, pTrans, pDb) != 0) {
mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
goto DROP_DB_OVER;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndDbActionEncode(pDb);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto DROP_DB_OVER;
}
if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto DROP_DB_OVER;
}
if (mndSetDropDbUndoActions(pMnode, pTrans, pDb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto DROP_DB_OVER;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
goto DROP_DB_OVER;
}
code = 0;
DROP_DB_OVER:
mndTransDrop(pTrans);
return 0;
return code;
}
static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) {
@ -751,29 +828,27 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SSyncDbMsg *pSync = pMsg->rpcMsg.pCont;
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
if (pDb == NULL) {
mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr());
return -1;
} else {
mndReleaseDb(pMnode, pDb);
return 0;
}
mndReleaseDb(pMnode, pDb);
return 0;
}
static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont;
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
if (pDb == NULL) {
mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr());
return -1;
} else {
mndReleaseDb(pMnode, pDb);
return 0;
}
mndReleaseDb(pMnode, pDb);
return 0;
}
static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {

32
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -18,10 +18,10 @@
#include "mndMnode.h"
#include "mndShow.h"
#include "mndTrans.h"
#include "ttime.h"
#include "tep.h"
#include "ttime.h"
#define TSDB_DNODE_VER 1
#define TSDB_DNODE_VER_NUMBER 1
#define TSDB_DNODE_RESERVE_SIZE 64
#define TSDB_CONFIG_OPTION_LEN 16
#define TSDB_CONIIG_VALUE_LEN 48
@ -101,14 +101,14 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj);
if (pRaw == NULL) return -1;
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
if (sdbSetRawStatus(pRaw, SDB_STATUS_READY) != 0) return -1;
mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id);
return sdbWrite(pMnode->pSdb, pRaw);
}
static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE);
SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER_NUMBER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -127,7 +127,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_DNODE_VER) {
if (sver != TSDB_DNODE_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to decode dnode since %s", terrstr());
return NULL;
@ -150,21 +150,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) {
static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) {
mTrace("dnode:%d, perform insert action", pDnode->id);
pDnode->rebootTime = 0;
pDnode->lastAccessTime = 0;
pDnode->accessTimes = 0;
pDnode->numOfMnodes = 0;
pDnode->numOfVnodes = 0;
pDnode->numOfQnodes = 0;
pDnode->numOfSupportMnodes = 0;
pDnode->numOfSupportVnodes = 0;
pDnode->numOfSupportQnodes = 0;
pDnode->numOfCores = 0;
pDnode->status = DND_STATUS_OFFLINE;
pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED;
snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port);
return 0;
}
@ -225,7 +212,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) {
bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) {
int64_t ms = taosGetTimestampMs();
int64_t interval = ABS(pDnode->lastAccessTime - ms);
if (interval > 3000 * pMnode->cfg.statusInterval) {
if (interval > 3500 * pMnode->cfg.statusInterval) {
return false;
}
return true;
@ -267,12 +254,9 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) {
return DND_REASON_STATUS_INTERVAL_NOT_MATCH;
}
int64_t checkTime = 0;
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) {
if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) {
mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone,
pCfg->checkTime, checkTime);
pCfg->checkTime, pMnode->checkTime);
return DND_REASON_TIME_ZONE_NOT_MATCH;
}

17
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -19,7 +19,8 @@
#include "mndShow.h"
#include "mndTrans.h"
#define SDB_MNODE_VER 1
#define TSDB_MNODE_VER_NUMBER 1
#define TSDB_MNODE_RESERVE_SIZE 64
static int32_t mndCreateDefaultMnode(SMnode *pMnode);
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj);
@ -60,8 +61,12 @@ int32_t mndInitMnode(SMnode *pMnode) {
void mndCleanupMnode(SMnode *pMnode) {}
static SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_MNODE, &mnodeId);
SSdb *pSdb = pMnode->pSdb;
SMnodeObj *pObj = sdbAcquire(pSdb, SDB_MNODE, &mnodeId);
if (pObj == NULL) {
terrno = TSDB_CODE_MND_MNODE_NOT_EXIST;
}
return pObj;
}
static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pMnodeObj) {
@ -97,13 +102,14 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
}
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, SDB_MNODE_VER, sizeof(SMnodeObj));
SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, TSDB_MNODE_VER_NUMBER, sizeof(SMnodeObj) + TSDB_MNODE_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, pMnodeObj->id);
SDB_SET_INT64(pRaw, dataPos, pMnodeObj->createdTime)
SDB_SET_INT64(pRaw, dataPos, pMnodeObj->updateTime)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE)
return pRaw;
}
@ -112,7 +118,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != SDB_MNODE_VER) {
if (sver != TSDB_MNODE_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to decode mnode since %s", terrstr());
return NULL;
@ -126,6 +132,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, pRow, dataPos, &pMnodeObj->id)
SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->updateTime)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_MNODE_RESERVE_SIZE)
return pRow;
}

296
source/dnode/mnode/impl/src/mndStb.c
View File

@ -21,9 +21,10 @@
#include "mndShow.h"
#include "mndTrans.h"
#include "mndUser.h"
#include "mndVgroup.h"
#include "tname.h"
#define TSDB_STB_VER_NUM 1
#define TSDB_STB_VER_NUMBER 1
#define TSDB_STB_RESERVE_SIZE 64
static SSdbRaw *mndStbActionEncode(SStbObj *pStb);
@ -70,7 +71,7 @@ void mndCleanupStb(SMnode *pMnode) {}
static SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUM, size);
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, TSDB_STB_VER_NUMBER, size);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -103,7 +104,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_STB_VER_NUM) {
if (sver != TSDB_STB_VER_NUMBER) {
mError("failed to decode stable since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL;
@ -176,8 +177,12 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOldStb, SStbObj *pNewStb
}
SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_STB, stbName);
SSdb *pSdb = pMnode->pSdb;
SStbObj *pStb = sdbAcquire(pSdb, SDB_STB, stbName);
if (pStb == NULL) {
terrno = TSDB_CODE_MND_STB_NOT_EXIST;
}
return pStb;
}
void mndReleaseStb(SMnode *pMnode, SStbObj *pStb) {
@ -195,7 +200,54 @@ static SDbObj *mndAcquireDbByStb(SMnode *pMnode, char *stbName) {
return mndAcquireDb(pMnode, db);
}
static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) {
static SCreateStbInternalMsg *mndBuildCreateStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) {
int32_t totalCols = pStb->numOfTags + pStb->numOfColumns;
int32_t contLen = totalCols * sizeof(SSchema) + sizeof(SCreateStbInternalMsg);
SCreateStbInternalMsg *pCreate = calloc(1, contLen);
if (pCreate == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pCreate->head.contLen = htonl(contLen);
pCreate->head.vgId = htonl(pVgroup->vgId);
memcpy(pCreate->name, pStb->name, TSDB_TABLE_FNAME_LEN);
pCreate->suid = htobe64(pStb->uid);
pCreate->sverson = htonl(pStb->version);
pCreate->ttl = 0;
pCreate->keep = 0;
pCreate->numOfTags = htonl(pStb->numOfTags);
pCreate->numOfColumns = htonl(pStb->numOfColumns);
memcpy(pCreate->pSchema, pStb->pSchema, totalCols * sizeof(SSchema));
for (int32_t t = 0; t < totalCols; ++t) {
SSchema *pSchema = &pCreate->pSchema[t];
pSchema->bytes = htonl(pSchema->bytes);
pSchema->colId = htonl(pSchema->colId);
}
return pCreate;
}
static SDropStbInternalMsg *mndBuildDropStbMsg(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb) {
int32_t contLen = sizeof(SDropStbInternalMsg);
SDropStbInternalMsg *pDrop = calloc(1, contLen);
if (pDrop == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pDrop->head.contLen = htonl(contLen);
pDrop->head.vgId = htonl(pVgroup->vgId);
memcpy(pDrop->name, pStb->name, TSDB_TABLE_FNAME_LEN);
pDrop->suid = htobe64(pStb->uid);
return pDrop;
}
static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) {
pCreate->numOfColumns = htonl(pCreate->numOfColumns);
pCreate->numOfTags = htonl(pCreate->numOfTags);
int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags;
@ -244,6 +296,103 @@ static int32_t mndCheckStbMsg(SCreateStbMsg *pCreate) {
return 0;
}
static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
}
static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pDb->uid) continue;
SCreateStbInternalMsg *pMsg = mndBuildCreateStbMsg(pMnode, pVgroup, pStb);
if (pMsg == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = pMsg;
action.contLen = htonl(pMsg->head.contLen);
action.msgType = TSDB_MSG_TYPE_CREATE_STB_IN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMsg);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pDb->uid) continue;
SDropStbInternalMsg *pMsg = mndBuildDropStbMsg(pMnode, pVgroup, pStb);
if (pMsg == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
action.pCont = pMsg;
action.contLen = sizeof(SDropStbInternalMsg);
action.msgType = TSDB_MSG_TYPE_DROP_STB_IN;
if (mndTransAppendUndoAction(pTrans, &action) != 0) {
free(pMsg);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pVgroup);
return -1;
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) {
SStbObj stbObj = {0};
tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
@ -265,6 +414,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre
}
memcpy(stbObj.pSchema, pCreate->pSchema, totalSize);
int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
@ -272,29 +422,30 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre
}
mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
SSdbRaw *pRedoRaw = mndStbActionEncode(&stbObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetCreateStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto CREATE_STB_OVER;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING);
SSdbRaw *pUndoRaw = mndStbActionEncode(&stbObj);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetCreateStbUndoLogs(pMnode, pTrans, pDb, &stbObj) != 0) {
mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
goto CREATE_STB_OVER;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetCreateStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto CREATE_STB_OVER;
}
if (mndSetCreateStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_STB_OVER;
}
if (mndSetCreateStbUndoActions(pMnode, pTrans, pDb, &stbObj) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_STB_OVER;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
@ -302,8 +453,11 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre
return -1;
}
code = 0;
CREATE_STB_OVER:
mndTransDrop(pTrans);
return 0;
return code;
}
static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) {
@ -312,7 +466,7 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) {
mDebug("stb:%s, start to create", pCreate->name);
if (mndCheckStbMsg(pCreate) != 0) {
if (mndCheckCreateStbMsg(pCreate) != 0) {
mError("stb:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
@ -349,7 +503,10 @@ static int32_t mndProcessCreateStbMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCreateStbInRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndCheckAlterStbMsg(SAlterStbMsg *pAlter) {
SSchema *pSchema = &pAlter->schema;
@ -410,9 +567,44 @@ static int32_t mndProcessAlterStbMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessAlterStbInRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
}
static int32_t mndSetDropStbUndoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; }
static int32_t mndSetDropStbUndoActions(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { return 0; }
static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("stb:%s, failed to drop since %s", pStb->name, terrstr());
@ -420,36 +612,39 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pMsg, SStbObj *pStb) {
}
mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name);
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto DROP_STB_OVER;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetDropStbUndoLogs(pMnode, pTrans, pStb) != 0) {
mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr());
goto DROP_STB_OVER;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto DROP_STB_OVER;
}
if (mndSetDropStbRedoActions(pMnode, pTrans, pStb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto DROP_STB_OVER;
}
if (mndSetDropStbUndoActions(pMnode, pTrans, pStb) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto DROP_STB_OVER;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
goto DROP_STB_OVER;
}
code = 0;
DROP_STB_OVER:
mndTransDrop(pTrans);
return 0;
}
@ -484,7 +679,10 @@ static int32_t mndProcessDropStbMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropStbInRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;

217
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -17,17 +17,10 @@
#include "mndTrans.h"
#include "mndSync.h"
#define TSDB_TRANS_VER 1
#define TSDB_TRANS_VER_NUMBER 1
#define TSDB_TRN_ARRAY_SIZE 8
#define TSDB_TRN_RESERVE_SIZE 64
typedef struct {
SEpSet epSet;
int8_t msgType;
int32_t contLen;
void *pCont;
} STransAction;
static SSdbRaw *mndTransActionEncode(STrans *pTrans);
static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw);
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans);
@ -37,11 +30,11 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans);
static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle);
static void mndTransSendRpcRsp(STrans *pTrans, int32_t code);
static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw);
static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont);
static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction);
static void mndTransDropLogs(SArray *pArray);
static void mndTransDropActions(SArray *pArray);
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray);
static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray);
static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray);
static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans);
static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans);
@ -100,7 +93,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
rawDataLen += (sizeof(STransAction) + pAction->contLen);
}
SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen);
SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER_NUMBER, rawDataLen);
if (pRaw == NULL) {
mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr());
return NULL;
@ -167,7 +160,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) {
return NULL;
}
if (sver != TSDB_TRANS_VER) {
if (sver != TSDB_TRANS_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr());
return NULL;
@ -324,8 +317,12 @@ static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewT
}
STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_TRANS, &transId);
SSdb *pSdb = pMnode->pSdb;
STrans *pTrans = sdbAcquire(pSdb, SDB_TRANS, &transId);
if (pTrans == NULL) {
terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
}
return pTrans;
}
void mndReleaseTrans(SMnode *pMnode, STrans *pTrans) {
@ -343,10 +340,8 @@ char *mndTransStageStr(ETrnStage stage) {
return "commit";
case TRN_STAGE_ROLLBACK:
return "rollback";
case TRN_STAGE_RETRY:
return "retry";
case TRN_STAGE_OVER:
return "stop";
return "over";
default:
return "undefined";
}
@ -388,7 +383,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) {
return NULL;
}
mDebug("trans:%d, data:%p is created", pTrans->id, pTrans);
mDebug("trans:%d, is created", pTrans->id);
return pTrans;
}
@ -417,7 +412,7 @@ void mndTransDrop(STrans *pTrans) {
mndTransDropActions(pTrans->redoActions);
mndTransDropActions(pTrans->undoActions);
mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans);
// mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
tfree(pTrans);
}
@ -459,10 +454,8 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) {
return code;
}
static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont};
void *ptr = taosArrayPush(pArray, &action);
static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) {
void *ptr = taosArrayPush(pArray, pAction);
if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -471,15 +464,15 @@ static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgTy
return 0;
}
int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont);
mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen);
int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) {
int32_t code = mndTransAppendAction(pTrans->redoActions, pAction);
mTrace("trans:%d, msg:%s append to redo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code);
return code;
}
int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) {
int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont);
mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen);
int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) {
int32_t code = mndTransAppendAction(pTrans->undoActions, pAction);
mTrace("trans:%d, msg:%s append to undo actions, code:0x%x", pTrans->id, taosMsg[pAction->msgType], code);
return code;
}
@ -493,7 +486,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
}
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mTrace("trans:%d, start sync", pTrans->id);
mTrace("trans:%d, sync to other nodes", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
@ -515,7 +508,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
return -1;
}
mDebug("trans:%d, prepare finished", pNewTrans->id);
pNewTrans->rpcHandle = pTrans->rpcHandle;
mndTransExecute(pMnode, pNewTrans);
mndReleaseTrans(pMnode, pNewTrans);
@ -533,7 +525,7 @@ int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) {
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
if (taosArrayGetSize(pTrans->commitLogs) != 0) {
mTrace("trans:%d, start sync", pTrans->id);
mTrace("trans:%d, sync to other nodes", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
@ -563,7 +555,7 @@ int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) {
}
sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED);
mTrace("trans:%d, start sync", pTrans->id);
mTrace("trans:%d, sync to other nodes", pTrans->id);
int32_t code = mndSyncPropose(pMnode, pRaw);
if (code != 0) {
mError("trans:%d, failed to sync since %s", pTrans->id, terrstr());
@ -596,6 +588,50 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code)
// todo
}
void mndTransHandleActionRsp(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
int64_t sig = (int64_t)(pMsg->rpcMsg.ahandle);
int32_t transId = (int32_t)(sig >> 32);
int32_t action = (int32_t)((sig << 32) >> 32);
STrans *pTrans = mndAcquireTrans(pMnode, transId);
if (pTrans == NULL) {
mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr());
goto HANDLE_ACTION_RSP_OVER;
}
SArray *pArray = NULL;
if (pTrans->stage == TRN_STAGE_EXECUTE) {
pArray = pTrans->redoActions;
} else if (pTrans->stage == TRN_STAGE_ROLLBACK) {
pArray = pTrans->undoActions;
} else {
}
if (pArray == NULL) {
mError("trans:%d, invalid trans stage:%s", transId, mndTransStageStr(pTrans->stage));
goto HANDLE_ACTION_RSP_OVER;
}
int32_t actionNum = taosArrayGetSize(pTrans->redoActions);
if (action < 0 || action > actionNum) {
mError("trans:%d, invalid action:%d", transId, action);
goto HANDLE_ACTION_RSP_OVER;
}
STransAction *pAction = taosArrayGet(pArray, action);
if (pAction != NULL) {
pAction->msgReceived = 1;
pAction->errCode = pMsg->code;
}
mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->code);
mndTransExecute(pMnode, pTrans);
HANDLE_ACTION_RSP_OVER:
mndReleaseTrans(pMnode, pTrans);
}
static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) {
SSdb *pSdb = pMnode->pSdb;
int32_t arraySize = taosArrayGetSize(pArray);
@ -618,7 +654,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) {
if (code != 0) {
mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr())
} else {
mTrace("trans:%d, execute redo logs finished", pTrans->id)
mDebug("trans:%d, execute redo logs finished", pTrans->id)
}
}
@ -632,7 +668,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) {
if (code != 0) {
mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr())
} else {
mTrace("trans:%d, execute undo logs finished", pTrans->id)
mDebug("trans:%d, execute undo logs finished", pTrans->id)
}
}
@ -646,47 +682,70 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) {
if (code != 0) {
mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr())
} else {
mTrace("trans:%d, execute commit logs finished", pTrans->id)
mDebug("trans:%d, execute commit logs finished", pTrans->id)
}
}
return code;
}
static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) {
#if 0
int32_t arraySize = taosArrayGetSize(pArray);
for (int32_t i = 0; i < arraySize; ++i) {
STransAction *pAction = taosArrayGet(pArray, i);
static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) {
int32_t numOfActions = taosArrayGetSize(pArray);
if (numOfActions == 0) return 0;
SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen};
for (int32_t action = 0; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pArray, action);
if (pAction == NULL) continue;
if (pAction->msgSent) continue;
int64_t signature = pTrans->id;
signature = (signature << 32);
signature += action;
SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .ahandle = (void *)signature};
rpcMsg.pCont = rpcMallocCont(pAction->contLen);
if (rpcMsg.pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen);
pAction->msgSent = 1;
pAction->msgReceived = 0;
pAction->errCode = 0;
mDebug("trans:%d, action:%d is sent", pTrans->id, action);
mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg);
}
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
#else
return 0;
#endif
int32_t numOfReceivedMsgs = 0;
int32_t errorCode = 0;
for (int32_t action = 0; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pArray, action);
if (pAction == NULL) continue;
if (pAction->msgSent && pAction->msgReceived) {
numOfReceivedMsgs++;
if (pAction->errCode != 0) {
errorCode = pAction->errCode;
}
}
}
if (numOfReceivedMsgs == numOfActions) {
mDebug("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errorCode);
terrno = errorCode;
return errorCode;
} else {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
}
static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0;
mTrace("trans:%d, start to execute redo actions", pTrans->id);
return mndTransExecuteActions(pMnode, pTrans->redoActions);
return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions);
}
static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) {
if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0;
mTrace("trans:%d, start to execute undo actions", pTrans->id);
return mndTransExecuteActions(pMnode, pTrans->undoActions);
return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions);
}
static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {
@ -694,7 +753,7 @@ static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) {
if (code == 0) {
pTrans->stage = TRN_STAGE_EXECUTE;
mTrace("trans:%d, stage from prepare to execute", pTrans->id);
mDebug("trans:%d, stage from prepare to execute", pTrans->id);
} else {
pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr());
@ -708,17 +767,17 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) {
if (code == 0) {
pTrans->stage = TRN_STAGE_COMMIT;
mTrace("trans:%d, stage from execute to commit", pTrans->id);
mDebug("trans:%d, stage from execute to commit", pTrans->id);
} else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mTrace("trans:%d, stage keep on execute since %s", pTrans->id, terrstr(code));
mDebug("trans:%d, stage keep on execute since %s", pTrans->id, tstrerror(code));
return code;
} else {
if (pTrans->policy == TRN_POLICY_ROLLBACK) {
pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr());
} else {
pTrans->stage = TRN_STAGE_RETRY;
mError("trans:%d, stage from execute to retry since %s", pTrans->id, terrstr());
pTrans->stage = TRN_STAGE_EXECUTE;
mError("trans:%d, stage keep on execute since %s", pTrans->id, terrstr());
}
}
@ -726,29 +785,16 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) {
}
static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans);
if (code == 0) {
pTrans->stage = TRN_STAGE_OVER;
mTrace("trans:%d, commit stage finished", pTrans->id);
} else {
if (pTrans->policy == TRN_POLICY_ROLLBACK) {
pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage from commit to rollback since %s", pTrans->id, terrstr());
} else {
pTrans->stage = TRN_STAGE_RETRY;
mError("trans:%d, stage from commit to retry since %s", pTrans->id, terrstr());
}
}
return code;
mndTransExecuteCommitLogs(pMnode, pTrans);
pTrans->stage = TRN_STAGE_OVER;
return 0;
}
static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteUndoActions(pMnode, pTrans);
if (code == 0) {
mTrace("trans:%d, rollbacked", pTrans->id);
mDebug("trans:%d, rollbacked", pTrans->id);
} else {
pTrans->stage = TRN_STAGE_ROLLBACK;
mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr());
@ -757,20 +803,6 @@ static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) {
return code;
}
static int32_t mndTransPerformRetryStage(SMnode *pMnode, STrans *pTrans) {
int32_t code = mndTransExecuteRedoActions(pMnode, pTrans);
if (code == 0) {
pTrans->stage = TRN_STAGE_COMMIT;
mTrace("trans:%d, stage from retry to commit", pTrans->id);
} else {
pTrans->stage = TRN_STAGE_RETRY;
mError("trans:%d, stage keep on retry since %s", pTrans->id, terrstr());
}
return code;
}
static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
@ -785,7 +817,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
case TRN_STAGE_COMMIT:
code = mndTransCommit(pMnode, pTrans);
if (code == 0) {
code = mndTransPerformCommitStage(pMnode, pTrans);
mndTransPerformCommitStage(pMnode, pTrans);
}
break;
case TRN_STAGE_ROLLBACK:
@ -794,9 +826,6 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) {
code = mndTransRollback(pMnode, pTrans);
}
break;
case TRN_STAGE_RETRY:
code = mndTransPerformRetryStage(pMnode, pTrans);
break;
default:
mndTransSendRpcRsp(pTrans, 0);
return;

14
source/dnode/mnode/impl/src/mndUser.c
View File

@ -20,7 +20,7 @@
#include "mndTrans.h"
#include "tkey.h"
#define TSDB_USER_VER 1
#define TSDB_USER_VER_NUMBER 1
#define TSDB_USER_RESERVE_SIZE 64
static int32_t mndCreateDefaultUsers(SMnode *pMnode);
@ -94,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) {
}
static SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE);
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -114,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_USER_VER) {
if (sver != TSDB_USER_VER_NUMBER) {
mError("failed to decode user since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL;
@ -175,8 +175,12 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOldUser, SUserObj *pNe
}
SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_USER, userName);
SSdb *pSdb = pMnode->pSdb;
SUserObj *pUser = sdbAcquire(pSdb, SDB_USER, userName);
if (pUser == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_EXIST;
}
return pUser;
}
void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) {

82
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -21,7 +21,7 @@
#include "mndShow.h"
#include "mndTrans.h"
#define TSDB_VGROUP_VER_NUM 1
#define TSDB_VGROUP_VER_NUMBER 1
#define TSDB_VGROUP_RESERVE_SIZE 64
static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw);
@ -70,7 +70,7 @@ int32_t mndInitVgroup(SMnode *pMnode) {
void mndCleanupVgroup(SMnode *pMnode) {}
SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE);
SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
@ -98,7 +98,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL;
if (sver != TSDB_VGROUP_VER_NUM) {
if (sver != TSDB_VGROUP_VER_NUMBER) {
mError("failed to decode vgroup since %s", terrstr());
terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
return NULL;
@ -142,14 +142,20 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNe
mTrace("vgId:%d, perform update action", pOldVgroup->vgId);
pOldVgroup->updateTime = pNewVgroup->updateTime;
pOldVgroup->version = pNewVgroup->version;
pOldVgroup->hashBegin = pNewVgroup->hashBegin;
pOldVgroup->hashEnd = pNewVgroup->hashEnd;
pOldVgroup->replica = pNewVgroup->replica;
memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
return 0;
}
SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) {
SSdb *pSdb = pMnode->pSdb;
return sdbAcquire(pSdb, SDB_VGROUP, &vgId);
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId);
if (pVgroup == NULL) {
terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
}
return pVgroup;
}
void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
@ -158,16 +164,17 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) {
}
SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg));
SCreateVnodeMsg *pCreate = calloc(1, sizeof(SCreateVnodeMsg));
if (pCreate == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pCreate->dnodeId = htonl(pDnode->id);
pCreate->vgId = htonl(pVgroup->vgId);
pCreate->dnodeId = htonl(pDnode->id);
memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
pCreate->dbUid = htobe64(pDb->uid);
pCreate->vgVersion = htonl(pVgroup->version);
pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks);
pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile);
@ -193,7 +200,6 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb
SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pVgidDnode == NULL) {
free(pCreate);
terrno = TSDB_CODE_MND_APP_ERROR;
return NULL;
}
@ -217,7 +223,7 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb
}
SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) {
SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg));
SDropVnodeMsg *pDrop = calloc(1, sizeof(SDropVnodeMsg));
if (pDrop == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
@ -269,7 +275,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
return -1;
}
int32_t alloceVgroups = 0;
int32_t allocedVgroups = 0;
int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP);
uint32_t hashMin = 0;
uint32_t hashMax = UINT32_MAX;
@ -281,7 +287,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
pVgroup->createdTime = taosGetTimestampMs();
pVgroup->updateTime = pVgroups->createdTime;
pVgroup->version = 1;
pVgroup->dbUid = pDb->uid;
pVgroup->hashBegin = hashMin + hashInterval * v;
if (v == pDb->cfg.numOfVgroups - 1) {
pVgroup->hashEnd = hashMax;
@ -290,6 +295,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
}
memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN);
pVgroup->dbUid = pDb->uid;
pVgroup->replica = pDb->cfg.replications;
if (mndGetAvailableDnode(pMnode, pVgroup) != 0) {
@ -298,22 +304,46 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
return -1;
}
alloceVgroups++;
allocedVgroups++;
}
*ppVgroups = pVgroups;
return 0;
}
static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; }
SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup) {
SEpSet epset = {0};
for (int32_t v = 0; v < pVgroup->replica; ++v) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode == NULL) continue;
if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
epset.inUse = epset.numOfEps;
}
epset.port[epset.numOfEps] = pDnode->port;
memcpy(&epset.fqdn[epset.numOfEps], pDnode->fqdn, TSDB_FQDN_LEN);
epset.numOfEps++;
mndReleaseDnode(pMnode, pDnode);
}
return epset;
}
static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) {
SSdb *pSdb = pMnode->pSdb;
SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
@ -329,7 +359,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (strcmp(pVgroup->dbName, dbName) == 0) {
if (pVgroup->dbUid == pDb->uid) {
replica = MAX(replica, pVgroup->replica);
numOfVgroups++;
}
@ -441,11 +471,25 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) {
}
static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) {
if (dnodeId == 0) {
return 0;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfVnodes = 0;
void *pIter = NULL;
while (1) {
SVgObj *pVgroup = NULL;
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
for (int32_t v = 0; v < pVgroup->replica; ++v) {
if (pVgroup->vnodeGid[v].dnodeId == dnodeId) {
numOfVnodes++;
}
}
sdbRelease(pSdb, pVgroup);
}
return 0;
return numOfVnodes;
}
static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {

3
source/dnode/mnode/impl/src/mnode.c
View File

@ -237,6 +237,9 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) {
return NULL;
}
char timestr[24] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep));
if (pMnode->pSteps == NULL) {
free(pMnode);

2
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -240,6 +240,8 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
}
void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
*ppObj = NULL;
SHashObj *hash = sdbGetHash(pSdb, type);
if (hash == NULL) return NULL;

4
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -27,12 +27,12 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) {
pRaw->sver = sver;
pRaw->dataLen = dataLen;
mTrace("raw:%p, is created, len:%d", pRaw, dataLen);
// mTrace("raw:%p, is created, len:%d", pRaw, dataLen);
return pRaw;
}
void sdbFreeRaw(SSdbRaw *pRaw) {
mTrace("raw:%p, is freed", pRaw);
// mTrace("raw:%p, is freed", pRaw);
free(pRaw);
}

2
source/dnode/vnode/impl/CMakeLists.txt
View File

@ -19,5 +19,5 @@ target_link_libraries(
# test
if(${BUILD_TEST})
#add_subdirectory(test)
add_subdirectory(test)
endif(${BUILD_TEST})

1
source/dnode/vnode/impl/inc/vnodeDef.h
View File

@ -71,6 +71,7 @@ struct SVnode {
SWal* pWal;
SVnodeSync* pSync;
SVnodeFS* pFs;
tsem_t canCommit;
};
int vnodeScheduleTask(SVnodeTask* task);

3
source/dnode/vnode/impl/src/vnodeCommit.c
View File

@ -38,9 +38,10 @@ int vnodeCommit(void *arg) {
metaCommit(pVnode->pMeta);
tqCommit(pVnode->pTq);
tsdbCommit(pVnode->pTq);
tsdbCommit(pVnode->pTsdb);
vnodeBufPoolRecycle(pVnode);
tsem_post(&(pVnode->canCommit));
// TODO
return 0;
}

50
source/dnode/vnode/impl/src/vnodeInt.c
View File

@ -15,53 +15,31 @@
#define _DEFAULT_SOURCE
#include "vnodeInt.h"
#include "tqueue.h"
int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; }
SVnode *vnodeCreate(int32_t vgId, const char *path, const SVnodeCfg *pCfg) { return NULL; }
void vnodeDrop(SVnode *pVnode) {}
int32_t vnodeCompact(SVnode *pVnode) { return 0; }
int32_t vnodeSync(SVnode *pVnode) { return 0; }
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { return 0; }
SVnodeMsg *vnodeInitMsg(int32_t msgNum) {
SVnodeMsg *pMsg = taosAllocateQitem(msgNum * sizeof(SRpcMsg *) + sizeof(SVnodeMsg));
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
} else {
pMsg->allocNum = msgNum;
return pMsg;
}
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("query message is processed");
return 0;
}
int32_t vnodeAppendMsg(SVnodeMsg *pMsg, SRpcMsg *pRpcMsg) {
if (pMsg->curNum >= pMsg->allocNum) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pMsg->rpcMsg[pMsg->curNum++] = *pRpcMsg;
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("fetch message is processed");
return 0;
}
void vnodeCleanupMsg(SVnodeMsg *pMsg) {
for (int32_t i = 0; i < pMsg->curNum; ++i) {
rpcFreeCont(pMsg->rpcMsg[i].pCont);
}
taosFreeQitem(pMsg);
int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("sync message is processed");
return 0;
}
void vnodeProcessMsg(SVnode *pVnode, SVnodeMsg *pMsg, EVnMsgType msgType) {
switch (msgType) {
case VN_MSG_TYPE_WRITE:
break;
case VN_MSG_TYPE_APPLY:
break;
case VN_MSG_TYPE_SYNC:
break;
case VN_MSG_TYPE_QUERY:
break;
case VN_MSG_TYPE_FETCH:
break;
}
int vnodeProcessConsumeReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("consume message is processed");
return 0;
}

3
source/dnode/vnode/impl/src/vnodeMain.c
View File

@ -74,11 +74,14 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) {
pVnode->path = strdup(path);
vnodeOptionsCopy(&(pVnode->config), pVnodeCfg);
tsem_init(&(pVnode->canCommit), 0, 1);
return pVnode;
}
static void vnodeFree(SVnode *pVnode) {
if (pVnode) {
tsem_destroy(&(pVnode->canCommit));
tfree(pVnode->path);
free(pVnode);
}

119
source/dnode/vnode/impl/src/vnodeWrite.c
View File

@ -16,29 +16,25 @@
#include "vnodeDef.h"
int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) {
SVnodeReq *pVnodeReq;
switch (pMsg->msgType) {
case TSDB_MSG_TYPE_MQ_SET:
case TSDB_MSG_TYPE_MQ_SET_CUR:
if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) {
// TODO: handle error
}
break;
}
void *pBuf = pMsg->pCont;
return 0;
}
int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
SRpcMsg *pMsg;
SRpcMsg * pMsg;
SVnodeReq *pVnodeReq;
for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
// ser request version
void *pBuf = pMsg->pCont;
void * pBuf = pMsg->pCont;
int64_t ver = pVnode->state.processed++;
taosEncodeFixedU64(&pBuf, ver);
@ -49,68 +45,63 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
walFsync(pVnode->pWal, false);
// Apply each request now
for (int i = 0; i < taosArrayGetSize(pMsgs); i++) {
pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i);
SVnodeReq vReq;
// Apply the request
{
void *ptr = vnodeMalloc(pVnode, pMsg->contLen);
if (ptr == NULL) {
// TODO: handle error
}
// TODO: copy here need to be extended
memcpy(ptr, pMsg->pCont, pMsg->contLen);
// todo: change the interface here
uint64_t ver;
taosDecodeFixedU64(pMsg->pCont, &ver);
if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) {
// TODO: handle error
}
vnodeParseReq(pMsg->pCont, &vReq, pMsg->msgType);
switch (pMsg->msgType) {
case TSDB_MSG_TYPE_CREATE_TABLE:
if (metaCreateTable(pVnode->pMeta, &(vReq.ctReq)) < 0) {
// TODO: handle error
}
// TODO: maybe need to clear the requst struct
break;
case TSDB_MSG_TYPE_DROP_TABLE:
if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) {
// TODO: handle error
}
break;
case TSDB_MSG_TYPE_SUBMIT:
if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) {
// TODO: handle error
}
break;
default:
break;
}
pVnode->state.applied = ver;
}
// Check if it needs to commit
if (vnodeShouldCommit(pVnode)) {
if (vnodeAsyncCommit(pVnode) < 0) {
// TODO: handle error
}
}
}
// TODO: Integrate RAFT module here
return 0;
}
int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
// TODO
SVnodeReq vReq;
void * ptr = vnodeMalloc(pVnode, pMsg->contLen);
if (ptr == NULL) {
// TODO: handle error
}
// TODO: copy here need to be extended
memcpy(ptr, pMsg->pCont, pMsg->contLen);
// todo: change the interface here
uint64_t ver;
taosDecodeFixedU64(pMsg->pCont, &ver);
if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) {
// TODO: handle error
}
vnodeParseReq(pMsg->pCont, &vReq, pMsg->msgType);
switch (pMsg->msgType) {
case TSDB_MSG_TYPE_CREATE_STB_IN:
case TSDB_MSG_TYPE_CREATE_TABLE:
if (metaCreateTable(pVnode->pMeta, &(vReq.ctReq)) < 0) {
// TODO: handle error
}
// TODO: maybe need to clear the requst struct
break;
case TSDB_MSG_TYPE_DROP_STB_IN:
case TSDB_MSG_TYPE_DROP_TABLE:
if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) {
// TODO: handle error
}
break;
case TSDB_MSG_TYPE_SUBMIT:
if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr) < 0) {
// TODO: handle error
}
break;
default:
break;
}
pVnode->state.applied = ver;
// Check if it needs to commit
if (vnodeShouldCommit(pVnode)) {
tsem_wait(&(pVnode->canCommit));
if (vnodeAsyncCommit(pVnode) < 0) {
// TODO: handle error
}
}
return 0;
}

36
source/dnode/vnode/impl/test/vnodeApiTests.cpp
View File

@ -81,7 +81,8 @@ static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) {
pSchema = vtCreateBasicSchema();
pTagSchema = vtCreateBasicTagSchema();
SVnodeReq vCreateSTbReq = VNODE_INIT_CREATE_STB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema);
SVnodeReq vCreateSTbReq;
vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema);
zs = vnodeBuildReq(NULL, &vCreateSTbReq, TSDB_MSG_TYPE_CREATE_TABLE);
pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs);
@ -104,7 +105,8 @@ static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) {
int tz;
SKVRow pTag = vtCreateBasicTag();
SVnodeReq vCreateCTbReq = VNODE_INIT_CREATE_CTB_REQ(tbname, UINT32_MAX, UINT32_MAX, suid, pTag);
SVnodeReq vCreateCTbReq;
vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag);
tz = vnodeBuildReq(NULL, &vCreateCTbReq, TSDB_MSG_TYPE_CREATE_TABLE);
pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz);
@ -166,6 +168,21 @@ static void vtClearMsgBatch(SArray *pMsgArr) {
taosArrayClear(pMsgArr);
}
static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) {
int rcode;
SRpcMsg *pReq;
SRpcMsg *pRsp;
rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
GTEST_ASSERT_EQ(rcode, 0);
for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) {
pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i);
rcode = vnodeApplyWMsg(pVnode, pReq, NULL);
GTEST_ASSERT_EQ(rcode, 0);
}
}
TEST(vnodeApiTest, vnode_simple_create_table_test) {
tb_uid_t suid = 1638166374163;
SRpcMsg *pMsg;
@ -189,8 +206,7 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) {
sprintf(tbname, "st");
vtBuildCreateStbReq(suid, tbname, &pMsg);
taosArrayPush(pMsgArr, &pMsg);
rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
ASSERT_EQ(rcode, 0);
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
// CREATE A LOT OF CHILD TABLES
@ -203,8 +219,7 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) {
}
// Process request batch
rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
ASSERT_EQ(rcode, 0);
vtProcessAndApplyReqs(pVnode, pMsgArr);
// Clear request batch
vtClearMsgBatch(pMsgArr);
@ -242,16 +257,14 @@ TEST(vnodeApiTest, vnode_simple_insert_test) {
sprintf(tbname, "st");
vtBuildCreateStbReq(suid, tbname, &pMsg);
taosArrayPush(pMsgArr, &pMsg);
rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
GTEST_ASSERT_EQ(rcode, 0);
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
// 2. CREATE A CHILD TABLE
sprintf(tbname, "t0");
vtBuildCreateCtbReq(suid, tbname, &pMsg);
taosArrayPush(pMsgArr, &pMsg);
rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
GTEST_ASSERT_EQ(rcode, 0);
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
// 3. WRITE A LOT OF TIME-SERIES DATA
@ -260,8 +273,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) {
vtBuildSubmitReq(&pMsg);
taosArrayPush(pMsgArr, &pMsg);
}
rcode = vnodeProcessWMsgs(pVnode, pMsgArr);
GTEST_ASSERT_EQ(rcode, 0);
vtProcessAndApplyReqs(pVnode, pMsgArr);
vtClearMsgBatch(pMsgArr);
}

1
source/dnode/vnode/tq/CMakeLists.txt
View File

@ -11,6 +11,7 @@ target_link_libraries(
PUBLIC wal
PUBLIC os
PUBLIC util
PUBLIC common
)
if(${BUILD_TEST})

11
source/dnode/vnode/tq/inc/tqInt.h
View File

@ -17,11 +17,20 @@
#define _TD_TQ_INT_H_
#include "tq.h"
#include "tlog.h"
#ifdef __cplusplus
extern "C" {
#endif
extern int32_t tqDebugFlag;
#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }}
#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }}
#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }}
#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }}
#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }}
#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }}
// create persistent storage for meta info such as consuming offset
// return value > 0: cgId
// return value <= 0: error code

2
source/dnode/vnode/tq/inc/tqMetaStore.h
View File

@ -17,7 +17,7 @@
#define _TQ_META_STORE_H_
#include "os.h"
#include "tq.h"
#include "tqInt.h"
#ifdef __cplusplus
extern "C" {

331
source/dnode/vnode/tq/src/tq.c
View File

@ -24,89 +24,82 @@
// handle management message
//
int tqGetgHandleSSize(const STqGroupHandle* gHandle);
int tqBufHandleSSize();
int tqBufItemSSize();
int tqGroupSSize(const STqGroup* pGroup);
int tqTopicSSize();
int tqItemSSize();
STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
STqGroupHandle* gHandle;
return NULL;
}
void* tqSerializeListHandle(STqList* listHandle, void* ptr);
void* tqSerializeTopic(STqTopic* pTopic, void* ptr);
void* tqSerializeItem(STqMsgItem* pItem, void* ptr);
void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr);
void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr);
void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr);
const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle);
const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem);
const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic);
const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem);
STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) {
STQ* pTq = malloc(sizeof(STQ));
if (pTq == NULL) {
// TODO: memory error
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL;
}
pTq->path = strdup(path);
pTq->tqConfig = tqConfig;
pTq->tqLogReader = tqLogReader;
#if 0
pTq->tqMemRef.pAlloctorFactory = allocFac;
// pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
if (pTq->tqMemRef.pAllocator == NULL) {
// TODO
// TODO: error code of buffer pool
}
pTq->tqMeta =
tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0);
#endif
pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0);
if (pTq->tqMeta == NULL) {
// TODO: free STQ
return NULL;
}
return pTq;
}
void tqClose(STQ*pTq) {
void tqClose(STQ* pTq) {
// TODO
}
static int tqProtoCheck(TmqMsgHead *pMsg) {
return pMsg->protoVer == 0;
}
static int tqProtoCheck(STqMsgHead* pMsg) { return pMsg->protoVer == 0; }
static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) {
static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) {
// clean old item and move forward
int32_t consumeOffset = pAck->consumeOffset;
int idx = consumeOffset % TQ_BUFFER_SIZE;
ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor);
tfree(bHandle->buffer[idx].content);
ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor);
tfree(pTopic->buffer[idx].content);
if (1 /* TODO: need to launch new query */) {
STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg));
if (pNewQuery == NULL) {
// TODO: memory insufficient
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
// TODO: lock executor
pNewQuery->exec->executor = bHandle->buffer[idx].executor;
// TODO: read from wal and assign to src
pNewQuery->exec->src = 0;
pNewQuery->exec->dest = &bHandle->buffer[idx];
pNewQuery->next = *ppQuery;
*ppQuery = pNewQuery;
/*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/
/*pNewQuery->exec->src = 0;*/
/*pNewQuery->exec->dest = &pTopic->buffer[idx];*/
/*pNewQuery->next = *ppQuery;*/
/**ppQuery = pNewQuery;*/
}
return 0;
}
static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) {
static int tqAck(STqGroup* pGroup, STqAcks* pAcks) {
int32_t ackNum = pAcks->ackNum;
TmqOneAck* acks = pAcks->acks;
STqOneAck* acks = pAcks->acks;
// double ptr for acks and list
int i = 0;
STqListHandle* node = gHandle->head;
int ackCnt = 0;
STqQueryMsg* pQuery = NULL;
int i = 0;
STqList* node = pGroup->head;
int ackCnt = 0;
STqQueryMsg* pQuery = NULL;
while (i < ackNum && node->next) {
if (acks[i].topicId == node->next->bufHandle.topicId) {
if (acks[i].topicId == node->next->topic.topicId) {
ackCnt++;
tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery);
} else if (acks[i].topicId < node->next->bufHandle.topicId) {
tqAckOneTopic(&node->next->topic, &acks[i], &pQuery);
} else if (acks[i].topicId < node->next->topic.topicId) {
i++;
} else {
node = node->next;
@ -118,52 +111,56 @@ static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) {
return ackCnt;
}
static int tqCommitTCGroup(STqGroupHandle* handle) {
static int tqCommitGroup(STqGroup* pGroup) {
// persist modification into disk
return 0;
}
int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) {
int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) {
// create in disk
STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle));
if (gHandle == NULL) {
STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup));
if (pGroup == NULL) {
// TODO
return -1;
}
memset(gHandle, 0, sizeof(STqGroupHandle));
*ppGroup = pGroup;
memset(pGroup, 0, sizeof(STqGroup));
return 0;
}
STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId);
if (gHandle == NULL) {
int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle);
if (code != 0) {
STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId);
if (pGroup == NULL) {
int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup);
if (code < 0) {
// TODO
return NULL;
}
tqHandleMovePut(pTq->tqMeta, cId, pGroup);
}
ASSERT(pGroup);
// create
// open
return gHandle;
return pGroup;
}
int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; }
int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
// TODO
return 0;
}
int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) {
// delete from disk
return 0;
}
static int tqFetch(STqGroupHandle* gHandle, void** msg) {
STqListHandle* head = gHandle->head;
STqListHandle* node = head;
int totSize = 0;
static int tqFetch(STqGroup* pGroup, void** msg) {
STqList* head = pGroup->head;
STqList* node = head;
int totSize = 0;
// TODO: make it a macro
int sizeLimit = 4 * 1024;
TmqMsgContent* buffer = malloc(sizeLimit);
STqMsgContent* buffer = malloc(sizeLimit);
if (buffer == NULL) {
// TODO:memory insufficient
return -1;
@ -172,25 +169,25 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) {
// until all topic iterated or msgs over sizeLimit
while (node->next) {
node = node->next;
STqBufferHandle* bufHandle = &node->bufHandle;
int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) {
totSize += bufHandle->buffer[idx].size;
STqTopic* topicHandle = &node->topic;
int idx = topicHandle->nextConsumeOffset % TQ_BUFFER_SIZE;
if (topicHandle->buffer[idx].content != NULL && topicHandle->buffer[idx].offset == topicHandle->nextConsumeOffset) {
totSize += topicHandle->buffer[idx].size;
if (totSize > sizeLimit) {
void* ptr = realloc(buffer, totSize);
if (ptr == NULL) {
totSize -= bufHandle->buffer[idx].size;
totSize -= topicHandle->buffer[idx].size;
// TODO:memory insufficient
// return msgs already copied
break;
}
}
*((int64_t*)buffer) = bufHandle->topicId;
*((int64_t*)buffer) = topicHandle->topicId;
buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
*((int64_t*)buffer) = bufHandle->buffer[idx].size;
*((int64_t*)buffer) = topicHandle->buffer[idx].size;
buffer = POINTER_SHIFT(buffer, sizeof(int64_t));
memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size);
buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size);
memcpy(buffer, topicHandle->buffer[idx].content, topicHandle->buffer[idx].size);
buffer = POINTER_SHIFT(buffer, topicHandle->buffer[idx].size);
if (totSize > sizeLimit) {
break;
}
@ -199,11 +196,19 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) {
return totSize;
}
STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; }
STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); }
int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; }
int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; }
int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) {
if (tqQueryExecuting(bufItem->status)) {
return 0;
}
bufItem->status = 1;
// load data from wal or buffer pool
// put into exec
// send exec into non blocking queue
// when query finished, put into buffer pool
return 0;
}
/*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/
/*return 0;*/
@ -220,23 +225,96 @@ int tqCommit(STQ* pTq) {
return 0;
}
int tqSetCursor(STQ* pTq, void* msg) {
int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) {
int code;
memset(pTopic->buffer, 0, sizeof(pTopic->buffer));
// launch query
for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) {
int pos = i % TQ_BUFFER_SIZE;
code = tqSendLaunchQuery(&pTopic->buffer[pos], offset);
if (code < 0) {
// TODO: error handling
}
}
// set offset
pTopic->nextConsumeOffset = offset;
pTopic->floatingCursor = offset;
return 0;
}
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
if (!tqProtoCheck((TmqMsgHead*)pMsg)) {
// proto version invalid
return -1;
}
int64_t clientId = pMsg->head.clientId;
STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId);
STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) {
// TODO
return NULL;
}
int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) {
int code;
int64_t clientId = pMsg->head.clientId;
int64_t topicId = pMsg->topicId;
int64_t offset = pMsg->offset;
STqGroup* gHandle = tqGetGroup(pTq, clientId);
if (gHandle == NULL) {
// client not connect
return -1;
}
STqTopic* topicHandle = tqFindTopic(gHandle, topicId);
if (topicHandle == NULL) {
return -1;
}
if (pMsg->offset == topicHandle->nextConsumeOffset) {
return 0;
}
// TODO: check log last version
code = tqBufferSetOffset(topicHandle, offset);
if (code < 0) {
// set error code
return -1;
}
return 0;
}
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
int64_t clientId = pMsg->head.clientId;
STqGroup* pGroup = tqGetGroup(pTq, clientId);
if (pGroup == NULL) {
terrno = TSDB_CODE_TQ_GROUP_NOT_SET;
return -1;
}
STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg;
int numOfMsgs = tqFetch(pGroup, (void**)&pRsp->msgs);
if (numOfMsgs < 0) {
return -1;
}
if (numOfMsgs == 0) {
// most recent data has been fetched
// enable timer for blocking wait
// once new data written during wait time
// launch query and response
}
// fetched a num of msgs, rpc response
return 0;
}
#if 0
int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
if (!tqProtoCheck((STqMsgHead*)pMsg)) {
// proto version invalid
return -1;
}
int64_t clientId = pMsg->head.clientId;
STqGroup* pGroup = tqGetGroup(pTq, clientId);
if (pGroup == NULL) {
// client not connect
return -1;
}
if (pMsg->acks.ackNum != 0) {
if (tqAck(gHandle, &pMsg->acks) != 0) {
if (tqAck(pGroup, &pMsg->acks) != 0) {
// ack not success
return -1;
}
@ -244,22 +322,23 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) {
STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg;
if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) {
if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) {
// fetch error
return -1;
}
// judge and launch new query
if (tqLaunchQuery(gHandle)) {
// launch query error
return -1;
}
/*if (tqSendLaunchQuery(gHandle)) {*/
// launch query error
/*return -1;*/
/*}*/
return 0;
}
#endif
int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) {
int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) {
// calculate size
int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead);
int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead);
if (sz > (*ppHead)->ssize) {
void* tmpPtr = realloc(*ppHead, sz);
if (tmpPtr == NULL) {
@ -272,53 +351,53 @@ int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** pp
}
void* ptr = (*ppHead)->content;
// do serialization
*(int64_t*)ptr = gHandle->cId;
*(int64_t*)ptr = pGroup->clientId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = gHandle->cgId;
*(int64_t*)ptr = pGroup->cgId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int32_t*)ptr = gHandle->topicNum;
*(int32_t*)ptr = pGroup->topicNum;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
if (gHandle->topicNum > 0) {
tqSerializeListHandle(gHandle->head, ptr);
if (pGroup->topicNum > 0) {
tqSerializeListHandle(pGroup->head, ptr);
}
return 0;
}
void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) {
STqListHandle* node = listHandle;
void* tqSerializeListHandle(STqList* listHandle, void* ptr) {
STqList* node = listHandle;
ASSERT(node != NULL);
while (node) {
ptr = tqSerializeBufHandle(&node->bufHandle, ptr);
ptr = tqSerializeTopic(&node->topic, ptr);
node = node->next;
}
return ptr;
}
void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) {
*(int64_t*)ptr = bufHandle->nextConsumeOffset;
void* tqSerializeTopic(STqTopic* pTopic, void* ptr) {
*(int64_t*)ptr = pTopic->nextConsumeOffset;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int64_t*)ptr = bufHandle->topicId;
*(int64_t*)ptr = pTopic->topicId;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
*(int32_t*)ptr = bufHandle->head;
*(int32_t*)ptr = pTopic->head;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
*(int32_t*)ptr = bufHandle->tail;
*(int32_t*)ptr = pTopic->tail;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr);
ptr = tqSerializeItem(&pTopic->buffer[i], ptr);
}
return ptr;
}
void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) {
void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) {
// TODO: do we need serialize this?
// mainly for executor
return ptr;
}
const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) {
STqGroupHandle* gHandle = *ppGHandle;
const void* ptr = pHead->content;
gHandle->cId = *(int64_t*)ptr;
const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) {
STqGroup* gHandle = *ppGroup;
const void* ptr = pHead->content;
gHandle->clientId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
gHandle->cgId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
@ -326,63 +405,63 @@ const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHan
gHandle->topicNum = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
gHandle->head = NULL;
STqListHandle* node = gHandle->head;
STqList* node = gHandle->head;
for (int i = 0; i < gHandle->topicNum; i++) {
if (gHandle->head == NULL) {
if ((node = malloc(sizeof(STqListHandle))) == NULL) {
if ((node = malloc(sizeof(STqList))) == NULL) {
// TODO: error
return NULL;
}
node->next = NULL;
ptr = tqDeserializeBufHandle(ptr, &node->bufHandle);
ptr = tqDeserializeTopic(ptr, &node->topic);
gHandle->head = node;
} else {
node->next = malloc(sizeof(STqListHandle));
node->next = malloc(sizeof(STqList));
if (node->next == NULL) {
// TODO: error
return NULL;
}
node->next->next = NULL;
ptr = tqDeserializeBufHandle(ptr, &node->next->bufHandle);
ptr = tqDeserializeTopic(ptr, &node->next->topic);
node = node->next;
}
}
return ptr;
}
const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) {
const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) {
const void* ptr = pBytes;
bufHandle->nextConsumeOffset = *(int64_t*)ptr;
topic->nextConsumeOffset = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
bufHandle->topicId = *(int64_t*)ptr;
topic->topicId = *(int64_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int64_t));
bufHandle->head = *(int32_t*)ptr;
topic->head = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
bufHandle->tail = *(int32_t*)ptr;
topic->tail = *(int32_t*)ptr;
ptr = POINTER_SHIFT(ptr, sizeof(int32_t));
for (int i = 0; i < TQ_BUFFER_SIZE; i++) {
ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]);
ptr = tqDeserializeItem(ptr, &topic->buffer[i]);
}
return ptr;
}
const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; }
const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; }
// TODO: make this a macro
int tqGetgHandleSSize(const STqGroupHandle* gHandle) {
int tqGroupSSize(const STqGroup* gHandle) {
return sizeof(int64_t) * 2 // cId + cgId
+ sizeof(int32_t) // topicNum
+ gHandle->topicNum * tqBufHandleSSize();
+ gHandle->topicNum * tqTopicSSize();
}
// TODO: make this a macro
int tqBufHandleSSize() {
int tqTopicSSize() {
return sizeof(int64_t) * 2 // nextConsumeOffset + topicId
+ sizeof(int32_t) * 2 // head + tail
+ TQ_BUFFER_SIZE * tqBufItemSSize();
+ TQ_BUFFER_SIZE * tqItemSSize();
}
int tqBufItemSSize() {
int tqItemSSize() {
// TODO: do this need serialization?
// mainly for executor
return 0;

58
source/dnode/vnode/tq/src/tqMetaStore.c
View File

@ -56,6 +56,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) {
int offset = tqSeekLastPage(fd);
int nBytes;
if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
}
if (nBytes == 0) {
@ -71,7 +72,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
int32_t tqConfigFlag) {
STqMetaStore* pMeta = malloc(sizeof(STqMetaStore));
if (pMeta == NULL) {
// close
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL;
}
memset(pMeta, 0, sizeof(STqMetaStore));
@ -79,8 +80,9 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
// concat data file name and index file name
size_t pathLen = strlen(path);
pMeta->dirPath = malloc(pathLen + 1);
if (pMeta->dirPath != NULL) {
// TODO: memory insufficient
if (pMeta->dirPath == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL;
}
strcpy(pMeta->dirPath, path);
@ -88,13 +90,14 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
strcpy(name, path);
if (taosDirExist(name) != 0 && taosMkDir(name) != 0) {
ASSERT(false);
terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR;
tqError("failed to create dir:%s since %s ", name, terrstr());
}
strcat(name, "/" TQ_IDX_NAME);
int idxFd = open(name, O_RDWR | O_CREAT, 0755);
if (idxFd < 0) {
ASSERT(false);
// close file
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s ", name, terrstr());
// free memory
return NULL;
}
@ -102,9 +105,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
pMeta->idxFd = idxFd;
pMeta->unpersistHead = malloc(sizeof(STqMetaList));
if (pMeta->unpersistHead == NULL) {
ASSERT(false);
// close file
// free memory
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return NULL;
}
memset(pMeta->unpersistHead, 0, sizeof(STqMetaList));
@ -114,7 +115,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
strcat(name, "/" TQ_META_NAME);
int fileFd = open(name, O_RDWR | O_CREAT, 0755);
if (fileFd < 0) {
ASSERT(false);
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to open file:%s since %s", name, terrstr());
return NULL;
}
@ -129,7 +131,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
STqIdxPageBuf idxBuf;
STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE);
if (serializedObj == NULL) {
// TODO:memory insufficient
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
}
int idxRead;
int allocated = TQ_PAGE_SIZE;
@ -137,14 +139,16 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) {
if (idxRead == -1) {
// TODO: handle error
ASSERT(false);
terrno = TAOS_SYSTEM_ERROR(errno);
tqError("failed to read tq index file since %s", terrstr());
}
ASSERT(idxBuf.head.writeOffset == idxRead);
// loop read every entry
for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) {
STqMetaList* pNode = malloc(sizeof(STqMetaList));
if (pNode == NULL) {
// TODO: free memory and return error
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
// TODO: free memory
}
memset(pNode, 0, sizeof(STqMetaList));
memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE);
@ -153,7 +157,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial
if (allocated < pNode->handle.serializedSize) {
void* ptr = realloc(serializedObj, pNode->handle.serializedSize);
if (ptr == NULL) {
// TODO: memory insufficient
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
// TODO: free memory
}
serializedObj = ptr;
allocated = pNode->handle.serializedSize;
@ -292,7 +297,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) {
STqMetaList* pNode = pHead->unpersistNext;
STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead));
if (pSHead == NULL) {
// TODO: memory error
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
pSHead->ver = TQ_SVER;
@ -403,7 +408,6 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
// TODO: think about thread safety
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInUse);
}
@ -416,7 +420,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu
}
STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if (pNewNode == NULL) {
// TODO: memory error
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
memset(pNewNode, 0, sizeof(STqMetaList));
@ -470,10 +474,10 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
STqMetaList* pNode = pMeta->bucket[bucketKey];
while (pNode) {
if (pNode->handle.key == key) {
// TODO: think about thread safety
if (pNode->handle.valueInTxn) {
if (TqDupIntxnReject(pMeta->tqConfigFlag)) {
return -2;
if (tqDupIntxnReject(pMeta->tqConfigFlag)) {
terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN;
return -1;
}
if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) {
pMeta->pDeleter(pNode->handle.valueInTxn);
@ -488,7 +492,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va
}
STqMetaList* pNewNode = malloc(sizeof(STqMetaList));
if (pNewNode == NULL) {
// TODO: memory error
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
memset(pNewNode, 0, sizeof(STqMetaList));
@ -505,7 +509,7 @@ int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return
int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) {
void* vmem = malloc(vsize);
if (vmem == NULL) {
// TODO: memory error
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
return -1;
}
memcpy(vmem, value, vsize);
@ -535,6 +539,7 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
while (pNode) {
if (pNode->handle.key == key) {
if (pNode->handle.valueInTxn == NULL) {
terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
return -1;
}
if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) {
@ -548,7 +553,8 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) {
pNode = pNode->next;
}
}
return -2;
terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
}
int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
@ -564,12 +570,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) {
tqLinkUnpersist(pMeta, pNode);
return 0;
}
terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN;
return -1;
} else {
pNode = pNode->next;
}
}
return -2;
terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
}
int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
@ -588,7 +596,7 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) {
pNode = pNode->next;
}
}
// no such key
terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY;
return -1;
}

115
source/dnode/vnode/tq/test/tqMetaTest.cpp
View File

@ -10,8 +10,8 @@ struct Foo {
};
int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
Foo* foo = (Foo*) pObj;
if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
Foo* foo = (Foo*)pObj;
if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) {
*ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t));
(*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t);
}
@ -20,7 +20,7 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) {
}
const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
if(*ppObj == NULL) {
if (*ppObj == NULL) {
*ppObj = realloc(*ppObj, sizeof(int32_t));
}
Foo* pFoo = *(Foo**)ppObj;
@ -28,28 +28,20 @@ const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) {
return NULL;
}
void FooDeleter(void* pObj) {
free(pObj);
}
void FooDeleter(void* pObj) { free(pObj); }
class TqMetaUpdateAppendTest : public ::testing::Test {
protected:
protected:
void SetUp() override {
taosRemoveDir(pathName);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
}
void SetUp() override {
taosRemoveDir(pathName);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
ASSERT(pMeta);
}
void TearDown() override { tqStoreClose(pMeta); }
void TearDown() override {
tqStoreClose(pMeta);
}
TqMetaStore* pMeta;
const char* pathName = "/tmp/tq_test";
STqMetaStore* pMeta;
const char* pathName = "/tmp/tq_test";
};
TEST_F(TqMetaUpdateAppendTest, copyPutTest) {
@ -57,11 +49,11 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) {
foo.a = 3;
tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo));
Foo* pFoo = (Foo*) tqHandleGet(pMeta, 1);
Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqHandleCommit(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo->a, 3);
}
@ -78,10 +70,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) {
EXPECT_EQ(pBar == NULL, true);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pBar = (Foo*)tqHandleGet(pMeta, 1);
@ -97,7 +86,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) {
pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
}
@ -106,11 +95,11 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) {
pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqHandleAbort(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
}
@ -119,32 +108,29 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) {
pFoo->a = 3;
tqHandleMovePut(pMeta, 1, pFoo);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqHandleCommit(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
ASSERT_EQ(pFoo != NULL, true);
EXPECT_EQ(pFoo->a, 3);
tqHandleDel(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
ASSERT_EQ(pFoo != NULL, true);
EXPECT_EQ(pFoo->a, 3);
tqHandleCommit(pMeta, 1);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pFoo = (Foo*) tqHandleGet(pMeta, 1);
pFoo = (Foo*)tqHandleGet(pMeta, 1);
EXPECT_EQ(pFoo == NULL, true);
}
@ -162,10 +148,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
EXPECT_EQ(pFoo1->a, 3);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
@ -177,10 +160,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
EXPECT_EQ(pFoo1->a, 4);
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
pFoo1 = (Foo*)tqHandleGet(pMeta, 1);
@ -190,13 +170,13 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) {
TEST_F(TqMetaUpdateAppendTest, multiplePage) {
srand(0);
std::vector<int> v;
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
v.push_back(rand());
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for(int i = 0; i < 500; i++) {
for (int i = 0; i < 500; i++) {
tqHandleCommit(pMeta, i);
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
@ -204,38 +184,34 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) {
}
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
for(int i = 500; i < 1000; i++) {
for (int i = 500; i < 1000; i++) {
tqHandleCommit(pMeta, i);
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
}
TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
srand(0);
std::vector<int> v;
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
v.push_back(rand());
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for(int i = 0; i < 500; i++) {
for (int i = 0; i < 500; i++) {
tqHandleCommit(pMeta, i);
v[i] = rand();
Foo foo;
@ -243,25 +219,22 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for(int i = 500; i < 1000; i++) {
for (int i = 500; i < 1000; i++) {
v[i] = rand();
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
tqHandleCommit(pMeta, i);
}
tqStoreClose(pMeta);
pMeta = tqStoreOpen(pathName,
FooSerializer, FooDeserializer, FooDeleter,
TQ_UPDATE_APPEND
);
pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND);
ASSERT(pMeta);
for(int i = 500; i < 1000; i++) {
for (int i = 500; i < 1000; i++) {
v[i] = rand();
Foo foo;
foo.a = v[i];
@ -269,40 +242,38 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) {
tqHandleCommit(pMeta, i);
}
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
}
TEST_F(TqMetaUpdateAppendTest, dupCommit) {
srand(0);
std::vector<int> v;
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
v.push_back(rand());
Foo foo;
foo.a = v[i];
tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo));
}
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
int ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, 0);
ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, -1);
}
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
int ret = tqHandleCommit(pMeta, i);
EXPECT_EQ(ret, -1);
}
for(int i = 0; i < 1000; i++) {
for (int i = 0; i < 1000; i++) {
Foo* pFoo = (Foo*)tqHandleGet(pMeta, i);
ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n";
EXPECT_EQ(pFoo->a, v[i]);
}
}

16
source/dnode/vnode/tsdb/CMakeLists.txt
View File

@ -1,10 +1,24 @@
aux_source_directory(src TSDB_SRC)
add_library(tsdb ${TSDB_SRC})
if(0)
add_library(tsdb ${TSDB_SRC})
else(0)
add_library(tsdb "")
target_sources(tsdb
PRIVATE
"src/tsdbCommit.c"
"src/tsdbMain.c"
"src/tsdbMemTable.c"
"src/tsdbOptions.c"
"src/tsdbWrite.c"
)
endif(0)
target_include_directories(
tsdb
PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
tsdb
PUBLIC os

2
src/tsdb/inc/tsdbCommit.h → source/dnode/vnode/tsdb/inc/tsdbCommit.h
View File

@ -16,6 +16,7 @@
#ifndef _TD_TSDB_COMMIT_H_
#define _TD_TSDB_COMMIT_H_
#if 0
typedef struct {
int minFid;
int midFid;
@ -53,5 +54,6 @@ static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
return -1;
}
}
#endif
#endif /* _TD_TSDB_COMMIT_H_ */

4
src/tsdb/inc/tsdbCompact.h → source/dnode/vnode/tsdb/inc/tsdbCompact.h
View File

@ -19,8 +19,12 @@
extern "C" {
#endif
#if 0
void *tsdbCompactImpl(STsdbRepo *pRepo);
#endif
#ifdef __cplusplus
}
#endif

4
src/tsdb/inc/tsdbFS.h → source/dnode/vnode/tsdb/inc/tsdbFS.h
View File

@ -16,6 +16,8 @@
#ifndef _TD_TSDB_FS_H_
#define _TD_TSDB_FS_H_
#if 0
#define TSDB_FS_VERSION 0
// ================== TSDB global config
@ -113,4 +115,6 @@ static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) {
return 0;
}
#endif
#endif /* _TD_TSDB_FS_H_ */

3
src/tsdb/inc/tsdbFile.h → source/dnode/vnode/tsdb/inc/tsdbFile.h
View File

@ -16,6 +16,8 @@
#ifndef _TS_TSDB_FILE_H_
#define _TS_TSDB_FILE_H_
#if 0
#define TSDB_FILE_HEAD_SIZE 512
#define TSDB_FILE_DELIMITER 0xF00AFA0F
#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
@ -364,4 +366,5 @@ static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet* pSet) {
return true;
}
#endif
#endif /* _TS_TSDB_FILE_H_ */

4
src/tsdb/inc/tsdbHealth.h → source/dnode/vnode/tsdb/inc/tsdbHealth.h
View File

@ -16,10 +16,14 @@
#ifndef _TD_TSDB_HEALTH_H_
#define _TD_TSDB_HEALTH_H_
#if 0
bool tsdbUrgeQueryFree(STsdbRepo* pRepo);
int32_t tsdbInsertNewBlock(STsdbRepo* pRepo);
bool tsdbIdleMemEnough();
bool tsdbAllowNewBlock(STsdbRepo* pRepo);
#endif
#endif /* _TD_TSDB_BUFFER_H_ */

27
source/dnode/vnode/tsdb/inc/tsdbIdx.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TSDB_IDX_H_
#define _TD_TSDB_IDX_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_TSDB_IDX_H_*/

3
src/tsdb/inc/tsdbReadImpl.h → source/dnode/vnode/tsdb/inc/tsdbReadImpl.h
View File

@ -15,6 +15,7 @@
#ifndef _TD_TSDB_READ_IMPL_H_
#define _TD_TSDB_READ_IMPL_H_
#if 0
#include "tfs.h"
#include "tsdb.h"
@ -150,4 +151,6 @@ static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
return 0;
}
#endif
#endif /*_TD_TSDB_READ_IMPL_H_*/

4
src/tsdb/inc/tsdbRowMergeBuf.h → source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h
View File

@ -16,6 +16,8 @@
#ifndef TSDB_ROW_MERGE_BUF_H
#define TSDB_ROW_MERGE_BUF_H
#if 0
#ifdef __cplusplus
extern "C" {
#endif
@ -42,4 +44,6 @@ static FORCE_INLINE void tsdbFreeMergeBuf(SMergeBuf buf) {
}
#endif
#endif
#endif /* ifndef TSDB_ROW_MERGE_BUF_H */

27
source/dnode/vnode/tsdb/inc/tsdbSMA.h
View File

@ -1,27 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TSDB_SMA_H_
#define _TD_TSDB_SMA_H_
#ifdef __cplusplus
extern "C" {
#endif
#ifdef __cplusplus
}
#endif
#endif /*_TD_TSDB_SMA_H_*/

2
src/tsdb/inc/tsdbint.h → source/dnode/vnode/tsdb/inc/tsdbint.h
View File

@ -16,6 +16,7 @@
#ifndef _TD_TSDB_INT_H_
#define _TD_TSDB_INT_H_
#if 0
// // TODO: remove the include
// #include <errno.h>
// #include <fcntl.h>
@ -144,4 +145,5 @@ static FORCE_INLINE int tsdbGetNextMaxTables(int tid) {
}
#endif
#endif
#endif /* _TD_TSDB_INT_H_ */

1669
source/dnode/vnode/tsdb/src/tsdbCommit.c
View File

File diff suppressed because it is too large Load Diff

0
src/tsdb/src/tsdbCompact.c → source/dnode/vnode/tsdb/src/tsdbCompact.c
View File

0
src/tsdb/src/tsdbFS.c → source/dnode/vnode/tsdb/src/tsdbFS.c
View File

0
src/tsdb/src/tsdbFile.c → source/dnode/vnode/tsdb/src/tsdbFile.c
View File

0
src/tsdb/src/tsdbHealth.c → source/dnode/vnode/tsdb/src/tsdbHealth.c
View File

14
source/dnode/vnode/tsdb/src/tsdbIdx.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

1010
source/dnode/vnode/tsdb/src/tsdbMain.c
View File

File diff suppressed because it is too large Load Diff

0
src/tsdb/src/tsdbRead.c → source/dnode/vnode/tsdb/src/tsdbRead.c
View File

0
src/tsdb/src/tsdbReadImpl.c → source/dnode/vnode/tsdb/src/tsdbReadImpl.c
View File

0
src/tsdb/src/tsdbRowMergeBuf.c → source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c
View File

14
source/dnode/vnode/tsdb/src/tsdbSMA.c
View File

@ -1,14 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

0
src/tsdb/src/tsdbScan.c → source/dnode/vnode/tsdb/src/tsdbScan.c
View File

0
src/tsdb/src/tsdbSync.c → source/dnode/vnode/tsdb/src/tsdbSync.c
View File

4
source/libs/catalog/test/catalogTests.cpp
View File

@ -61,8 +61,8 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) {
pReq->daysToKeep0 = htonl(3650);
pReq->daysToKeep1 = htonl(3650);
pReq->daysToKeep2 = htonl(3650);
pReq->minRowsPerFileBlock = htonl(100);
pReq->maxRowsPerFileBlock = htonl(4096);
pReq->minRows = htonl(100);
pReq->maxRows = htonl(4096);
pReq->commitTime = htonl(3600);
pReq->fsyncPeriod = htonl(3000);
pReq->walLevel = 1;

99
source/libs/index/inc/indexInt.h
View File

@ -18,43 +18,42 @@
#include "index.h"
#include "index_fst.h"
#include "tlog.h"
#include "thash.h"
#include "taos.h"
#include "thash.h"
#include "tlog.h"
#ifdef USE_LUCENE
#include <lucene++/Lucene_c.h>
#endif
#ifdef __cplusplus
extern "C" {
#endif
typedef enum {kTypeValue, kTypeDeletion} STermValueType ;
typedef enum { kTypeValue, kTypeDeletion } STermValueType;
typedef struct SIndexStat {
int32_t totalAdded; //
int32_t totalDeled; //
int32_t totalUpdated; //
int32_t totalTerms; //
int32_t distinctCol; // distinct column
int32_t totalAdded; //
int32_t totalDeled; //
int32_t totalUpdated; //
int32_t totalTerms; //
int32_t distinctCol; // distinct column
} SIndexStat;
struct SIndex {
#ifdef USE_LUCENE
index_t *index;
index_t *index;
#endif
void *cache;
void *tindex;
SHashObj *colObj;// < field name, field id>
void * cache;
void * tindex;
SHashObj *colObj; // < field name, field id>
int64_t suid; // current super table id, -1 is normal table
int colId; // field id allocated to cache
int32_t cVersion; // current version allocated to cache
int64_t suid; // current super table id, -1 is normal table
int colId; // field id allocated to cache
int32_t cVersion; // current version allocated to cache
SIndexStat stat;
pthread_mutex_t mtx;
SIndexStat stat;
pthread_mutex_t mtx;
};
struct SIndexOpts {
@ -66,39 +65,65 @@ struct SIndexOpts {
int32_t cacheSize; // MB
// add cache module later
#endif
};
struct SIndexMultiTermQuery {
EIndexOperatorType opera;
SArray *query;
SArray * query;
};
// field and key;
typedef struct SIndexTerm {
int64_t suid;
SIndexOperOnColumn operType; // oper type, add/del/update
uint8_t colType; // term data type, str/interger/json
char *colName;
int32_t nColName;
char *colVal;
int32_t nColVal;
int64_t suid;
SIndexOperOnColumn operType; // oper type, add/del/update
uint8_t colType; // term data type, str/interger/json
char * colName;
int32_t nColName;
char * colVal;
int32_t nColVal;
} SIndexTerm;
typedef struct SIndexTermQuery {
SIndexTerm* term;
SIndexTerm * term;
EIndexQueryType qType;
} SIndexTermQuery;
#define indexFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("index FATAL ", 255, __VA_ARGS__); }} while(0)
#define indexError(...) do { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("index ERROR ", 255, __VA_ARGS__); }} while(0)
#define indexWarn(...) do { if (sDebugFlag & DEBUG_WARN) { taosPrintLog("index WARN ", 255, __VA_ARGS__); }} while(0)
#define indexInfo(...) do { if (sDebugFlag & DEBUG_INFO) { taosPrintLog("index ", 255, __VA_ARGS__); }} while(0)
#define indexDebug(...) do { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0)
#define indexTrace(...) do { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("index ", sDebugFlag, __VA_ARGS__); }} while(0)
#define indexFatal(...) \
do { \
if (sDebugFlag & DEBUG_FATAL) { \
taosPrintLog("index FATAL ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexError(...) \
do { \
if (sDebugFlag & DEBUG_ERROR) { \
taosPrintLog("index ERROR ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexWarn(...) \
do { \
if (sDebugFlag & DEBUG_WARN) { \
taosPrintLog("index WARN ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexInfo(...) \
do { \
if (sDebugFlag & DEBUG_INFO) { \
taosPrintLog("index ", 255, __VA_ARGS__); \
} \
} while (0)
#define indexDebug(...) \
do { \
if (sDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define indexTrace(...) \
do { \
if (sDebugFlag & DEBUG_TRACE) { \
taosPrintLog("index ", sDebugFlag, __VA_ARGS__); \
} \
} while (0)
#ifdef __cplusplus
}

16
source/libs/index/inc/index_cache.h
View File

@ -22,8 +22,10 @@
// ----------------- key structure in skiplist ---------------------
/* A data row, the format is like below:
* content: |<--totalLen-->|<-- fieldid-->|<--field type -->|<-- value len--->|<-- value -->|<-- uid -->|<--version--->|<-- itermType -->|
* len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|<--valuelen->|<--uint64_t->|<-- int32_t-->|<-- int8_t --->|
* content: |<--totalLen-->|<-- fieldid-->|<--field type-->|<-- value len--->|
* |<-- value -->|<--uid -->|<--version--->|<-- itermType -->|
* len : |<--int32_t -->|<-- int16_t-->|<-- int8_t --->|<--- int32_t --->|
* <--valuelen->|<--uint64_t->| * <-- int32_t-->|<-- int8_t --->|
*/
#ifdef __cplusplus
@ -35,21 +37,19 @@ typedef struct IndexCache {
SSkipList *skiplist;
} IndexCache;
//
IndexCache *indexCacheCreate();
void indexCacheDestroy(void *cache);
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid);
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid);
//int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s);
// int indexCacheGet(void *cache, uint64_t *rst);
int indexCacheSearch(
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s);
#ifdef __cplusplus
}
#endif
#endif

218
source/libs/index/inc/index_fst.h
View File

@ -20,19 +20,19 @@
extern "C" {
#endif
#include "tarray.h"
#include "index_fst_util.h"
#include "index_fst_registry.h"
#include "index_fst_counting_writer.h"
#include "index_fst_automation.h"
#include "index_fst_counting_writer.h"
#include "index_fst_registry.h"
#include "index_fst_util.h"
#include "tarray.h"
#define OUTPUT_PREFIX(a, b) ((a) > (b) ? (b) : (a)
typedef struct Fst Fst;
typedef struct FstNode FstNode;
typedef struct Fst Fst;
typedef struct FstNode FstNode;
typedef struct StreamWithState StreamWithState;
typedef enum { Included, Excluded, Unbounded} FstBound;
typedef enum { Included, Excluded, Unbounded } FstBound;
typedef struct FstBoundWithData {
FstSlice data;
@ -40,8 +40,8 @@ typedef struct FstBoundWithData {
} FstBoundWithData;
typedef struct FstStreamBuilder {
Fst *fst;
AutomationCtx *aut;
Fst * fst;
AutomationCtx * aut;
FstBoundWithData *min;
FstBoundWithData *max;
} FstStreamBuilder, FstStreamWithStateBuilder;
@ -51,17 +51,14 @@ typedef struct FstRange {
uint64_t end;
} FstRange;
typedef enum { GE, GT, LE, LT } RangeType;
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal } State;
typedef enum { Ordered, OutOfOrdered, DuplicateKey } OrderType;
typedef enum {GE, GT, LE, LT} RangeType;
typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State;
typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType;
FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data);
bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice);
bool fstBoundWithDataIsEmpty(FstBoundWithData *bound);
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound);
FstBoundWithData *fstBoundStateCreate(FstBound type, FstSlice *data);
bool fstBoundWithDataExceededBy(FstBoundWithData *bound, FstSlice *slice);
bool fstBoundWithDataIsEmpty(FstBoundWithData *bound);
bool fstBoundWithDataIsIncluded(FstBoundWithData *bound);
typedef struct FstOutput {
bool null;
@ -74,87 +71,82 @@ typedef struct FstOutput {
* TODO: simple function name
*/
typedef struct FstUnFinishedNodes {
SArray *stack; // <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
SArray *stack; // <FstBuilderNodeUnfinished> } FstUnFinishedNodes;
} FstUnFinishedNodes;
#define FST_UNFINISHED_NODES_LEN(nodes) taosArrayGetSize(nodes->stack)
FstUnFinishedNodes *fstUnFinishedNodesCreate();
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node);
void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal);
FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes);
FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr);
FstBuilderNode *fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes);
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out);
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr);
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out);
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs);
void fstUnFinishedNodesDestroy(FstUnFinishedNodes *node);
void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal);
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes *node, Output out);
void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *node, CompiledAddr addr);
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *node, FstSlice bs, Output out);
uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs);
FstBuilderNode * fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes);
FstBuilderNode * fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledAddr addr);
FstBuilderNode * fstUnFinishedNodesPopEmpty(FstUnFinishedNodes *nodes);
uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstSlice bs, Output in, Output *out);
typedef struct FstBuilder {
FstCountingWriter *wrt; // The FST raw data is written directly to `wtr`.
FstUnFinishedNodes *unfinished; // The stack of unfinished nodes
FstRegistry* registry; // A map of finished nodes.
FstSlice last; // The last word added
CompiledAddr lastAddr; // The address of the last compiled node
uint64_t len; // num of keys added
FstCountingWriter * wrt; // The FST raw data is written directly to `wtr`.
FstUnFinishedNodes *unfinished; // The stack of unfinished nodes
FstRegistry * registry; // A map of finished nodes.
FstSlice last; // The last word added
CompiledAddr lastAddr; // The address of the last compiled node
uint64_t len; // num of keys added
} FstBuilder;
FstBuilder *fstBuilderCreate(void *w, FstType ty);
void fstBuilderDestroy(FstBuilder *b);
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in);
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup);
void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate);
void fstBuilderDestroy(FstBuilder *b);
void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in);
bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in);
void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate);
void * fstBuilerIntoInner(FstBuilder *b);
void fstBuilderFinish(FstBuilder *b);
OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup);
CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn);
void* fstBuilerIntoInner(FstBuilder *b);
void fstBuilderFinish(FstBuilder *b);
typedef struct FstTransitions {
FstNode *node;
FstRange range;
FstNode *node;
FstRange range;
} FstTransitions;
//FstState and relation function
// FstState and relation function
typedef struct FstState {
State state;
State state;
uint8_t val;
} FstState;
FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr);
FstState fstStateCreateFrom(FstSlice *data, CompiledAddr addr);
FstState fstStateCreate(State state);
//compile
// compile
void fstStateCompileForOneTransNext(FstCountingWriter *w, CompiledAddr addr, uint8_t inp);
void fstStateCompileForOneTrans(FstCountingWriter *w, CompiledAddr addr, FstTransition *trn);
void fstStateCompileForAnyTrans(FstCountingWriter *w, CompiledAddr addr, FstBuilderNode *node);
// set_comm_input
void fstStateSetCommInput(FstState* state, uint8_t inp);
void fstStateSetCommInput(FstState *state, uint8_t inp);
// comm_input
uint8_t fstStateCommInput(FstState* state, bool *null);
uint8_t fstStateCommInput(FstState *state, bool *null);
// input_len
uint64_t fstStateInputLen(FstState* state);
uint64_t fstStateInputLen(FstState *state);
// end_addr
uint64_t fstStateEndAddrForOneTransNext(FstState* state, FstSlice *data);
uint64_t fstStateEndAddrForOneTransNext(FstState *state, FstSlice *data);
uint64_t fstStateEndAddrForOneTrans(FstState *state, FstSlice *data, PackSizes sizes);
uint64_t fstStateEndAddrForAnyTrans(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
uint64_t fstStateEndAddrForAnyTrans(
FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
// input
uint8_t fstStateInput(FstState *state, FstNode *node);
uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i);
uint8_t fstStateInput(FstState *state, FstNode *node);
uint8_t fstStateInputForAnyTrans(FstState *state, FstNode *node, uint64_t i);
// trans_addr
CompiledAddr fstStateTransAddr(FstState *state, FstNode *node);
@ -172,7 +164,7 @@ void fstStateSetFinalState(FstState *state, bool yes);
bool fstStateIsFinalState(FstState *state);
void fstStateSetStateNtrans(FstState *state, uint8_t n);
// state_ntrans
uint8_t fstStateStateNtrans(FstState *state, bool *null);
uint8_t fstStateStateNtrans(FstState *state, bool *null);
uint64_t fstStateTotalTransSize(FstState *state, uint64_t version, PackSizes size, uint64_t nTrans);
uint64_t fstStateTransIndexSize(FstState *state, uint64_t version, uint64_t nTrans);
uint64_t fstStateNtransLen(FstState *state);
@ -180,14 +172,11 @@ uint64_t fstStateNtrans(FstState *state, FstSlice *slice);
Output fstStateFinalOutput(FstState *state, uint64_t version, FstSlice *date, PackSizes sizes, uint64_t nTrans);
uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null);
#define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext)
#define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans)
#define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans)
#define FST_STATE_EMPTY_FINAL(node) (node->state.state == EmptyFinal)
typedef struct FstLastTransition {
uint8_t inp;
Output out;
@ -198,13 +187,12 @@ typedef struct FstLastTransition {
* TODO: simple function name
*/
typedef struct FstBuilderNodeUnfinished {
FstBuilderNode *node;
FstLastTransition* last;
FstBuilderNode * node;
FstLastTransition *last;
} FstBuilderNodeUnfinished;
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished *node, CompiledAddr addr);
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *node, Output out);
/*
@ -219,31 +207,35 @@ typedef struct FstNode {
bool isFinal;
uint64_t nTrans;
PackSizes sizes;
Output finalOutput;
Output finalOutput;
} FstNode;
// If this node is final and has a terminal output value, then it is, returned. Otherwise, a zero output is returned
// If this node is final and has a terminal output value, then it is, returned.
// Otherwise, a zero output is returned
#define FST_NODE_FINAL_OUTPUT(node) node->finalOutput
// Returns true if and only if this node corresponds to a final or "match", state in the finite state transducer.
// Returns true if and only if this node corresponds to a final or "match",
// state in the finite state transducer.
#define FST_NODE_IS_FINAL(node) node->isFinal
// Returns the number of transitions in this node, The maximum number of transitions is 256.
// Returns the number of transitions in this node, The maximum number of
// transitions is 256.
#define FST_NODE_LEN(node) node->nTrans
// Returns true if and only if this node has zero transitions.
#define FST_NODE_IS_EMPTYE(node) (node->nTrans == 0)
// Return the address of this node.
#define FST_NODE_ADDR(node) node->start
FstNode *fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice *data);
void fstNodeDestroy(FstNode *fstNode);
void fstNodeDestroy(FstNode *fstNode);
FstTransitions fstNodeTransitionIter(FstNode *node);
FstTransitions *fstNodeTransitions(FstNode *node);
bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res);
bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res);
bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res);
FstTransitions fstNodeTransitionIter(FstNode *node);
FstTransitions* fstNodeTransitions(FstNode *node);
bool fstNodeGetTransitionAt(FstNode *node, uint64_t i, FstTransition *res);
bool fstNodeGetTransitionAddrAt(FstNode *node, uint64_t i, CompiledAddr *res);
bool fstNodeFindInput(FstNode *node, uint8_t b, uint64_t *res);
bool fstNodeCompile(FstNode *node, void *w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode *builderNode);
FstSlice fstNodeAsSlice(FstNode *node);
FstSlice fstNodeAsSlice(FstNode *node);
// ops
@ -253,8 +245,7 @@ typedef struct FstIndexedValue {
} FstIndexedValue;
FstLastTransition *fstLastTransitionCreate(uint8_t inp, Output out);
void fstLastTransitionDestroy(FstLastTransition *trn);
void fstLastTransitionDestroy(FstLastTransition *trn);
typedef struct FstMeta {
uint64_t version;
@ -265,67 +256,69 @@ typedef struct FstMeta {
} FstMeta;
typedef struct Fst {
FstMeta *meta;
FstMeta * meta;
FstSlice *data; //
FstNode *root; //
FstNode * root; //
} Fst;
// refactor simple function
Fst* fstCreate(FstSlice *data);
void fstDestroy(Fst *fst);
Fst *fstCreate(FstSlice *data);
void fstDestroy(Fst *fst);
bool fstGet(Fst *fst, FstSlice *b, Output *out);
FstNode* fstGetNode(Fst *fst, CompiledAddr);
FstNode* fstGetRoot(Fst *fst);
FstType fstGetType(Fst *fst);
CompiledAddr fstGetRootAddr(Fst *fst);
Output fstEmptyFinalOutput(Fst *fst, bool *null);
bool fstGet(Fst *fst, FstSlice *b, Output *out);
FstNode * fstGetNode(Fst *fst, CompiledAddr);
FstNode * fstGetRoot(Fst *fst);
FstType fstGetType(Fst *fst);
CompiledAddr fstGetRootAddr(Fst *fst);
Output fstEmptyFinalOutput(Fst *fst, bool *null);
FstStreamBuilder *fstSearch(Fst *fst, AutomationCtx *ctx);
FstStreamWithStateBuilder *fstSearchWithState(Fst *fst, AutomationCtx *ctx);
// into stream to expand later
StreamWithState* streamBuilderIntoStream(FstStreamBuilder *sb);
StreamWithState *streamBuilderIntoStream(FstStreamBuilder *sb);
bool fstVerify(Fst *fst);
bool fstVerify(Fst *fst);
//refactor this function
// refactor this function
bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
typedef struct StreamState {
FstNode *node;
FstNode * node;
uint64_t trans;
FstOutput out;
void *autState;
void * autState;
} StreamState;
void streamStateDestroy(void *s);
typedef struct StreamWithState {
Fst *fst;
AutomationCtx *aut;
SArray *inp;
FstOutput emptyOutput;
SArray *stack; // <StreamState>
Fst * fst;
AutomationCtx * aut;
SArray * inp;
FstOutput emptyOutput;
SArray * stack; // <StreamState>
FstBoundWithData *endAt;
} StreamWithState;
typedef struct StreamWithStateResult {
FstSlice data;
FstSlice data;
FstOutput out;
void *state;
void * state;
} StreamWithStateResult;
StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state);
void swsResultDestroy(StreamWithStateResult *result);
void swsResultDestroy(StreamWithStateResult *result);
typedef void *(*StreamCallback)(void *);
StreamWithState *streamWithStateCreate(
Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max);
typedef void* (*StreamCallback)(void *);
StreamWithState *streamWithStateCreate(Fst *fst, AutomationCtx *automation, FstBoundWithData *min, FstBoundWithData *max) ;
void streamWithStateDestroy(StreamWithState *sws);
bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min);
StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback);
StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallback callback);
FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut);
// set up bound range
@ -333,7 +326,6 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, AutomationCtx *aut);
FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type);
#ifdef __cplusplus
}
#endif

34
source/libs/index/inc/index_fst_automation.h
View File

@ -21,16 +21,12 @@ extern "C" {
#include "index_fst_util.h"
typedef struct AutomationCtx AutomationCtx;
typedef enum AutomationType {
AUTOMATION_PREFIX,
AUTMMATION_MATCH
} AutomationType;
typedef enum AutomationType { AUTOMATION_PREFIX, AUTMMATION_MATCH } AutomationType;
typedef struct StartWith {
AutomationCtx *autoSelf;
AutomationCtx *autoSelf;
} StartWith;
typedef struct Complement {
@ -40,41 +36,39 @@ typedef struct Complement {
// automation
typedef struct AutomationCtx {
AutomationType type;
void *stdata;
char *data;
void * stdata;
char * data;
} AutomationCtx;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY} ValueType;
typedef enum ValueType { FST_INT, FST_CHAR, FST_ARRAY } ValueType;
typedef enum StartWithStateKind { Done, Running } StartWithStateKind;
typedef struct StartWithStateValue {
StartWithStateKind kind;
ValueType type;
ValueType type;
union {
int val;
char *ptr;
int val;
char * ptr;
SArray *arr;
// add more type
} ;
};
} StartWithStateValue;
StartWithStateValue *startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void *val);
StartWithStateValue *startWithStateValueDump(StartWithStateValue *sv);
void startWithStateValueDestroy(void *sv);
void startWithStateValueDestroy(void *sv);
typedef struct AutomationFunc {
void* (*start)(AutomationCtx *ctx) ;
void *(*start)(AutomationCtx *ctx);
bool (*isMatch)(AutomationCtx *ctx, void *);
bool (*canMatch)(AutomationCtx *ctx, void *data);
bool (*willAlwaysMatch)(AutomationCtx *ctx, void *state);
void* (*accept)(AutomationCtx *ctx, void *state, uint8_t byte);
void* (*acceptEof)(AutomationCtx *ct, void *state);
void *(*accept)(AutomationCtx *ctx, void *state, uint8_t byte);
void *(*acceptEof)(AutomationCtx *ct, void *state);
} AutomationFunc;
AutomationCtx *automCtxCreate(void *data, AutomationType atype);
void automCtxDestroy(AutomationCtx *ctx);
void automCtxDestroy(AutomationCtx *ctx);
extern AutomationFunc automFuncs[];
#ifdef __cplusplus

4
source/libs/index/inc/index_fst_common.h
View File

@ -1,9 +1,9 @@
#ifndef __INDEX_FST_COMM_H__
#define __INDEX_FST_COMM_H__
#include "tutil.h"
extern const uint8_t COMMON_INPUTS[];
extern char const COMMON_INPUTS_INV[];
extern const char COMMON_INPUTS_INV[];
#ifdef __cplusplus
extern "C" {

35
source/libs/index/inc/index_fst_counting_writer.h
View File

@ -22,11 +22,10 @@ extern "C" {
#include "tfile.h"
#define DefaultMem 1024*1024
#define DefaultMem 1024 * 1024
static char tmpFile[] = "./index";
typedef enum WriterType {TMemory, TFile} WriterType;
typedef enum WriterType { TMemory, TFile } WriterType;
typedef struct WriterCtx {
int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len);
@ -34,8 +33,14 @@ typedef struct WriterCtx {
int (*flush)(struct WriterCtx *ctx);
WriterType type;
union {
int fd;
void *mem;
struct {
int fd;
bool readOnly;
} file;
struct {
int32_t capa;
char * buf;
} mem;
};
int32_t offset;
int32_t limit;
@ -45,15 +50,14 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len);
static int writeCtxDoFlush(WriterCtx *ctx);
WriterCtx* writerCtxCreate(WriterType type, bool readOnly);
void writerCtxDestroy(WriterCtx *w);
WriterCtx *writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity);
void writerCtxDestroy(WriterCtx *w);
typedef uint32_t CheckSummer;
typedef struct FstCountingWriter {
void* wrt; // wrap any writer that counts and checksum bytes written
uint64_t count;
void * wrt; // wrap any writer that counts and checksum bytes written
uint64_t count;
CheckSummer summer;
} FstCountingWriter;
@ -63,17 +67,14 @@ int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len);
int fstCountingWriterFlush(FstCountingWriter *write);
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write);
FstCountingWriter *fstCountingWriterCreate(void *wtr, bool readOnly);
void fstCountingWriterDestroy(FstCountingWriter *w);
FstCountingWriter *fstCountingWriterCreate(void *wtr);
void fstCountingWriterDestroy(FstCountingWriter *w);
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes);
void fstCountingWriterPackUintIn(FstCountingWriter *writer, uint64_t n, uint8_t nBytes);
uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n);
#define FST_WRITER_COUNT(writer) (writer->count)
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
@ -83,5 +84,3 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n);
#endif
#endif

15
source/libs/index/inc/index_fst_node.h
View File

@ -20,22 +20,22 @@
extern "C" {
#endif
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal)
#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0)
#define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0)
typedef struct FstTransition {
uint8_t inp; //The byte input associated with this transition.
Output out; //The output associated with this transition
CompiledAddr addr; //The address of the node that this transition points to
uint8_t inp; // The byte input associated with this transition.
Output out; // The output associated with this transition
CompiledAddr addr; // The address of the node that this transition points to
} FstTransition;
typedef struct FstBuilderNode {
bool isFinal;
Output finalOutput;
bool isFinal;
Output finalOutput;
SArray *trans; // <FstTransition>
} FstBuilderNode;
@ -45,7 +45,8 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src);
void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src);
//bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
// CompiledAddr lastAddr, CompiledAddr startAddr);
bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2);
void fstBuilderNodeDestroy(FstBuilderNode *node);

35
source/libs/index/inc/index_fst_registry.h
View File

@ -19,49 +19,48 @@
extern "C" {
#endif
#include "index_fst_node.h"
#include "index_fst_util.h"
#include "tarray.h"
#include "index_fst_node.h"
typedef struct FstRegistryCell {
CompiledAddr addr;
CompiledAddr addr;
FstBuilderNode *node;
} FstRegistryCell;
#define FST_REGISTRY_CELL_IS_EMPTY(cell) (cell->addr == NONE_ADDRESS)
#define FST_REGISTRY_CELL_INSERT(cell, tAddr) do {cell->addr = tAddr;} while(0)
#define FST_REGISTRY_CELL_INSERT(cell, tAddr) \
do { \
cell->addr = tAddr; \
} while (0)
//typedef struct FstRegistryCache {
// typedef struct FstRegistryCache {
// SArray *cells;
// uint32_t start;
// uint32_t end;
//} FstRegistryCache;
typedef enum {FOUND, NOTFOUND, REJECTED} FstRegistryEntryState;
typedef enum { FOUND, NOTFOUND, REJECTED } FstRegistryEntryState;
typedef struct FstRegistryEntry {
FstRegistryEntryState state;
CompiledAddr addr;
FstRegistryCell *cell;
CompiledAddr addr;
FstRegistryCell * cell;
} FstRegistryEntry;
// Registry relation function
typedef struct FstRegistry {
SArray *table; //<FstRegistryCell>
uint64_t tableSize; // num of rows
uint64_t mruSize; // num of columns
SArray * table; //<FstRegistryCell>
uint64_t tableSize; // num of rows
uint64_t mruSize; // num of columns
} FstRegistry;
//
FstRegistry* fstRegistryCreate(uint64_t tableSize, uint64_t mruSize);
void fstRegistryDestroy(FstRegistry *registry);
FstRegistry *fstRegistryCreate(uint64_t tableSize, uint64_t mruSize);
void fstRegistryDestroy(FstRegistry *registry);
FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode);
void fstRegistryEntryDestroy(FstRegistryEntry *entry);
FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode);
void fstRegistryEntryDestroy(FstRegistryEntry *entry);
#ifdef __cplusplus
}

72
source/libs/index/inc/index_fst_util.h
View File

@ -13,7 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __INDEX_FST_UTIL_H__
#define __INDEX_FST_UTIL_H__
@ -21,16 +20,15 @@
extern "C" {
#endif
#include "tarray.h"
#include "index_fst_common.h"
#include "tarray.h"
typedef uint64_t FstType;
typedef uint64_t CompiledAddr;
typedef uint64_t Output;
typedef uint8_t PackSizes;
typedef uint8_t PackSizes;
//A sentinel value used to indicate an empty final state
// A sentinel value used to indicate an empty final state
extern const CompiledAddr EMPTY_ADDRESS;
/// A sentinel value used to indicate an invalid state.
extern const CompiledAddr NONE_ADDRESS;
@ -38,7 +36,7 @@ extern const CompiledAddr NONE_ADDRESS;
// This version number is written to every finite state transducer created by
// this version When a finite state transducer is read, its version number is
// checked against this value.
extern const uint64_t VERSION;
extern const uint64_t VERSION;
// The threshold (in number of transitions) at which an index is created for
// a node's transitions. This speeds up lookup time at the expense of FST size
@ -48,56 +46,60 @@ extern const uint64_t TRANS_INDEX_THRESHOLD;
//
// `0` is a legal value which means there are no transitions/outputs
#define FST_SET_TRANSITION_PACK_SIZE(v, sz) \
do { \
v = (v & 0b00001111) | (sz << 4); \
} while (0)
#define FST_GET_TRANSITION_PACK_SIZE(v) (((v)&0b11110000) >> 4)
#define FST_SET_OUTPUT_PACK_SIZE(v, sz) \
do { \
v = (v & 0b11110000) | sz; \
} while (0)
#define FST_GET_OUTPUT_PACK_SIZE(v) ((v)&0b00001111)
#define FST_SET_TRANSITION_PACK_SIZE(v, sz) do {v = (v & 0b00001111) | (sz << 4); } while(0)
#define FST_GET_TRANSITION_PACK_SIZE(v) (((v) & 0b11110000) >> 4)
#define FST_SET_OUTPUT_PACK_SIZE(v, sz) do { v = (v & 0b11110000) | sz; } while(0)
#define FST_GET_OUTPUT_PACK_SIZE(v) ((v) & 0b00001111)
#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx)-1]
#define COMMON_INPUT(idx) COMMON_INPUTS_INV[(idx) - 1]
#define COMMON_INDEX(v, max, val) \
do { \
val = ((uint16_t)COMMON_INPUTS[v] + 1) % 256; \
val = val > max ? 0 : val; \
} while (0)
#define COMMON_INDEX(v, max, val) do { \
val = ((uint16_t)COMMON_INPUTS[v] + 1)%256; \
val = val > max ? 0: val; \
} while(0)
//uint8_t commonInput(uint8_t idx);
//uint8_t commonIdx(uint8_t v, uint8_t max);
// uint8_t commonInput(uint8_t idx);
// uint8_t commonIdx(uint8_t v, uint8_t max);
uint8_t packSize(uint64_t n);
uint64_t unpackUint64(uint8_t *ch, uint8_t sz);
uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr);
CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr);
typedef struct FstString {
uint8_t *data;
uint32_t len;
int32_t ref;
int32_t ref;
} FstString;
typedef struct FstSlice {
FstString *str;
int32_t start;
int32_t end;
FstString *str;
int32_t start;
int32_t end;
} FstSlice;
FstSlice fstSliceCreate(uint8_t *data, uint64_t len);
FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end);
FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end);
bool fstSliceIsEmpty(FstSlice *s);
int fstSliceCompare(FstSlice *s1, FstSlice *s2);
void fstSliceDestroy(FstSlice *s);
bool fstSliceIsEmpty(FstSlice *s);
int fstSliceCompare(FstSlice *s1, FstSlice *s2);
void fstSliceDestroy(FstSlice *s);
uint8_t *fstSliceData(FstSlice *s, int32_t *sz);
#define FST_SLICE_LEN(s) (s->end - s->start + 1)
//// stack
//
//typedef (*StackFreeElemFn)(void *elem);
// typedef (*StackFreeElemFn)(void *elem);
//
//typedef struct FstStack {
// typedef struct FstStack {
// void *first;
// void *end;
// size_t elemSize;
@ -106,15 +108,13 @@ uint8_t *fstSliceData(FstSlice *s, int32_t *sz);
//} FstStack;
//
//
//FstStack* fstStackCreate(size_t elemSize, stackFreeElem);
//void *fstStackPush(FstStack *s, void *elem);
//void *fstStackTop(FstStack *s);
//size_t fstStackLen(FstStack *s);
//void fstStackDestory(FstStack *);
// FstStack* fstStackCreate(size_t elemSize, stackFreeElem);
// void *fstStackPush(FstStack *s, void *elem);
// void *fstStackTop(FstStack *s);
// size_t fstStackLen(FstStack *s);
// void fstStackDestory(FstStack *);
//
#ifdef __cplusplus
}
#endif

81
source/libs/index/inc/index_tfile.h
View File

@ -17,30 +17,97 @@
#include "index.h"
#include "indexInt.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_tfile.h"
#include "tlockfree.h"
#include "tskiplist.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef struct IndexTFile {
// tfile header
// |<---suid--->|<---version--->|<--colLen-->|<-colName->|<---type-->|
// |<-uint64_t->|<---int32_t--->|<--int32_t->|<-colLen-->|<-uint8_t->|
typedef struct TFileReadHeader {
uint64_t suid;
int32_t version;
char colName[128]; //
uint8_t colType;
} TFileReadHeader;
#define TFILE_HEADER_SIZE (sizeof(TFILE_HEADER_SIZE) + sizeof(uint32_t));
#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef struct TFileCacheKey {
uint64_t suid;
uint8_t colType;
int32_t version;
const char *colName;
int32_t nColName;
} TFileCacheKey;
// table cache
// refactor to LRU cache later
typedef struct TFileCache {
SHashObj *tableCache;
int16_t capacity;
// add more param
} TFileCache;
typedef struct TFileWriter {
FstBuilder *fb;
WriterCtx * ctx;
} TFileWriter;
typedef struct TFileReader {
T_REF_DECLARE()
Fst * fst;
WriterCtx * ctx;
TFileReadHeader header;
} TFileReader;
typedef struct IndexTFile {
char * path;
TFileCache * cache;
TFileWriter *tw;
} IndexTFile;
typedef struct TFileWriterOpt {
uint64_t suid;
int8_t colType;
char * colName;
int32_t nColName;
int32_t version;
} TFileWriterOpt;
typedef struct TFileReaderOpt {
uint64_t suid;
char * colName;
int32_t nColName;
} TFileReaderOpt;
IndexTFile *indexTFileCreate();
// tfile cache, manage tindex reader
TFileCache * tfileCacheCreate(const char *path);
void tfileCacheDestroy(TFileCache *tcache);
TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key);
void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader);
TFileReader *tfileReaderCreate();
void TFileReaderDestroy(TFileReader *reader);
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result);
TFileWriter *tfileWriterCreate(const char *suid, const char *colName);
void tfileWriterDestroy(TFileWriter *tw);
//
IndexTFile *indexTFileCreate(const char *path);
int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid);
int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result);
#ifdef __cplusplus
}
#endif
#endif

52
source/libs/index/inc/index_util.h Normal file
View File

@ -0,0 +1,52 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __INDEX_UTIL_H__
#define __INDEX_UTIL_H__
#ifdef __cplusplus
extern "C" {
#endif
#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \
do { \
memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \
buf += sizeof(key->mem); \
} while (0)
#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \
do { \
memcpy((void *)buf, (void *)key->mem, len); \
buf += len; \
} while (0)
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(var) == sizeof(type)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)
#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \
do { \
memcpy((void *)buf, (void *)var, len); \
buf += len; \
} while (0)
#ifdef __cplusplus
}
#endif
#endif

176
source/libs/index/src/index.c
View File

@ -26,38 +26,42 @@
static int uidCompare(const void *a, const void *b) {
uint64_t u1 = *(uint64_t *)a;
uint64_t u2 = *(uint64_t *)b;
if (u1 == u2) { return 0; }
else { return u1 < u2 ? -1 : 1; }
if (u1 == u2) {
return 0;
} else {
return u1 < u2 ? -1 : 1;
}
}
typedef struct SIdxColInfo {
int colId; // generated by index internal
int colId; // generated by index internal
int cVersion;
} SIdxColInfo;
static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit();
static void indexInit();
static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *term, SArray **result);
static int indexMergeCacheIntoTindex(SIndex *sIdx);
static void indexInterResultsDestroy(SArray *results);
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult);
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *finalResult);
int indexOpen(SIndexOpts *opts, const char *path, SIndex **index) {
pthread_once(&isInit, indexInit);
SIndex *sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { return -1; }
if (sIdx == NULL) {
return -1;
}
#ifdef USE_LUCENE
index_t *index = index_open(path);
sIdx->index = index;
#endif
sIdx->cache = (void*)indexCacheCreate();
sIdx->tindex = NULL;
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->colId = 1;
sIdx->cache = (void *)indexCacheCreate();
sIdx->tindex = NULL;
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->colId = 1;
sIdx->cVersion = 1;
pthread_mutex_init(&sIdx->mtx, NULL);
@ -80,30 +84,29 @@ void indexClose(SIndex *sIdx) {
return;
}
int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) {
int indexPut(SIndex *index, SIndexMultiTerm *fVals, uint64_t uid) {
#ifdef USE_LUCENE
index_document_t *doc = index_document_create();
index_document_t *doc = index_document_create();
char buf[16] = {0};
sprintf(buf, "%d", uid);
char buf[16] = {0};
sprintf(buf, "%d", uid);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1);
}
index_document_add(doc, NULL, 0, buf, strlen(buf), 0);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
index_document_add(doc, (const char *)(p->key), p->nKey, (const char *)(p->val), p->nVal, 1);
}
index_document_add(doc, NULL, 0, buf, strlen(buf), 0);
index_put(index->index, doc);
index_document_destroy(doc);
index_put(index->index, doc);
index_document_destroy(doc);
#endif
#ifdef USE_INVERTED_INDEX
//TODO(yihao): reduce the lock range
// TODO(yihao): reduce the lock range
pthread_mutex_lock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
SIndexTerm * p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
if (fi == NULL) {
SIdxColInfo tfi = {.colId = index->colId};
@ -111,18 +114,18 @@ int indexPut(SIndex *index, SIndexMultiTerm * fVals, uint64_t uid) {
index->colId++;
taosHashPut(index->colObj, p->colName, p->nColName, &tfi, sizeof(tfi));
} else {
//TODO, del
// TODO, del
}
}
pthread_mutex_unlock(&index->mtx);
for (int i = 0; i < taosArrayGetSize(fVals); i++) {
SIndexTerm *p = taosArrayGetP(fVals, i);
SIndexTerm * p = taosArrayGetP(fVals, i);
SIdxColInfo *fi = taosHashGet(index->colObj, p->colName, p->nColName);
assert(fi != NULL);
assert(fi != NULL);
int32_t colId = fi->colId;
int32_t version = index->cVersion;
int ret = indexCachePut(index->cache, p, colId, version, uid);
int ret = indexCachePut(index->cache, p, colId, version, uid);
if (ret != 0) {
return ret;
}
@ -135,24 +138,24 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
#ifdef USE_LUCENE
EIndexOperatorType opera = multiQuerys->opera;
int nQuery = taosArrayGetSize(multiQuerys->query);
int nQuery = taosArrayGetSize(multiQuerys->query);
char **fields = malloc(sizeof(char *) * nQuery);
char **keys = malloc(sizeof(char *) * nQuery);
int *types = malloc(sizeof(int) * nQuery);
char **keys = malloc(sizeof(char *) * nQuery);
int * types = malloc(sizeof(int) * nQuery);
for (int i = 0; i < nQuery; i++) {
SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i);
SIndexTerm *term = p->field_value;
SIndexTermQuery *p = taosArrayGet(multiQuerys->query, i);
SIndexTerm * term = p->field_value;
fields[i] = calloc(1, term->nKey + 1);
keys[i] = calloc(1, term->nVal + 1);
fields[i] = calloc(1, term->nKey + 1);
keys[i] = calloc(1, term->nVal + 1);
memcpy(fields[i], term->key, term->nKey);
memcpy(keys[i], term->val, term->nVal);
types[i] = (int)(p->type);
memcpy(fields[i], term->key, term->nKey);
memcpy(keys[i], term->val, term->nVal);
types[i] = (int)(p->type);
}
int *tResult = NULL;
int tsz= 0;
int tsz = 0;
index_multi_search(index->index, (const char **)fields, (const char **)keys, types, nQuery, opera, &tResult, &tsz);
for (int i = 0; i < tsz; i++) {
@ -172,12 +175,12 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
EIndexOperatorType opera = multiQuerys->opera; // relation of querys
SArray *interResults = taosArrayInit(4, POINTER_BYTES);
int nQuery = taosArrayGetSize(multiQuerys->query);
int nQuery = taosArrayGetSize(multiQuerys->query);
for (size_t i = 0; i < nQuery; i++) {
SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i);
SArray *tResult = NULL;
indexTermSearch(index, qTerm, &tResult);
taosArrayPush(interResults, (void *)&tResult);
SIndexTermQuery *qTerm = taosArrayGet(multiQuerys->query, i);
SArray * tResult = NULL;
indexTermSearch(index, qTerm, &tResult);
taosArrayPush(interResults, (void *)&tResult);
}
indexMergeFinalResults(interResults, opera, result);
indexInterResultsDestroy(interResults);
@ -186,39 +189,37 @@ int indexSearch(SIndex *index, SIndexMultiTermQuery *multiQuerys, SArray *result
return 1;
}
int indexDelete(SIndex *index, SIndexMultiTermQuery *query) {
#ifdef USE_INVERTED_INDEX
#endif
return 1;
}
int indexRebuild(SIndex *index, SIndexOpts *opts) {
int indexRebuild(SIndex *index, SIndexOpts *opts){
#ifdef USE_INVERTED_INDEX
#endif
}
SIndexOpts *indexOptsCreate() {
#ifdef USE_LUCENE
#endif
return NULL;
return NULL;
}
void indexOptsDestroy(SIndexOpts *opts) {
void indexOptsDestroy(SIndexOpts *opts){
#ifdef USE_LUCENE
#endif
}
/*
* @param: oper
*
*/
} /*
* @param: oper
*
*/
SIndexMultiTermQuery *indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery *p = (SIndexMultiTermQuery *)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { return NULL; }
p->opera = opera;
if (p == NULL) {
return NULL;
}
p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p;
}
@ -230,22 +231,24 @@ void indexMultiTermQueryDestroy(SIndexMultiTermQuery *pQuery) {
taosArrayDestroy(pQuery->query);
free(pQuery);
};
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType){
int indexMultiTermQueryAdd(SIndexMultiTermQuery *pQuery, SIndexTerm *term, EIndexQueryType qType) {
SIndexTermQuery q = {.qType = qType, .term = term};
taosArrayPush(pQuery->query, &q);
return 0;
}
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, int32_t nColName, const char *colVal, int32_t nColVal) {
SIndexTerm *indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName,
int32_t nColName, const char *colVal, int32_t nColVal) {
SIndexTerm *t = (SIndexTerm *)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { return NULL; }
if (t == NULL) {
return NULL;
}
t->suid = suid;
t->operType= oper;
t->suid = suid;
t->operType = oper;
t->colType = colType;
t->colName = (char *)calloc(1, nColName + 1);
t->colName = (char *)calloc(1, nColName + 1);
memcpy(t->colName, colName, nColName);
t->nColName = nColName;
@ -260,13 +263,11 @@ void indexTermDestroy(SIndexTerm *p) {
free(p);
}
SIndexMultiTerm *indexMultiTermCreate() {
return taosArrayInit(4, sizeof(SIndexTerm *));
}
SIndexMultiTerm *indexMultiTermCreate() { return taosArrayInit(4, sizeof(SIndexTerm *)); }
int indexMultiTermAdd(SIndexMultiTerm *terms, SIndexTerm *term) {
taosArrayPush(terms, &term);
return 0;
taosArrayPush(terms, &term);
return 0;
}
void indexMultiTermDestroy(SIndexMultiTerm *terms) {
for (int32_t i = 0; i < taosArrayGetSize(terms); i++) {
@ -277,29 +278,29 @@ void indexMultiTermDestroy(SIndexMultiTerm *terms) {
}
void indexInit() {
//do nothing
// do nothing
}
static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result) {
int32_t version = -1;
int16_t colId = -1;
int32_t version = -1;
int16_t colId = -1;
SIdxColInfo *colInfo = NULL;
SIndexTerm *term = query->term;
const char *colName = term->colName;
int32_t nColName = term->nColName;
int32_t nColName = term->nColName;
pthread_mutex_lock(&sIdx->mtx);
colInfo = taosHashGet(sIdx->colObj, colName, nColName);
colInfo = taosHashGet(sIdx->colObj, colName, nColName);
if (colInfo == NULL) {
pthread_mutex_unlock(&sIdx->mtx);
return -1;
}
colId = colInfo->colId;
colId = colInfo->colId;
version = colInfo->cVersion;
pthread_mutex_unlock(&sIdx->mtx);
*result = taosArrayInit(4, sizeof(uint64_t));
//TODO: iterator mem and tidex
// TODO: iterator mem and tidex
STermValueType s;
if (0 == indexCacheSearch(sIdx->cache, query, colId, version, *result, &s)) {
if (s == kTypeDeletion) {
@ -308,8 +309,8 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result
return 0;
} else {
if (0 != indexTFileSearch(sIdx->tindex, query, *result)) {
indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal);
return -1;
indexError("corrupt at index(TFile) col:%s val: %s", term->colName, term->colVal);
return -1;
}
}
} else {
@ -319,7 +320,9 @@ static int indexTermSearch(SIndex *sIdx, SIndexTermQuery *query, SArray **result
return 0;
}
static void indexInterResultsDestroy(SArray *results) {
if (results == NULL) { return; }
if (results == NULL) {
return;
}
size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) {
@ -327,19 +330,22 @@ static void indexInterResultsDestroy(SArray *results) {
taosArrayDestroy(p);
}
taosArrayDestroy(results);
}
static int indexMergeFinalResults(SArray *interResults, EIndexOperatorType oType, SArray *fResults) {
//refactor, merge interResults into fResults by oType
// refactor, merge interResults into fResults by oType
SArray *first = taosArrayGetP(interResults, 0);
taosArraySort(first, uidCompare);
taosArrayRemoveDuplicate(first, uidCompare, NULL);
if (oType == MUST) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
} else if (oType == SHOULD) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
// tag1 condistion || tag2 condition
} else if (oType == NOT) {
// just one column index, enhance later
taosArrayAddAll(fResults, first);
// not use currently
}
return 0;

94
source/libs/index/src/index_cache.c
View File

@ -14,16 +14,17 @@
*/
#include "index_cache.h"
#include "index_util.h"
#include "tcompare.h"
#define MAX_INDEX_KEY_LEN 256// test only, change later
#define MAX_INDEX_KEY_LEN 256 // test only, change later
// ref index_cache.h:22
#define CACHE_KEY_LEN(p) (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType))
#define CACHE_KEY_LEN(p) \
(sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + \
sizeof(p->operType))
static char* getIndexKey(const void *pData) {
return NULL;
}
static char * getIndexKey(const void *pData) { return NULL; }
static int32_t compareKey(const void *l, const void *r) {
char *lp = (char *)l;
char *rp = (char *)r;
@ -36,11 +37,11 @@ static int32_t compareKey(const void *l, const void *r) {
rp += sizeof(int32_t);
// compare field id
int16_t lf, rf; // field id
int16_t lf, rf; // field id
memcpy(&lf, lp, sizeof(lf));
memcpy(&rf, rp, sizeof(rf));
if (lf != rf) {
return lf < rf ? -1: 1;
return lf < rf ? -1 : 1;
}
lp += sizeof(lf);
rp += sizeof(rf);
@ -63,11 +64,17 @@ static int32_t compareKey(const void *l, const void *r) {
// compare value
int32_t i, j;
for (i = 0, j = 0; i < lfl && j < rfl; i++, j++) {
if (lp[i] == rp[j]) { continue; }
else { return lp[i] < rp[j] ? -1 : 1;}
if (lp[i] == rp[j]) {
continue;
} else {
return lp[i] < rp[j] ? -1 : 1;
}
}
if (i < lfl) {
return 1;
} else if (j < rfl) {
return -1;
}
if (i < lfl) { return 1;}
else if (j < rfl) { return -1; }
lp += lfl;
rp += rfl;
@ -90,84 +97,71 @@ static int32_t compareKey(const void *l, const void *r) {
// not care item type
return 0;
}
IndexCache *indexCacheCreate() {
IndexCache *cache = calloc(1, sizeof(IndexCache));
cache->skiplist = tSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
cache->skiplist = tSkipListCreate(
MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_BINARY, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
return cache;
}
void indexCacheDestroy(void *cache) {
IndexCache *pCache = cache;
if (pCache == NULL) { return; }
if (pCache == NULL) {
return;
}
tSkipListDestroy(pCache->skiplist);
free(pCache);
}
int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, uint64_t uid) {
if (cache == NULL) { return -1;}
if (cache == NULL) {
return -1;
}
IndexCache *pCache = cache;
// encode data
int32_t total = CACHE_KEY_LEN(term);
char * buf = calloc(1, total);
char * p = buf;
char *buf = calloc(1, total);
char *p = buf;
SERIALIZE_VAR_TO_BUF(p, total, int32_t);
SERIALIZE_VAR_TO_BUF(p, colId, int16_t);
memcpy(p, &total, sizeof(total));
p += sizeof(total);
SERIALIZE_MEM_TO_BUF(p, term, colType);
SERIALIZE_MEM_TO_BUF(p, term, nColVal);
SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal);
memcpy(p, &colId, sizeof(colId));
p += sizeof(colId);
SERIALIZE_VAR_TO_BUF(p, version, int32_t);
SERIALIZE_VAR_TO_BUF(p, uid, uint64_t);
memcpy(p, &term->colType, sizeof(term->colType));
p += sizeof(term->colType);
memcpy(p, &term->nColVal, sizeof(term->nColVal));
p += sizeof(term->nColVal);
memcpy(p, term->colVal, term->nColVal);
p += term->nColVal;
memcpy(p, &version, sizeof(version));
p += sizeof(version);
memcpy(p, &uid, sizeof(uid));
p += sizeof(uid);
memcpy(p, &term->operType, sizeof(term->operType));
p += sizeof(term->operType);
SERIALIZE_MEM_TO_BUF(p, term, operType);
tSkipListPut(pCache->skiplist, (void *)buf);
return 0;
// encode end
}
int indexCacheDel(void *cache, int32_t fieldId, const char *fieldValue, int32_t fvlen, uint64_t uid, int8_t operType) {
IndexCache *pCache = cache;
return 0;
}
int indexCacheSearch(void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) {
if (cache == NULL) { return -1; }
IndexCache *pCache = cache;
SIndexTerm *term = query->term;
EIndexQueryType qtype = query->qType;
int indexCacheSearch(
void *cache, SIndexTermQuery *query, int16_t colId, int32_t version, SArray *result, STermValueType *s) {
if (cache == NULL) {
return -1;
}
IndexCache * pCache = cache;
SIndexTerm * term = query->term;
EIndexQueryType qtype = query->qType;
int32_t keyLen = CACHE_KEY_LEN(term);
char *buf = calloc(1, keyLen);
if (qtype == QUERY_TERM) {
} else if (qtype == QUERY_PREFIX) {
} else if (qtype == QUERY_SUFFIX) {
} else if (qtype == QUERY_REGEX) {
}
return 0;
}

Some files were not shown because too many files have changed in this diff Show More