xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/TD-11463-3.0

This commit is contained in:
Cary Xu 2022-03-09 18:33:47 +08:00
parent a455105de3 f3d39e08e9
commit acffaea365
72 changed files with 2727 additions and 1029 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -10,3 +10,6 @@
[submodule "deps/TSZ"]
path = deps/TSZ
url = https://github.com/taosdata/TSZ.git
[submodule "examples/rust"]
path = examples/rust
url = https://github.com/songtianyi/tdengine-rust-bindings.git

1
examples/rust Submodule

@ -0,0 +1 @@
Subproject commit 1c8924dc668e6aa848214c2fc54e3ace3f5bf8df

6
include/common/tcommon.h
View File

@ -135,7 +135,7 @@ static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock)
return (void*)buf;
}
static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) {
static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) {
int32_t tlen = 0;
int32_t sz = 0;
tlen += taosEncodeFixedI64(buf, pRsp->consumerId);
@ -156,7 +156,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp
return tlen;
}
static FORCE_INLINE void* tDecodeSMqConsumeRsp(void* buf, SMqConsumeRsp* pRsp) {
static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) {
int32_t sz;
buf = taosDecodeFixedI64(buf, &pRsp->consumerId);
buf = taosDecodeFixedI64(buf, &pRsp->reqOffset);
@ -194,7 +194,7 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) {
// tfree(pBlock);
}
static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) {
static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) {
if (pRsp->schemas) {
if (pRsp->schemas->nCols) {
tfree(pRsp->schemas->pSchema);

12
include/common/tmsg.h
View File

@ -795,7 +795,7 @@ typedef struct SVgroupInfo {
int32_t vgId;
uint32_t hashBegin;
uint32_t hashEnd;
SEpSet epset;
SEpSet epSet;
} SVgroupInfo;
typedef struct {
@ -1876,8 +1876,8 @@ typedef struct {
} SVCreateTSmaReq;
typedef struct {
int8_t type; // 0 status report, 1 update data
char indexName[TSDB_INDEX_NAME_LEN + 1]; //
int8_t type; // 0 status report, 1 update data
char indexName[TSDB_INDEX_NAME_LEN + 1]; //
STimeWindow windows;
} STSmaMsg;
@ -2073,7 +2073,7 @@ typedef struct {
int32_t skipLogNum;
int32_t numOfTopics;
SArray* pBlockData; // SArray<SSDataBlock>
} SMqConsumeRsp;
} SMqPollRsp;
// one req for one vg+topic
typedef struct {
@ -2086,7 +2086,7 @@ typedef struct {
int64_t currentOffset;
char topic[TSDB_TOPIC_FNAME_LEN];
} SMqConsumeReq;
} SMqPollReq;
typedef struct {
int32_t vgId;
@ -2108,7 +2108,7 @@ typedef struct {
struct tmq_message_t {
SMqRspHead head;
union {
SMqConsumeRsp consumeRsp;
SMqPollRsp consumeRsp;
SMqCMGetSubEpRsp getEpRsp;
};
void* extra;

141
include/libs/qcom/query.h
View File

@ -44,10 +44,10 @@ enum {
};
typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema
uint8_t precision; // the number of precision
int16_t numOfColumns; // the number of columns
int32_t rowSize; // row size of the schema
uint8_t numOfTags; // the number of tags in schema
uint8_t precision; // the number of precision
int16_t numOfColumns; // the number of columns
int32_t rowSize; // row size of the schema
} STableComInfo;
/*
@ -56,49 +56,45 @@ typedef struct STableComInfo {
* The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info.
*/
typedef struct SCTableMeta {
int32_t vgId:24;
int32_t vgId : 24;
int8_t tableType;
uint64_t uid;
uint64_t suid;
} SCTableMeta;
/*
* Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta.
* Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a
* SCTableMeta.
*/
typedef struct STableMeta {
//BEGIN: KEEP THIS PART SAME WITH SCTableMeta
int32_t vgId:24;
int8_t tableType;
uint64_t uid;
uint64_t suid;
//END: KEEP THIS PART SAME WITH SCTableMeta
// BEGIN: KEEP THIS PART SAME WITH SCTableMeta
int32_t vgId : 24;
int8_t tableType;
uint64_t uid;
uint64_t suid;
// END: KEEP THIS PART SAME WITH SCTableMeta
// if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info
int16_t sversion;
int16_t tversion;
STableComInfo tableInfo;
SSchema schema[];
// if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta
// info
int16_t sversion;
int16_t tversion;
STableComInfo tableInfo;
SSchema schema[];
} STableMeta;
typedef struct SDBVgInfo {
int32_t vgVersion;
int8_t hashMethod;
SHashObj *vgHash; //key:vgId, value:SVgroupInfo
SHashObj* vgHash; // key:vgId, value:SVgroupInfo
} SDBVgInfo;
typedef struct SUseDbOutput {
char db[TSDB_DB_FNAME_LEN];
uint64_t dbId;
SDBVgInfo *dbVgroup;
char db[TSDB_DB_FNAME_LEN];
uint64_t dbId;
SDBVgInfo* dbVgroup;
} SUseDbOutput;
enum {
META_TYPE_NULL_TABLE = 1,
META_TYPE_CTABLE,
META_TYPE_TABLE,
META_TYPE_BOTH_TABLE
};
enum { META_TYPE_NULL_TABLE = 1, META_TYPE_CTABLE, META_TYPE_TABLE, META_TYPE_BOTH_TABLE };
typedef struct STableMetaOutput {
int32_t metaType;
@ -107,30 +103,30 @@ typedef struct STableMetaOutput {
char ctbName[TSDB_TABLE_NAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
SCTableMeta ctbMeta;
STableMeta *tbMeta;
STableMeta* tbMeta;
} STableMetaOutput;
typedef struct SDataBuf {
void *pData;
uint32_t len;
void *handle;
void* pData;
uint32_t len;
void* handle;
} SDataBuf;
typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32_t code);
typedef int32_t (*__async_exec_fn_t)(void* param);
typedef struct SMsgSendInfo {
__async_send_cb_fn_t fp; //async callback function
void *param;
uint64_t requestId;
uint64_t requestObjRefId;
int32_t msgType;
SDataBuf msgInfo;
__async_send_cb_fn_t fp; // async callback function
void* param;
uint64_t requestId;
uint64_t requestObjRefId;
int32_t msgType;
SDataBuf msgInfo;
} SMsgSendInfo;
typedef struct SQueryNodeAddr {
int32_t nodeId; // vgId or qnodeId
SEpSet epset;
SEpSet epSet;
} SQueryNodeAddr;
int32_t initTaskQueue();
@ -154,32 +150,67 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code);
* @param pInfo
* @return
*/
int32_t asyncSendMsgToServer(void *pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransporterId, const SMsgSendInfo* pInfo);
int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp);
int32_t queryBuildUseDbOutput(SUseDbOutput* pOut, SUseDbRsp* usedbRsp);
void initQueryModuleMsgHandle();
const SSchema* tGetTbnameColumnSchema();
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta);
int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen);
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize);
extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen);
extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize);
#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE
#define SET_META_TYPE_CTABLE(t) (t) = META_TYPE_CTABLE
#define SET_META_TYPE_TABLE(t) (t) = META_TYPE_TABLE
#define SET_META_TYPE_BOTH_TABLE(t) (t) = META_TYPE_BOTH_TABLE
#define qFatal(...) do { if (qDebugFlag & DEBUG_FATAL) { taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); }} while(0)
#define qError(...) do { if (qDebugFlag & DEBUG_ERROR) { taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); }} while(0)
#define qWarn(...) do { if (qDebugFlag & DEBUG_WARN) { taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); }} while(0)
#define qInfo(...) do { if (qDebugFlag & DEBUG_INFO) { taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); }} while(0)
#define qDebugL(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); }} while(0)
#define qFatal(...) \
do { \
if (qDebugFlag & DEBUG_FATAL) { \
taosPrintLog("QRY FATAL ", DEBUG_FATAL, qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qError(...) \
do { \
if (qDebugFlag & DEBUG_ERROR) { \
taosPrintLog("QRY ERROR ", DEBUG_ERROR, qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qWarn(...) \
do { \
if (qDebugFlag & DEBUG_WARN) { \
taosPrintLog("QRY WARN ", DEBUG_WARN, qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qInfo(...) \
do { \
if (qDebugFlag & DEBUG_INFO) { \
taosPrintLog("QRY ", DEBUG_INFO, qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qDebug(...) \
do { \
if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qTrace(...) \
do { \
if (qDebugFlag & DEBUG_TRACE) { \
taosPrintLog("QRY ", DEBUG_TRACE, qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#define qDebugL(...) \
do { \
if (qDebugFlag & DEBUG_DEBUG) { \
taosPrintLongString("QRY ", DEBUG_DEBUG, qDebugFlag, __VA_ARGS__); \
} \
} while (0)
#ifdef __cplusplus
}

7
include/libs/sync/sync.h
View File

@ -31,9 +31,9 @@ typedef int64_t SyncIndex;
typedef uint64_t SyncTerm;
typedef enum {
TAOS_SYNC_STATE_FOLLOWER = 0,
TAOS_SYNC_STATE_CANDIDATE = 1,
TAOS_SYNC_STATE_LEADER = 2,
TAOS_SYNC_STATE_FOLLOWER = 100,
TAOS_SYNC_STATE_CANDIDATE = 101,
TAOS_SYNC_STATE_LEADER = 102,
} ESyncState;
typedef struct SSyncBuffer {
@ -134,6 +134,7 @@ typedef struct SSyncInfo {
SyncGroupId vgId;
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
char walPath[TSDB_FILENAME_LEN];
SSyncFSM* pFsm;
void* rpcClient;

22
include/os/osDir.h
View File

@ -16,10 +16,24 @@
#ifndef _TD_OS_DIR_H_
#define _TD_OS_DIR_H_
// If the error is in a third-party library, place this header file under the third-party library header file.
#ifndef ALLOW_FORBID_FUNC
#define opendir OPENDIR_FUNC_TAOS_FORBID
#define readdir READDIR_FUNC_TAOS_FORBID
#define closedir CLOSEDIR_FUNC_TAOS_FORBID
#define dirname DIRNAME_FUNC_TAOS_FORBID
#undef basename
#define basename BASENAME_FUNC_TAOS_FORBID
#endif
#ifdef __cplusplus
extern "C" {
#endif
typedef struct TdDir *TdDirPtr;
typedef struct TdDirEntry *TdDirEntryPtr;
void taosRemoveDir(const char *dirname);
bool taosDirExist(char *dirname);
int32_t taosMkDir(const char *dirname);
@ -27,6 +41,14 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays);
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen);
int32_t taosRealPath(char *dirname, int32_t maxlen);
bool taosIsDir(const char *dirname);
char* taosDirName(char *dirname);
char* taosDirEntryBaseName(char *dirname);
TdDirPtr taosOpenDir(const char *dirname);
TdDirEntryPtr taosReadDir(TdDirPtr pDir);
bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry);
char* taosGetDirEntryName(TdDirEntryPtr pDirEntry);
int32_t taosCloseDir(TdDirPtr pDir);
#ifdef __cplusplus
}

85
include/os/osSocket.h
View File

@ -21,7 +21,10 @@
#define socket SOCKET_FUNC_TAOS_FORBID
#define bind BIND_FUNC_TAOS_FORBID
#define listen LISTEN_FUNC_TAOS_FORBID
// #define accept ACCEPT_FUNC_TAOS_FORBID
#define accept ACCEPT_FUNC_TAOS_FORBID
#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID
#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID
#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID
#endif
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
@ -38,31 +41,6 @@
extern "C" {
#endif
#define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SOCKET;
typedef SOCKET EpollFd;
#define EpollClose(pollFd) taosCloseSocket(pollFd)
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
typedef SOCKET SocketFd;
#else
typedef int32_t SocketFd;
#endif
int32_t taosSendto(SocketFd fd, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
int32_t taosWriteSocket(SocketFd fd, void *msg, int len);
int32_t taosReadSocket(SocketFd fd, void *msg, int len);
int32_t taosCloseSocketNoCheck(SocketFd fd);
int32_t taosCloseSocket(SocketFd fd);
void taosShutDownSocketRD(SOCKET fd);
void taosShutDownSocketWR(SOCKET fd);
int32_t taosSetNonblocking(SOCKET sock, int32_t on);
int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);
int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t *optlen);
uint32_t taosInetAddr(const char *ipAddr);
const char *taosInetNtoa(struct in_addr ipInt);
#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32))
#define htobe64 htonll
#if defined(_TD_GO_DLL_)
@ -74,25 +52,54 @@ const char *taosInetNtoa(struct in_addr ipInt);
#define htobe64 htonll
#endif
int32_t taosReadn(SOCKET sock, char *buffer, int32_t len);
int32_t taosWriteMsg(SOCKET fd, void *ptr, int32_t nbytes);
int32_t taosReadMsg(SOCKET fd, void *ptr, int32_t nbytes);
int32_t taosNonblockwrite(SOCKET fd, char *ptr, int32_t nbytes);
int64_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len);
int32_t taosSetNonblocking(SOCKET sock, int32_t on);
#define TAOS_EPOLL_WAIT_TIME 500
SOCKET taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
SOCKET taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int32_t taosKeepTcpAlive(SOCKET sockFd);
typedef struct TdSocketServer *TdSocketServerPtr;
typedef struct TdSocket *TdSocketPtr;
typedef struct TdEpoll *TdEpollPtr;
void taosBlockSIGPIPE();
int32_t taosSendto(TdSocketPtr pSocket, void * msg, int len, unsigned int flags, const struct sockaddr * to, int tolen);
int32_t taosWriteSocket(TdSocketPtr pSocket, void *msg, int len);
int32_t taosReadSocket(TdSocketPtr pSocket, void *msg, int len);
int32_t taosReadFromSocket(TdSocketPtr pSocket, void *buf, int32_t len, int32_t flags, struct sockaddr *destAddr, socklen_t *addrLen);
int32_t taosCloseSocket(TdSocketPtr *ppSocket);
int32_t taosCloseSocketServer(TdSocketServerPtr *ppSocketServer);
int32_t taosShutDownSocketRD(TdSocketPtr pSocket);
int32_t taosShutDownSocketServerRD(TdSocketServerPtr pSocketServer);
int32_t taosShutDownSocketWR(TdSocketPtr pSocket);
int32_t taosShutDownSocketServerWR(TdSocketServerPtr pSocketServer);
int32_t taosShutDownSocketRDWR(TdSocketPtr pSocket);
int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer);
int32_t taosSetNonblocking(TdSocketPtr pSocket, int32_t on);
int32_t taosSetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t optlen);
int32_t taosGetSockOpt(TdSocketPtr pSocket, int32_t level, int32_t optname, void *optval, int32_t *optlen);
int32_t taosWriteMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosReadMsg(TdSocketPtr pSocket, void *ptr, int32_t nbytes);
int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes);
int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len);
TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort);
TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp);
TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port);
int32_t taosKeepTcpAlive(TdSocketPtr pSocket);
TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, socklen_t *addrLen);
int32_t taosGetSocketName(TdSocketPtr pSocket,struct sockaddr *destAddr, socklen_t *addrLen);
void taosBlockSIGPIPE();
uint32_t taosGetIpv4FromFqdn(const char *);
int32_t taosGetFqdn(char *);
void tinet_ntoa(char *ipstr, uint32_t ip);
uint32_t ip2uint(const char *const ip_addr);
void taosIgnSIGPIPE();
void taosSetMaskSIGPIPE();
void taosIgnSIGPIPE();
void taosSetMaskSIGPIPE();
uint32_t taosInetAddr(const char *ipAddr);
const char *taosInetNtoa(struct in_addr ipInt);
TdEpollPtr taosCreateEpoll(int32_t size);
int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
#ifdef __cplusplus
}

1
include/os/osSysinfo.h
View File

@ -33,6 +33,7 @@ typedef struct {
SDiskSize size;
} SDiskSpace;
bool taosCheckSystemIsSmallEnd();
void taosGetSystemInfo();
int32_t taosGetEmail(char *email, int32_t maxLen);
int32_t taosGetOsReleaseName(char *releaseName, int32_t maxLen);

4
source/client/src/clientImpl.c
View File

@ -517,7 +517,7 @@ void* doFetchRow(SRequestObj* pRequest) {
SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
SVgroupInfo* pVgroupInfo = taosArrayGet(pShowReqInfo->pArray, pShowReqInfo->currentIndex);
epSet = pVgroupInfo->epset;
epSet = pVgroupInfo->epSet;
} else if (pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) {
pRequest->type = TDMT_VND_SHOW_TABLES;
SShowReqInfo* pShowReqInfo = &pRequest->body.showInfo;
@ -534,7 +534,7 @@ void* doFetchRow(SRequestObj* pRequest) {
pRequest->body.requestMsg.pData = pShowReq;
SMsgSendInfo* body = buildMsgInfoImpl(pRequest);
epSet = pVgroupInfo->epset;
epSet = pVgroupInfo->epSet;
int64_t transporterId = 0;
STscObj* pTscObj = pRequest->pTscObj;

30
source/client/src/tmq.c
View File

@ -13,8 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "clientInt.h"
#include "clientLog.h"
#include "parser.h"
@ -606,17 +604,17 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return 0;
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
SMqPollRsp* pRsp = &tmq_message->consumeRsp;
return pRsp->skipLogNum;
}
void tmqShowMsg(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return;
static bool noPrintSchema;
char pBuf[128];
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
int32_t colNum = pRsp->schemas->nCols;
static bool noPrintSchema;
char pBuf[128];
SMqPollRsp* pRsp = &tmq_message->consumeRsp;
int32_t colNum = pRsp->schemas->nCols;
if (!noPrintSchema) {
printf("|");
for (int32_t i = 0; i < colNum; i++) {
@ -703,7 +701,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) {
goto WRITE_QUEUE_FAIL;
}
memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead));
tDecodeSMqConsumeRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp);
/*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/
if (pRsp->consumeRsp.numOfTopics == 0) {
/*printf("no data\n");*/
@ -874,7 +872,7 @@ tmq_resp_err_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) {
return TMQ_RESP_ERR__FAIL;
}
SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTopic* pTopic, SMqClientVg* pVg) {
int64_t reqOffset;
if (pVg->currentOffset >= 0) {
reqOffset = pVg->currentOffset;
@ -886,7 +884,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien
reqOffset = tmq->resetOffsetCfg;
}
SMqConsumeReq* pReq = malloc(sizeof(SMqConsumeReq));
SMqPollReq* pReq = malloc(sizeof(SMqPollReq));
if (pReq == NULL) {
return NULL;
}
@ -900,7 +898,7 @@ SMqConsumeReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClien
pReq->currentOffset = reqOffset;
pReq->head.vgId = htonl(pVg->vgId);
pReq->head.contLen = htonl(sizeof(SMqConsumeReq));
pReq->head.contLen = htonl(sizeof(SMqPollReq));
return pReq;
}
@ -914,7 +912,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
/*if (vgStatus != TMQ_VG_STATUS__IDLE) {*/
/*continue;*/
/*}*/
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
// TODO: out of mem
@ -941,7 +939,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) {
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.len = sizeof(SMqPollReq),
.handle = NULL,
};
sendInfo->requestId = generateRequestId();
@ -982,7 +980,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
if (vgStatus != TMQ_VG_STATUS__IDLE) {
continue;
}
SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, blockingTime, pTopic, pVg);
if (pReq == NULL) {
atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE);
tsem_post(&tmq->rspSem);
@ -1011,7 +1009,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) {
sendInfo->msgInfo = (SDataBuf){
.pData = pReq,
.len = sizeof(SMqConsumeReq),
.len = sizeof(SMqPollReq),
.handle = NULL,
};
sendInfo->requestId = generateRequestId();
@ -1271,7 +1269,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v
void tmq_message_destroy(tmq_message_t* tmq_message) {
if (tmq_message == NULL) return;
SMqConsumeRsp* pRsp = &tmq_message->consumeRsp;
SMqPollRsp* pRsp = &tmq_message->consumeRsp;
tDeleteSMqConsumeRsp(pRsp);
/*free(tmq_message);*/
taosFreeQitem(tmq_message);

4
source/common/src/tmsg.c
View File

@ -1481,7 +1481,7 @@ static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) {
if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1;
if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1;
}
return 0;
@ -1541,7 +1541,7 @@ int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) {
if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1;
if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1;
if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &vgInfo.epSet) < 0) return -1;
taosArrayPush(pRsp->pVgroupInfos, &vgInfo);
}

5
source/dnode/mgmt/daemon/src/dmnMain.c
View File

@ -97,6 +97,11 @@ int32_t dmnRunDnode() {
}
int main(int argc, char const *argv[]) {
if (!taosCheckSystemIsSmallEnd()) {
uError("TDengine does not run on non-small-end machines.");
return -1;
}
if (dmnParseOption(argc, argv) != 0) {
return -1;
}

6
source/dnode/mnode/impl/src/mndDb.c
View File

@ -900,10 +900,10 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
vgInfo.vgId = pVgroup->vgId;
vgInfo.hashBegin = pVgroup->hashBegin;
vgInfo.hashEnd = pVgroup->hashEnd;
vgInfo.epset.numOfEps = pVgroup->replica;
vgInfo.epSet.numOfEps = pVgroup->replica;
for (int32_t gid = 0; gid < pVgroup->replica; ++gid) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[gid];
SEp *pEp = &vgInfo.epset.eps[gid];
SEp *pEp = &vgInfo.epSet.eps[gid];
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
if (pDnode != NULL) {
memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
@ -911,7 +911,7 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
}
mndReleaseDnode(pMnode, pDnode);
if (pVgid->role == TAOS_SYNC_STATE_LEADER) {
vgInfo.epset.inUse = gid;
vgInfo.epSet.inUse = gid;
}
}
vindex++;

31
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -33,23 +33,29 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
SSdb* pSdb = pMnode->pSdb;
SVgObj* pVgroup = NULL;
SQueryDag* pDag = qStringToDag(pTopic->physicalPlan);
SArray* pAray = NULL;
SArray* unassignedVg = pSub->unassignedVg;
if (pDag == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
ASSERT(pSub->vgNum == 0);
int32_t levelNum = taosArrayGetSize(pDag->pSubplans);
if (levelNum != 1) {
qDestroyQueryDag(pDag);
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
return -1;
}
SArray* inner = taosArrayGet(pDag->pSubplans, 0);
SArray* plans = taosArrayGetP(pDag->pSubplans, 0);
int32_t opNum = taosArrayGetSize(inner);
int32_t opNum = taosArrayGetSize(plans);
if (opNum != 1) {
qDestroyQueryDag(pDag);
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
return -1;
}
SSubplan* plan = taosArrayGetP(inner, 0);
SSubplan* plan = taosArrayGetP(plans, 0);
void* pIter = NULL;
while (1) {
@ -62,17 +68,24 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
pSub->vgNum++;
plan->execNode.nodeId = pVgroup->vgId;
plan->execNode.epset = mndGetVgroupEpset(pMnode, pVgroup);
plan->execNode.epSet = mndGetVgroupEpset(pMnode, pVgroup);
SMqConsumerEp consumerEp = {0};
consumerEp.status = 0;
consumerEp.consumerId = -1;
consumerEp.epSet = plan->execNode.epset;
consumerEp.epSet = plan->execNode.epSet;
consumerEp.vgId = plan->execNode.nodeId;
int32_t msgLen;
int32_t code = qSubPlanToString(plan, &consumerEp.qmsg, &msgLen);
taosArrayPush(unassignedVg, &consumerEp);
if (qSubPlanToString(plan, &consumerEp.qmsg, &msgLen) < 0) {
sdbRelease(pSdb, pVgroup);
qDestroyQueryDag(pDag);
terrno = TSDB_CODE_QRY_INVALID_INPUT;
return -1;
}
taosArrayPush(pSub->unassignedVg, &consumerEp);
}
qDestroyQueryDag(pDag);
return 0;
}

15
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -93,7 +93,6 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj
strcpy(pSub->key, key);
if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
terrno = TSDB_CODE_MND_UNSUPPORTED_TOPIC;
tDeleteSMqSubscribeObj(pSub);
free(pSub);
return NULL;
@ -295,7 +294,11 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) {
for (int32_t k = 0; k < vgsz; k++) {
char offsetKey[TSDB_PARTITION_KEY_LEN];
SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k);
SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId, .offset = -1};
SMqSubVgEp vgEp = {
.epSet = pConsumerEp->epSet,
.vgId = pConsumerEp->vgId,
.offset = -1,
};
mndMakePartitionKey(offsetKey, pConsumer->cgroup, topicName, pConsumerEp->vgId);
SMqOffsetObj *pOffsetObj = mndAcquireOffset(pMnode, offsetKey);
if (pOffsetObj != NULL) {
@ -345,7 +348,7 @@ static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
if (pRebSub == NULL) {
pRebSub = tNewSMqRebSubscribe(key);
if (pRebSub == NULL) {
// TODO
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
taosHashPut(pHash, key, strlen(key), pRebSub, sizeof(SMqRebSubscribe));
@ -412,7 +415,11 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
}
if (taosHashGetSize(pRebMsg->rebSubHash) != 0) {
mInfo("mq rebalance will be triggered");
SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg)};
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_MQ_DO_REBALANCE,
.pCont = pRebMsg,
.contLen = sizeof(SMqDoRebalanceMsg),
};
pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg);
} else {
taosHashCleanup(pRebMsg->rebSubHash);

6
source/dnode/mnode/impl/src/mnode.c
View File

@ -96,7 +96,11 @@ static void mndCalMqRebalance(void *param, void *tmrId) {
if (mndIsMaster(pMnode)) {
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen};
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_MQ_TIMER,
.pCont = pReq,
.contLen = contLen,
};
pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg);
}

12
source/dnode/mnode/impl/test/db/db.cpp
View File

@ -292,9 +292,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
EXPECT_GT(pInfo->vgId, 0);
EXPECT_EQ(pInfo->hashBegin, 0);
EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1);
EXPECT_EQ(pInfo->epset.inUse, 0);
EXPECT_EQ(pInfo->epset.numOfEps, 1);
SEp* pAddr = &pInfo->epset.eps[0];
EXPECT_EQ(pInfo->epSet.inUse, 0);
EXPECT_EQ(pInfo->epSet.numOfEps, 1);
SEp* pAddr = &pInfo->epSet.eps[0];
EXPECT_EQ(pAddr->port, 9030);
EXPECT_STREQ(pAddr->fqdn, "localhost");
}
@ -307,9 +307,9 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) {
EXPECT_GT(pInfo->vgId, 0);
EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2);
EXPECT_EQ(pInfo->hashEnd, UINT32_MAX);
EXPECT_EQ(pInfo->epset.inUse, 0);
EXPECT_EQ(pInfo->epset.numOfEps, 1);
SEp* pAddr = &pInfo->epset.eps[0];
EXPECT_EQ(pInfo->epSet.inUse, 0);
EXPECT_EQ(pInfo->epSet.numOfEps, 1);
SEp* pAddr = &pInfo->epSet.eps[0];
EXPECT_EQ(pAddr->port, 9030);
EXPECT_STREQ(pAddr->fqdn, "localhost");
}

14
source/dnode/vnode/inc/tq.h
View File

@ -13,16 +13,14 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_TQ_H_
#define _TD_TQ_H_
#ifndef _TQ_H_
#define _TQ_H_
#include "tcommon.h"
#include "executor.h"
#include "tmallocator.h"
#include "meta.h"
#include "scheduler.h"
#include "taoserror.h"
#include "tlist.h"
#include "tcommon.h"
#include "tmallocator.h"
#include "tmsg.h"
#include "trpc.h"
#include "ttimer.h"
@ -54,7 +52,7 @@ void tqClose(STQ*);
int tqPushMsg(STQ*, void* msg, tmsg_t msgType, int64_t version);
int tqCommit(STQ*);
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessSetConnReq(STQ* pTq, char* msg);
int32_t tqProcessRebReq(STQ* pTq, char* msg);
@ -62,4 +60,4 @@ int32_t tqProcessRebReq(STQ* pTq, char* msg);
}
#endif
#endif /*_TD_TQ_H_*/
#endif /*_TQ_H_*/

10
source/dnode/vnode/src/inc/tqInt.h
View File

@ -19,7 +19,7 @@
#include "meta.h"
#include "tlog.h"
#include "tq.h"
#include "trpc.h"
#include "tqPush.h"
#ifdef __cplusplus
extern "C" {
@ -31,30 +31,35 @@ extern "C" {
taosPrintLog("TQ FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
} \
}
#define tqError(...) \
{ \
if (tqDebugFlag & DEBUG_ERROR) { \
taosPrintLog("TQ ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
} \
}
#define tqWarn(...) \
{ \
if (tqDebugFlag & DEBUG_WARN) { \
taosPrintLog("TQ WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
} \
}
#define tqInfo(...) \
{ \
if (tqDebugFlag & DEBUG_INFO) { \
taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); \
} \
}
#define tqDebug(...) \
{ \
if (tqDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); \
} \
}
#define tqTrace(...) \
{ \
if (tqDebugFlag & DEBUG_TRACE) { \
@ -138,9 +143,7 @@ typedef struct {
// topics that are not connectted
STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile
TdFilePtr pFile;
// TODO:temporaral use, to be replaced by unified tfile
TdFilePtr pIdxFile;
char* dirPath;
@ -157,6 +160,7 @@ struct STQ {
STqCfg* tqConfig;
STqMemRef tqMemRef;
STqMetaStore* tqMeta;
STqPushMgr* tqPushMgr;
SWal* pWal;
SMeta* pVnodeMeta;
};

77
source/dnode/vnode/src/inc/tqPush.h Normal file
View File

@ -0,0 +1,77 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TQ_PUSH_H_
#define _TQ_PUSH_H_
#include "thash.h"
#include "trpc.h"
#include "ttimer.h"
#ifdef __cplusplus
extern "C" {
#endif
enum {
TQ_PUSHER_TYPE__CLIENT = 1,
TQ_PUSHER_TYPE__STREAM,
};
typedef struct {
int8_t type;
int8_t reserved[3];
int32_t ttl;
int64_t consumerId;
SRpcMsg* pMsg;
// SMqPollRsp* rsp;
} STqClientPusher;
typedef struct {
int8_t type;
int8_t nodeType;
int8_t reserved[6];
int64_t streamId;
SEpSet epSet;
} STqStreamPusher;
typedef struct {
int8_t type; // mq or stream
} STqPusher;
typedef struct {
SHashObj* pHash; // <id, STqPush*>
} STqPushMgr;
typedef struct {
int8_t inited;
tmr_h timer;
} STqPushMgmt;
static STqPushMgmt tqPushMgmt;
int32_t tqPushMgrInit();
void tqPushMgrCleanUp();
STqPushMgr* tqPushMgrOpen();
void tqPushMgrClose(STqPushMgr* pushMgr);
STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl);
STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet);
#ifdef __cplusplus
}
#endif
#endif /*_TQ_PUSH_H_*/

63
source/dnode/vnode/src/tq/tq.c
View File

@ -12,28 +12,16 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tcompare.h"
#include "tqInt.h"
#include "tqMetaStore.h"
int tqInit() {
int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1);
if (old == 1) return 0;
int32_t tqInit() { return tqPushMgrInit(); }
tqMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
return 0;
}
void tqCleanUp() { tqPushMgrCleanUp(); }
void tqCleanUp() {
int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 1, 0);
if (old == 0) return;
taosTmrStop(tqMgmt.timer);
taosTmrCleanUp(tqMgmt.timer);
}
STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
STQ* tqOpen(const char* path, SWal* pWal, SMeta* pVnodeMeta, STqCfg* tqConfig, SMemAllocatorFactory* allocFac) {
STQ* pTq = malloc(sizeof(STQ));
if (pTq == NULL) {
terrno = TSDB_CODE_TQ_OUT_OF_MEMORY;
@ -42,7 +30,7 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
pTq->path = strdup(path);
pTq->tqConfig = tqConfig;
pTq->pWal = pWal;
pTq->pVnodeMeta = pMeta;
pTq->pVnodeMeta = pVnodeMeta;
#if 0
pTq->tqMemRef.pAllocatorFactory = allocFac;
pTq->tqMemRef.pAllocator = allocFac->create(allocFac);
@ -60,6 +48,13 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl
return NULL;
}
pTq->tqPushMgr = tqPushMgrOpen();
if (pTq->tqPushMgr == NULL) {
// free store
free(pTq);
return NULL;
}
return pTq;
}
@ -72,6 +67,8 @@ void tqClose(STQ* pTq) {
}
int tqPushMsg(STQ* pTq, void* msg, tmsg_t msgType, int64_t version) {
// iterate hash
// process all msg
// if waiting
// memcpy and send msg to fetch thread
// TODO: add reference
@ -199,7 +196,10 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
for (int j = 0; j < TQ_BUFFER_SIZE; j++) {
pTopic->buffer.output[j].status = 0;
STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pVnodeMeta);
SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pVnodeMeta};
SReadHandle handle = {
.reader = pReadHandle,
.meta = pTq->pVnodeMeta,
};
pTopic->buffer.output[j].pReadHandle = pReadHandle;
pTopic->buffer.output[j].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle);
}
@ -208,11 +208,11 @@ int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsu
return 0;
}
int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SMqConsumeReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
int64_t fetchOffset;
int64_t blockingTime = pReq->blockingTime;
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) {
SMqPollReq* pReq = pMsg->pCont;
int64_t consumerId = pReq->consumerId;
int64_t fetchOffset;
int64_t blockingTime = pReq->blockingTime;
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
fetchOffset = 0;
@ -222,7 +222,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
fetchOffset = pReq->currentOffset + 1;
}
SMqConsumeRsp rsp = {
SMqPollRsp rsp = {
.consumerId = consumerId,
.numOfTopics = 0,
.pBlockData = NULL,
@ -236,6 +236,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
rpcSendResponse(pMsg);
return 0;
}
int sz = taosArrayGetSize(pConsumer->topics);
ASSERT(sz == 1);
STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0);
@ -247,13 +248,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
SWalHead* pHead;
while (1) {
int8_t pos = fetchOffset % TQ_BUFFER_SIZE;
/*if (fetchOffset > walGetLastVer(pTq->pWal) || walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {*/
if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) {
// TODO: no more log, set timer to wait blocking time
// if data inserted during waiting, launch query and
// response to user
break;
}
int8_t pos = fetchOffset % TQ_BUFFER_SIZE;
pHead = pTopic->pReadhandle->pHead;
if (pHead->head.msgType == TDMT_VND_SUBMIT) {
SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body;
@ -280,7 +282,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
rsp.numOfTopics = 1;
rsp.pBlockData = pRes;
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
@ -290,7 +292,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
((SMqRspHead*)buf)->epoch = pReq->epoch;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqConsumeRsp(&abuf, &rsp);
tEncodeSMqPollRsp(&abuf, &rsp);
taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);
pMsg->pCont = buf;
pMsg->contLen = tlen;
@ -304,7 +306,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
}
}
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqConsumeRsp(NULL, &rsp);
/*if (blockingTime != 0) {*/
/*tqAddClientPusher(pTq->tqPushMgr, pMsg, consumerId, blockingTime);*/
/*} else {*/
int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqPollRsp(NULL, &rsp);
void* buf = rpcMallocCont(tlen);
if (buf == NULL) {
pMsg->code = -1;
@ -314,12 +319,14 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) {
((SMqRspHead*)buf)->epoch = pReq->epoch;
void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead));
tEncodeSMqConsumeRsp(&abuf, &rsp);
tEncodeSMqPollRsp(&abuf, &rsp);
rsp.pBlockData = NULL;
pMsg->pCont = buf;
pMsg->contLen = tlen;
pMsg->code = 0;
rpcSendResponse(pMsg);
/*}*/
return 0;
}

84
source/dnode/vnode/src/tq/tqPush.c Normal file
View File

@ -0,0 +1,84 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tqPush.h"
int32_t tqPushMgrInit() {
//
int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 0, 1);
if (old == 1) return 0;
tqPushMgmt.timer = taosTmrInit(0, 0, 0, "TQ");
return 0;
}
void tqPushMgrCleanUp() {
int8_t old = atomic_val_compare_exchange_8(&tqPushMgmt.inited, 1, 0);
if (old == 0) return;
taosTmrStop(tqPushMgmt.timer);
taosTmrCleanUp(tqPushMgmt.timer);
}
STqPushMgr* tqPushMgrOpen() {
STqPushMgr* mgr = malloc(sizeof(STqPushMgr));
if (mgr == NULL) {
return NULL;
}
mgr->pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
return mgr;
}
void tqPushMgrClose(STqPushMgr* pushMgr) {
taosHashCleanup(pushMgr->pHash);
free(pushMgr);
}
STqClientPusher* tqAddClientPusher(STqPushMgr* pushMgr, SRpcMsg* pMsg, int64_t consumerId, int64_t ttl) {
STqClientPusher* clientPusher = malloc(sizeof(STqClientPusher));
if (clientPusher == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
clientPusher->type = TQ_PUSHER_TYPE__CLIENT;
clientPusher->pMsg = pMsg;
clientPusher->consumerId = consumerId;
clientPusher->ttl = ttl;
if (taosHashPut(pushMgr->pHash, &consumerId, sizeof(int64_t), &clientPusher, sizeof(void*)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
free(clientPusher);
// TODO send rsp back
return NULL;
}
return clientPusher;
}
STqStreamPusher* tqAddStreamPusher(STqPushMgr* pushMgr, int64_t streamId, SEpSet* pEpSet) {
STqStreamPusher* streamPusher = malloc(sizeof(STqStreamPusher));
if (streamPusher == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
streamPusher->type = TQ_PUSHER_TYPE__STREAM;
streamPusher->nodeType = 0;
streamPusher->streamId = streamId;
memcpy(&streamPusher->epSet, pEpSet, sizeof(SEpSet));
if (taosHashPut(pushMgr->pHash, &streamId, sizeof(int64_t), &streamPusher, sizeof(void*)) < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
free(streamPusher);
return NULL;
}
return streamPusher;
}

2
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -365,7 +365,7 @@ int tsdbCreateDFile(STsdb *pRepo, SDFile *pDFile, bool updateHeader, TSDB_FILE_T
if (errno == ENOENT) {
// Try to create directory recursively
char *s = strdup(TSDB_FILE_REL_NAME(pDFile));
if (tfsMkdirRecurAt(pRepo->pTfs, dirname(s), TSDB_FILE_DID(pDFile)) < 0) {
if (tfsMkdirRecurAt(pRepo->pTfs, taosDirName(s), TSDB_FILE_DID(pDFile)) < 0) {
tfree(s);
return -1;
}

12
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -29,7 +29,7 @@ int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta};
switch (pMsg->msgType) {
case TDMT_VND_QUERY:{
case TDMT_VND_QUERY: {
return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg);
}
case TDMT_VND_QUERY_CONTINUE:
@ -63,7 +63,7 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
case TDMT_VND_TABLE_META:
return vnodeGetTableMeta(pVnode, pMsg);
case TDMT_VND_CONSUME:
return tqProcessConsumeReq(pVnode->pTq, pMsg);
return tqProcessPollReq(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in fetch queue", pMsg->msgType);
return TSDB_CODE_VND_APP_ERROR;
@ -71,8 +71,8 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) {
}
static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
STbCfg * pTbCfg = NULL;
STbCfg * pStbCfg = NULL;
STbCfg *pTbCfg = NULL;
STbCfg *pStbCfg = NULL;
tb_uid_t uid;
int32_t nCols;
int32_t nTagCols;
@ -204,9 +204,9 @@ static void freeItemHelper(void *pItem) {
*/
static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
SMTbCursor *pCur = metaOpenTbCursor(pVnode->pMeta);
SArray * pArray = taosArrayInit(10, POINTER_BYTES);
SArray *pArray = taosArrayInit(10, POINTER_BYTES);
char * name = NULL;
char *name = NULL;
int32_t totalLen = 0;
int32_t numOfTables = 0;
while ((name = metaTbCursorNext(pCur)) != NULL) {

2
source/libs/catalog/src/catalog.c
View File

@ -845,7 +845,7 @@ int32_t ctgGetTableMetaFromVnode(SCatalog* pCtg, void *pTrans, const SEpSet* pMg
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pTrans, &vgroupInfo->epset, &rpcMsg, &rpcRsp);
rpcSendRecv(pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {

48
source/libs/catalog/test/catalogTests.cpp
View File

@ -228,10 +228,10 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) {
vgInfo.vgId = i + 1;
vgInfo.hashBegin = i * hashUnit;
vgInfo.hashEnd = hashUnit * (i + 1) - 1;
vgInfo.epset.numOfEps = i % TSDB_MAX_REPLICA + 1;
vgInfo.epset.inUse = i % vgInfo.epset.numOfEps;
for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) {
SEp *addr = &vgInfo.epset.eps[n];
vgInfo.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1;
vgInfo.epSet.inUse = i % vgInfo.epSet.numOfEps;
for (int32_t n = 0; n < vgInfo.epSet.numOfEps; ++n) {
SEp *addr = &vgInfo.epSet.eps[n];
strcpy(addr->fqdn, "a0");
addr->port = n + 22;
}
@ -301,10 +301,10 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *
vg.hashEnd = htonl(UINT32_MAX);
}
vg.epset.numOfEps = i % TSDB_MAX_REPLICA + 1;
vg.epset.inUse = i % vg.epset.numOfEps;
for (int32_t n = 0; n < vg.epset.numOfEps; ++n) {
SEp *addr = &vg.epset.eps[n];
vg.epSet.numOfEps = i % TSDB_MAX_REPLICA + 1;
vg.epSet.inUse = i % vg.epSet.numOfEps;
for (int32_t n = 0; n < vg.epSet.numOfEps; ++n) {
SEp *addr = &vg.epSet.eps[n];
strcpy(addr->fqdn, "a0");
addr->port = n + 22;
}
@ -877,7 +877,7 @@ TEST(tableMeta, normalTable) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (0 == ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM)) {
usleep(50000);
@ -1384,7 +1384,7 @@ TEST(refreshGetMeta, normal2normal) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (true) {
uint64_t n = 0;
@ -1463,7 +1463,7 @@ TEST(refreshGetMeta, normal2notexist) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (true) {
uint64_t n = 0;
@ -1537,7 +1537,7 @@ TEST(refreshGetMeta, normal2child) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (true) {
uint64_t n = 0;
@ -1621,7 +1621,7 @@ TEST(refreshGetMeta, stable2child) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (true) {
uint64_t n = 0;
@ -1706,7 +1706,7 @@ TEST(refreshGetMeta, stable2stable) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (true) {
uint64_t n = 0;
@ -1794,7 +1794,7 @@ TEST(refreshGetMeta, child2stable) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
while (true) {
uint64_t n = 0;
@ -1879,7 +1879,7 @@ TEST(tableDistVgroup, normalTable) {
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 8);
ASSERT_EQ(vgInfo->epset.numOfEps, 3);
ASSERT_EQ(vgInfo->epSet.numOfEps, 3);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@ -1921,7 +1921,7 @@ TEST(tableDistVgroup, childTableCase) {
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 9);
ASSERT_EQ(vgInfo->epset.numOfEps, 4);
ASSERT_EQ(vgInfo->epSet.numOfEps, 4);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@ -1964,13 +1964,13 @@ TEST(tableDistVgroup, superTableCase) {
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 10);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(vgInfo->vgId, 1);
ASSERT_EQ(vgInfo->epset.numOfEps, 1);
ASSERT_EQ(vgInfo->epSet.numOfEps, 1);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 1);
ASSERT_EQ(vgInfo->vgId, 2);
ASSERT_EQ(vgInfo->epset.numOfEps, 2);
ASSERT_EQ(vgInfo->epSet.numOfEps, 2);
vgInfo = (SVgroupInfo *)taosArrayGet(vgList, 2);
ASSERT_EQ(vgInfo->vgId, 3);
ASSERT_EQ(vgInfo->epset.numOfEps, 3);
ASSERT_EQ(vgInfo->epSet.numOfEps, 3);
catalogDestroy();
memset(&gCtgMgmt, 0, sizeof(gCtgMgmt));
@ -2025,14 +2025,14 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 8);
ASSERT_EQ(vgInfo.epset.numOfEps, 3);
ASSERT_EQ(vgInfo.epSet.numOfEps, 3);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(pvgInfo->vgId, 8);
ASSERT_EQ(pvgInfo->epset.numOfEps, 3);
ASSERT_EQ(pvgInfo->epSet.numOfEps, 3);
taosArrayDestroy(vgList);
ctgTestBuildDBVgroup(&dbVgroup);
@ -2053,14 +2053,14 @@ TEST(dbVgroup, getSetDbVgroupCase) {
code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo);
ASSERT_EQ(code, 0);
ASSERT_EQ(vgInfo.vgId, 7);
ASSERT_EQ(vgInfo.epset.numOfEps, 2);
ASSERT_EQ(vgInfo.epSet.numOfEps, 2);
code = catalogGetTableDistVgInfo(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgList);
ASSERT_EQ(code, 0);
ASSERT_EQ(taosArrayGetSize((const SArray *)vgList), 1);
pvgInfo = (SVgroupInfo *)taosArrayGet(vgList, 0);
ASSERT_EQ(pvgInfo->vgId, 8);
ASSERT_EQ(pvgInfo->epset.numOfEps, 3);
ASSERT_EQ(pvgInfo->epSet.numOfEps, 3);
taosArrayDestroy(vgList);
catalogDestroy();

4
source/libs/executor/src/executorimpl.c
View File

@ -4977,7 +4977,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf
SSourceDataInfo *pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex);
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu,
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources);
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources);
pMsg->header.vgId = htonl(pSource->addr.nodeId);
pMsg->sId = htobe64(pSource->schedId);
@ -5000,7 +5000,7 @@ static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInf
pMsgSendInfo->fp = loadRemoteDataCallback;
int64_t transporterId = 0;
int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo);
int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epSet, &transporterId, pMsgSendInfo);
return TSDB_CODE_SUCCESS;
}

12
source/libs/index/src/index_tfile.c
View File

@ -722,13 +722,13 @@ static SArray* tfileGetFileList(const char* path) {
uint32_t version;
SArray* files = taosArrayInit(4, sizeof(void*));
DIR* dir = opendir(path);
if (NULL == dir) {
TdDirPtr pDir = taosOpenDir(path);
if (NULL == pDir) {
return NULL;
}
struct dirent* entry;
while ((entry = readdir(dir)) != NULL) {
char* file = entry->d_name;
TdDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char* file = taosGetDirEntryName(pDirEntry);
if (0 != tfileParseFileName(file, &suid, buf, &version)) {
continue;
}
@ -738,7 +738,7 @@ static SArray* tfileGetFileList(const char* path) {
sprintf(buf, "%s/%s", path, file);
taosArrayPush(files, &buf);
}
closedir(dir);
taosCloseDir(pDir);
taosArraySort(files, tfileCompare);
tfileRmExpireFile(files);

1
source/libs/monitor/CMakeLists.txt
View File

@ -3,7 +3,6 @@ add_library(monitor STATIC ${MONITOR_SRC})
target_include_directories(
monitor
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/monitor"
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/transport"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)

1
source/libs/monitor/inc/monInt.h
View File

@ -19,7 +19,6 @@
#include "monitor.h"
#include "tarray.h"
#include "tlockfree.h"
#include "tjson.h"
typedef struct {

2
source/libs/parser/src/dCDAstProcess.c
View File

@ -58,7 +58,7 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out
SVgroupInfo* info = taosArrayGet(array, 0);
pShowReq->head.vgId = htonl(info->vgId);
*pEpSet = info->epset;
*pEpSet = info->epSet;
*outputLen = sizeof(SVShowTablesReq);
*output = pShowReq;

20
source/libs/parser/test/mockCatalogService.cpp
View File

@ -43,11 +43,11 @@ public:
SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0, };
vgroup.epset.eps[0] = (SEp){"dnode_1", 6030};
vgroup.epset.eps[1] = (SEp){"dnode_2", 6030};
vgroup.epset.eps[2] = (SEp){"dnode_3", 6030};
vgroup.epset.inUse = 0;
vgroup.epset.numOfEps = 3;
vgroup.epSet.eps[0] = (SEp){"dnode_1", 6030};
vgroup.epSet.eps[1] = (SEp){"dnode_2", 6030};
vgroup.epSet.eps[2] = (SEp){"dnode_3", 6030};
vgroup.epSet.inUse = 0;
vgroup.epSet.numOfEps = 3;
meta_->vgs.emplace_back(vgroup);
return *this;
@ -122,7 +122,7 @@ public:
int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const {
// todo
vgInfo->vgId = 1;
addEpIntoEpSet(&vgInfo->epset, "node1", 6030);
addEpIntoEpSet(&vgInfo->epSet, "node1", 6030);
return 0;
}
@ -143,10 +143,10 @@ public:
meta_[db][tbname]->schema->uid = id_++;
SVgroupInfo vgroup = {.vgId = vgid, .hashBegin = 0, .hashEnd = 0,};
addEpIntoEpSet(&vgroup.epset, "dnode_1", 6030);
addEpIntoEpSet(&vgroup.epset, "dnode_2", 6030);
addEpIntoEpSet(&vgroup.epset, "dnode_3", 6030);
vgroup.epset.inUse = 0;
addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030);
addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030);
addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030);
vgroup.epSet.inUse = 0;
meta_[db][tbname]->vgs.emplace_back(vgroup);
// super table

4
source/libs/planner/src/physicalPlan.c
View File

@ -254,7 +254,7 @@ static SSubplan* initSubplan(SPlanContext* pCxt, int32_t type) {
static void vgroupInfoToNodeAddr(const SVgroupInfo* vg, SQueryNodeAddr* pNodeAddr) {
pNodeAddr->nodeId = vg->vgId;
pNodeAddr->epset = vg->epset;
pNodeAddr->epSet = vg->epSet;
}
static uint64_t splitSubplanByTable(SPlanContext* pCxt, SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo) {
@ -363,7 +363,7 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan
SSubplan* subplan = initSubplan(pCxt, QUERY_TYPE_MODIFY);
SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i);
subplan->execNode.epset = blocks->vg.epset;
subplan->execNode.epSet = blocks->vg.epSet;
subplan->pDataSink = createDataInserter(pCxt, blocks, NULL);
subplan->pNode = NULL;
subplan->type = QUERY_TYPE_MODIFY;

211
source/libs/planner/src/physicalPlanJson.c
View File

@ -13,9 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "plannerInt.h"
#include "parser.h"
#include "cJSON.h"
#include "parser.h"
#include "plannerInt.h"
typedef bool (*FToJson)(const void* obj, cJSON* json);
typedef bool (*FFromJson)(const cJSON* json, void* obj);
@ -30,7 +30,7 @@ static void copyString(const cJSON* json, const char* name, char* dst) {
}
static uint64_t getBigintFromString(const cJSON* json, const char* name) {
char* val = getString(json, name);
char* val = getString(json, name);
uint64_t intVal = strtoul(val, NULL, 10);
tfree(val);
@ -39,7 +39,7 @@ static uint64_t getBigintFromString(const cJSON* json, const char* name) {
static int64_t getNumber(const cJSON* json, const char* name) {
double d = cJSON_GetNumberValue(cJSON_GetObjectItem(json, name));
return (int64_t) d;
return (int64_t)d;
}
static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) {
@ -72,7 +72,8 @@ static bool fromObject(const cJSON* json, const char* name, FFromJson func, void
return func(jObj, obj);
}
static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size, bool required) {
static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson func, void** obj, int32_t size,
bool required) {
cJSON* jObj = cJSON_GetObjectItem(json, name);
if (NULL == jObj) {
return !required;
@ -85,7 +86,7 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f
}
static const char* jkPnodeType = "Type";
static int32_t getPnodeTypeSize(cJSON* json) {
static int32_t getPnodeTypeSize(cJSON* json) {
switch (getNumber(json, jkPnodeType)) {
case OP_StreamScan:
case OP_TableScan:
@ -119,7 +120,7 @@ static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void*
static bool fromPnodeArray(const cJSON* json, const char* name, FFromJson func, SArray** array) {
const cJSON* jArray = cJSON_GetObjectItem(json, name);
int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
if (size > 0) {
*array = taosArrayInit(size, POINTER_BYTES);
if (NULL == *array) {
@ -128,7 +129,7 @@ static bool fromPnodeArray(const cJSON* json, const char* name, FFromJson func,
}
for (int32_t i = 0; i < size; ++i) {
cJSON* jItem = cJSON_GetArrayItem(jArray, i);
void* item = calloc(1, getPnodeTypeSize(jItem));
void* item = calloc(1, getPnodeTypeSize(jItem));
if (NULL == item || !func(jItem, item)) {
return false;
}
@ -161,9 +162,10 @@ static bool addArray(cJSON* json, const char* name, FToJson func, const SArray*
return addTarray(json, name, func, array, true);
}
static bool fromTarray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize, bool isPoint) {
static bool fromTarray(const cJSON* json, const char* name, FFromJson func, SArray** array, int32_t itemSize,
bool isPoint) {
const cJSON* jArray = cJSON_GetObjectItem(json, name);
int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
int32_t size = (NULL == jArray ? 0 : cJSON_GetArraySize(jArray));
if (size > 0) {
*array = taosArrayInit(size, isPoint ? POINTER_BYTES : itemSize);
if (NULL == *array) {
@ -188,7 +190,8 @@ static bool fromArray(const cJSON* json, const char* name, FFromJson func, SArra
return fromTarray(json, name, func, array, itemSize, true);
}
static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize, int32_t size) {
static bool addRawArray(cJSON* json, const char* name, FToJson func, const void* array, int32_t itemSize,
int32_t size) {
if (size > 0) {
cJSON* jArray = cJSON_AddArrayToObject(json, name);
if (NULL == jArray) {
@ -218,7 +221,8 @@ static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t i
return true;
}
static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) {
static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize,
int32_t* size) {
const cJSON* jArray = getArray(json, name, size);
if (*size > 0) {
*array = calloc(1, itemSize * (*size));
@ -229,7 +233,8 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson
return fromItem(jArray, func, *array, itemSize, *size);
}
static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) {
static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize,
int32_t* size) {
const cJSON* jArray = getArray(json, name, size);
return fromItem(jArray, func, array, itemSize, *size);
}
@ -240,7 +245,7 @@ static const char* jkSchemaBytes = "Bytes";
// The 'name' field do not need to be serialized.
static bool schemaToJson(const void* obj, cJSON* jSchema) {
const SSlotSchema* schema = (const SSlotSchema*)obj;
bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type);
bool res = cJSON_AddNumberToObject(jSchema, jkSchemaType, schema->type);
if (res) {
res = cJSON_AddNumberToObject(jSchema, jkSchemaColId, schema->colId);
}
@ -264,7 +269,8 @@ static const char* jkDataBlockSchemaPrecision = "Precision";
static bool dataBlockSchemaToJson(const void* obj, cJSON* json) {
const SDataBlockSchema* schema = (const SDataBlockSchema*)obj;
bool res = addRawArray(json, jkDataBlockSchemaSlotSchema, schemaToJson, schema->pSchema, sizeof(SSlotSchema), schema->numOfCols);
bool res = addRawArray(json, jkDataBlockSchemaSlotSchema, schemaToJson, schema->pSchema, sizeof(SSlotSchema),
schema->numOfCols);
if (res) {
res = cJSON_AddNumberToObject(json, jkDataBlockSchemaResultRowSize, schema->resultRowSize);
}
@ -279,7 +285,8 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) {
schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize);
schema->precision = getNumber(json, jkDataBlockSchemaPrecision);
return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols);
return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema),
sizeof(SSlotSchema), &schema->numOfCols);
}
static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr";
@ -290,7 +297,7 @@ static const char* jkColumnFilterInfoUpperBnd = "UpperBnd";
static bool columnFilterInfoToJson(const void* obj, cJSON* jFilter) {
const SColumnFilterInfo* filter = (const SColumnFilterInfo*)obj;
bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr);
bool res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoLowerRelOptr, filter->lowerRelOptr);
if (res) {
res = cJSON_AddNumberToObject(jFilter, jkColumnFilterInfoUpperRelOptr, filter->upperRelOptr);
}
@ -323,7 +330,7 @@ static const char* jkColumnInfoFilterList = "FilterList";
static bool columnInfoToJson(const void* obj, cJSON* jCol) {
const SColumnInfo* col = (const SColumnInfo*)obj;
bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId);
bool res = cJSON_AddNumberToObject(jCol, jkColumnInfoColId, col->colId);
if (res) {
res = cJSON_AddNumberToObject(jCol, jkColumnInfoType, col->type);
}
@ -331,8 +338,9 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) {
res = cJSON_AddNumberToObject(jCol, jkColumnInfoBytes, col->bytes);
}
if (res) { // TODO: temporarily disable it
// res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo, sizeof(SColumnFilterInfo), col->flist.numOfFilters);
if (res) { // TODO: temporarily disable it
// res = addRawArray(jCol, jkColumnInfoFilterList, columnFilterInfoToJson, col->flist.filterInfo,
// sizeof(SColumnFilterInfo), col->flist.numOfFilters);
}
return res;
@ -341,10 +349,11 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) {
static bool columnInfoFromJson(const cJSON* json, void* obj) {
SColumnInfo* col = (SColumnInfo*)obj;
col->colId = getNumber(json, jkColumnInfoColId);
col->type = getNumber(json, jkColumnInfoType);
col->type = getNumber(json, jkColumnInfoType);
col->bytes = getNumber(json, jkColumnInfoBytes);
int32_t size = 0;
bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size);
bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson,
(void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size);
col->flist.numOfFilters = size;
return res;
}
@ -355,7 +364,7 @@ static const char* jkColumnInfo = "Info";
static bool columnToJson(const void* obj, cJSON* jCol) {
const SColumn* col = (const SColumn*)obj;
bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid);
bool res = cJSON_AddNumberToObject(jCol, jkColumnTableId, col->uid);
if (res) {
res = cJSON_AddNumberToObject(jCol, jkColumnFlag, col->flag);
}
@ -381,7 +390,7 @@ static const char* jkExprNodeRight = "Right";
static bool operatorToJson(const void* obj, cJSON* jOper) {
const tExprNode* exprInfo = (const tExprNode*)obj;
bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr);
bool res = cJSON_AddNumberToObject(jOper, jkExprNodeOper, exprInfo->_node.optr);
if (res) {
res = addObject(jOper, jkExprNodeLeft, exprNodeToJson, exprInfo->_node.pLeft);
}
@ -406,9 +415,10 @@ static const char* jkFunctionChild = "Child";
static bool functionToJson(const void* obj, cJSON* jFunc) {
const tExprNode* exprInfo = (const tExprNode*)obj;
bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName);
bool res = cJSON_AddStringToObject(jFunc, jkFunctionName, exprInfo->_function.functionName);
if (res && NULL != exprInfo->_function.pChild) {
res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*), exprInfo->_function.num);
res = addRawArray(jFunc, jkFunctionChild, exprNodeToJson, exprInfo->_function.pChild, sizeof(tExprNode*),
exprInfo->_function.num);
}
return res;
}
@ -420,7 +430,8 @@ static bool functionFromJson(const cJSON* json, void* obj) {
if (NULL == exprInfo->_function.pChild) {
return false;
}
return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild, sizeof(tExprNode*), &exprInfo->_function.num);
return fromRawArrayWithAlloc(json, jkFunctionChild, exprNodeFromJson, (void**)exprInfo->_function.pChild,
sizeof(tExprNode*), &exprInfo->_function.num);
}
static const char* jkVariantType = "Type";
@ -430,12 +441,12 @@ static const char* jkVariantValue = "Value";
static bool variantToJson(const void* obj, cJSON* jVar) {
const SVariant* var = (const SVariant*)obj;
bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType);
bool res = cJSON_AddNumberToObject(jVar, jkVariantType, var->nType);
if (res) {
res = cJSON_AddNumberToObject(jVar, jkVariantLen, var->nLen);
}
if (res) {
if (0/* in */) {
if (0 /* in */) {
res = addArray(jVar, jkVariantvalues, variantToJson, var->arr);
} else if (IS_NUMERIC_TYPE(var->nType)) {
res = cJSON_AddNumberToObject(jVar, jkVariantValue, var->d);
@ -450,7 +461,7 @@ static bool variantFromJson(const cJSON* json, void* obj) {
SVariant* var = (SVariant*)obj;
var->nType = getNumber(json, jkVariantType);
var->nLen = getNumber(json, jkVariantLen);
if (0/* in */) {
if (0 /* in */) {
return fromArray(json, jkVariantvalues, variantFromJson, &var->arr, sizeof(SVariant));
} else if (IS_NUMERIC_TYPE(var->nType)) {
var->d = getNumber(json, jkVariantValue);
@ -468,7 +479,7 @@ static const char* jkExprNodeValue = "Value";
static bool exprNodeToJson(const void* obj, cJSON* jExprInfo) {
const tExprNode* exprInfo = *(const tExprNode**)obj;
bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType);
bool res = cJSON_AddNumberToObject(jExprInfo, jkExprNodeType, exprInfo->nodeType);
if (res) {
switch (exprInfo->nodeType) {
case TEXPR_BINARYEXPR_NODE:
@ -502,7 +513,8 @@ static bool exprNodeFromJson(const cJSON* json, void* obj) {
case TEXPR_FUNCTION_NODE:
return fromObject(json, jkExprNodeFunction, functionFromJson, exprInfo, false);
case TEXPR_COL_NODE:
return fromObjectWithAlloc(json, jkExprNodeColumn, schemaFromJson, (void**)&exprInfo->pSchema, sizeof(SSchema), false);
return fromObjectWithAlloc(json, jkExprNodeColumn, schemaFromJson, (void**)&exprInfo->pSchema, sizeof(SSchema),
false);
case TEXPR_VALUE_NODE:
return fromObject(json, jkExprNodeValue, variantFromJson, exprInfo->pVal, false);
default:
@ -518,7 +530,7 @@ static const char* jkSqlExprParams = "Params";
// token does not need to be serialized.
static bool sqlExprToJson(const void* obj, cJSON* jExpr) {
const SSqlExpr* expr = (const SSqlExpr*)obj;
bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema);
bool res = addObject(jExpr, jkSqlExprSchema, schemaToJson, &expr->resSchema);
if (res) {
res = addRawArray(jExpr, jkSqlExprColumns, columnToJson, expr->pColumns, sizeof(SColumn), expr->numOfCols);
}
@ -533,9 +545,10 @@ static bool sqlExprToJson(const void* obj, cJSON* jExpr) {
static bool sqlExprFromJson(const cJSON* json, void* obj) {
SSqlExpr* expr = (SSqlExpr*)obj;
bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false);
bool res = fromObject(json, jkSqlExprSchema, schemaFromJson, &expr->resSchema, false);
if (res) {
res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn), &expr->numOfCols);
res = fromRawArrayWithAlloc(json, jkSqlExprColumns, columnFromJson, (void**)&expr->pColumns, sizeof(SColumn),
&expr->numOfCols);
}
if (res) {
expr->interBytes = getNumber(json, jkSqlExprInterBytes);
@ -553,7 +566,7 @@ static const char* jkExprInfoExpr = "Expr";
static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) {
const SExprInfo* exprInfo = (const SExprInfo*)obj;
bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base);
bool res = addObject(jExprInfo, jkExprInfoBase, sqlExprToJson, &exprInfo->base);
if (res) {
res = addObject(jExprInfo, jkExprInfoExpr, exprNodeToJson, &exprInfo->pExpr);
}
@ -562,9 +575,10 @@ static bool exprInfoToJson(const void* obj, cJSON* jExprInfo) {
static bool exprInfoFromJson(const cJSON* json, void* obj) {
SExprInfo* exprInfo = (SExprInfo*)obj;
bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true);
bool res = fromObject(json, jkExprInfoBase, sqlExprFromJson, &exprInfo->base, true);
if (res) {
res = fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true);
res =
fromObjectWithAlloc(json, jkExprInfoExpr, exprNodeFromJson, (void**)&exprInfo->pExpr, sizeof(tExprNode), true);
}
return res;
}
@ -576,12 +590,12 @@ static bool timeWindowToJson(const void* obj, cJSON* json) {
const STimeWindow* win = (const STimeWindow*)obj;
char tmp[40] = {0};
snprintf(tmp, tListLen(tmp),"%"PRId64, win->skey);
snprintf(tmp, tListLen(tmp), "%" PRId64, win->skey);
bool res = cJSON_AddStringToObject(json, jkTimeWindowStartKey, tmp);
if (res) {
memset(tmp, 0, tListLen(tmp));
snprintf(tmp, tListLen(tmp),"%"PRId64, win->ekey);
snprintf(tmp, tListLen(tmp), "%" PRId64, win->ekey);
res = cJSON_AddStringToObject(json, jkTimeWindowEndKey, tmp);
}
return res;
@ -604,7 +618,7 @@ static bool scanNodeToJson(const void* obj, cJSON* json) {
const SScanPhyNode* pNode = (const SScanPhyNode*)obj;
char uid[40] = {0};
snprintf(uid, tListLen(uid), "%"PRIu64, pNode->uid);
snprintf(uid, tListLen(uid), "%" PRIu64, pNode->uid);
bool res = cJSON_AddStringToObject(json, jkScanNodeTableId, uid);
if (res) {
@ -629,11 +643,11 @@ static bool scanNodeToJson(const void* obj, cJSON* json) {
static bool scanNodeFromJson(const cJSON* json, void* obj) {
SScanPhyNode* pNode = (SScanPhyNode*)obj;
pNode->uid = getBigintFromString(json, jkScanNodeTableId);
pNode->uid = getBigintFromString(json, jkScanNodeTableId);
pNode->tableType = getNumber(json, jkScanNodeTableType);
pNode->count = getNumber(json, jkScanNodeTableCount);
pNode->order = getNumber(json, jkScanNodeTableOrder);
pNode->reverse = getNumber(json, jkScanNodeTableRevCount);
pNode->count = getNumber(json, jkScanNodeTableCount);
pNode->order = getNumber(json, jkScanNodeTableOrder);
pNode->reverse = getNumber(json, jkScanNodeTableRevCount);
return true;
}
@ -644,7 +658,7 @@ static const char* jkColIndexName = "Name";
static bool colIndexToJson(const void* obj, cJSON* json) {
const SColIndex* col = (const SColIndex*)obj;
bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId);
bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId);
if (res) {
res = cJSON_AddNumberToObject(json, jkColIndexColIndex, col->colIndex);
}
@ -673,7 +687,7 @@ static const char* jkAggNodeGroupByList = "GroupByList";
static bool aggNodeToJson(const void* obj, cJSON* json) {
const SAggPhyNode* agg = (const SAggPhyNode*)obj;
bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo);
bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo);
if (res) {
res = cJSON_AddNumberToObject(json, jkAggNodeAggSplit, agg->aggSplit);
}
@ -703,7 +717,7 @@ static const char* jkTableScanNodeTagsConditions = "TagsConditions";
static bool tableScanNodeToJson(const void* obj, cJSON* json) {
const STableScanPhyNode* scan = (const STableScanPhyNode*)obj;
bool res = scanNodeToJson(obj, json);
bool res = scanNodeToJson(obj, json);
if (res) {
res = cJSON_AddNumberToObject(json, jkTableScanNodeFlag, scan->scanFlag);
}
@ -718,7 +732,7 @@ static bool tableScanNodeToJson(const void* obj, cJSON* json) {
static bool tableScanNodeFromJson(const cJSON* json, void* obj) {
STableScanPhyNode* scan = (STableScanPhyNode*)obj;
bool res = scanNodeFromJson(json, obj);
bool res = scanNodeFromJson(json, obj);
if (res) {
scan->scanFlag = getNumber(json, jkTableScanNodeFlag);
}
@ -736,7 +750,7 @@ static const char* jkEpAddrPort = "Port";
static bool epAddrToJson(const void* obj, cJSON* json) {
const SEp* ep = (const SEp*)obj;
bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn);
bool res = cJSON_AddStringToObject(json, jkEpAddrFqdn, ep->fqdn);
if (res) {
res = cJSON_AddNumberToObject(json, jkEpAddrPort, ep->port);
}
@ -750,46 +764,46 @@ static bool epAddrFromJson(const cJSON* json, void* obj) {
return true;
}
static const char* jkNodeAddrId = "NodeId";
static const char* jkNodeAddrInUse = "InUse";
static const char* jkNodeAddrId = "NodeId";
static const char* jkNodeAddrInUse = "InUse";
static const char* jkNodeAddrEpAddrs = "Ep";
static const char* jkNodeAddr = "NodeAddr";
static const char* jkNodeTaskId = "TaskId";
static const char* jkNodeAddr = "NodeAddr";
static const char* jkNodeTaskId = "TaskId";
static const char* jkNodeTaskSchedId = "SchedId";
static bool queryNodeAddrToJson(const void* obj, cJSON* json) {
const SQueryNodeAddr* pAddr = (const SQueryNodeAddr*) obj;
bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId);
const SQueryNodeAddr* pAddr = (const SQueryNodeAddr*)obj;
bool res = cJSON_AddNumberToObject(json, jkNodeAddrId, pAddr->nodeId);
if (res) {
res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epset.inUse);
res = cJSON_AddNumberToObject(json, jkNodeAddrInUse, pAddr->epSet.inUse);
}
if (res) {
res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epset.eps, sizeof(SEp), pAddr->epset.numOfEps);
res = addRawArray(json, jkNodeAddrEpAddrs, epAddrToJson, pAddr->epSet.eps, sizeof(SEp), pAddr->epSet.numOfEps);
}
return res;
}
static bool queryNodeAddrFromJson(const cJSON* json, void* obj) {
SQueryNodeAddr* pAddr = (SQueryNodeAddr*) obj;
SQueryNodeAddr* pAddr = (SQueryNodeAddr*)obj;
pAddr->nodeId = getNumber(json, jkNodeAddrId);
pAddr->epset.inUse = getNumber(json, jkNodeAddrInUse);
pAddr->epSet.inUse = getNumber(json, jkNodeAddrInUse);
int32_t numOfEps = 0;
bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epset.eps, sizeof(SEp), &numOfEps);
pAddr->epset.numOfEps = numOfEps;
bool res = fromRawArray(json, jkNodeAddrEpAddrs, epAddrFromJson, pAddr->epSet.eps, sizeof(SEp), &numOfEps);
pAddr->epSet.numOfEps = numOfEps;
return res;
}
static bool nodeAddrToJson(const void* obj, cJSON* json) {
const SDownstreamSource* pSource = (const SDownstreamSource*) obj;
bool res = cJSON_AddNumberToObject(json, jkNodeTaskId, pSource->taskId);
const SDownstreamSource* pSource = (const SDownstreamSource*)obj;
bool res = cJSON_AddNumberToObject(json, jkNodeTaskId, pSource->taskId);
if (res) {
char t[30] = {0};
snprintf(t, tListLen(t), "%"PRIu64, pSource->schedId);
snprintf(t, tListLen(t), "%" PRIu64, pSource->schedId);
res = cJSON_AddStringToObject(json, jkNodeTaskSchedId, t);
}
@ -813,9 +827,10 @@ static const char* jkExchangeNodeSrcEndPoints = "SrcAddrs";
static bool exchangeNodeToJson(const void* obj, cJSON* json) {
const SExchangePhyNode* exchange = (const SExchangePhyNode*)obj;
bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId);
bool res = cJSON_AddNumberToObject(json, jkExchangeNodeSrcTemplateId, exchange->srcTemplateId);
if (res) {
res = addRawArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints->pData, sizeof(SDownstreamSource), taosArrayGetSize(exchange->pSrcEndPoints));
res = addRawArray(json, jkExchangeNodeSrcEndPoints, nodeAddrToJson, exchange->pSrcEndPoints->pData,
sizeof(SDownstreamSource), taosArrayGetSize(exchange->pSrcEndPoints));
}
return res;
}
@ -823,7 +838,8 @@ static bool exchangeNodeToJson(const void* obj, cJSON* json) {
static bool exchangeNodeFromJson(const cJSON* json, void* obj) {
SExchangePhyNode* exchange = (SExchangePhyNode*)obj;
exchange->srcTemplateId = getNumber(json, jkExchangeNodeSrcTemplateId);
return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints, sizeof(SDownstreamSource));
return fromInlineArray(json, jkExchangeNodeSrcEndPoints, nodeAddrFromJson, &exchange->pSrcEndPoints,
sizeof(SDownstreamSource));
}
static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
@ -855,7 +871,7 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) {
case OP_AllTimeWindow:
case OP_AllMultiTableTimeInterval:
case OP_Order:
break; // todo
break; // todo
case OP_Exchange:
return exchangeNodeToJson(obj, json);
default:
@ -893,7 +909,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) {
case OP_AllTimeWindow:
case OP_AllMultiTableTimeInterval:
case OP_Order:
break; // todo
break; // todo
case OP_Exchange:
return exchangeNodeFromJson(json, obj);
default:
@ -910,7 +926,7 @@ static const char* jkPnodeChildren = "Children";
// The 'pParent' field do not need to be serialized.
static bool phyNodeToJson(const void* obj, cJSON* jNode) {
const SPhyNode* phyNode = (const SPhyNode*)obj;
bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type);
bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type);
if (res) {
res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name);
}
@ -933,7 +949,7 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) {
}
static bool phyNodeFromJson(const cJSON* json, void* obj) {
SPhyNode* node = (SPhyNode*) obj;
SPhyNode* node = (SPhyNode*)obj;
node->info.type = getNumber(json, jkPnodeType);
node->info.name = opTypeToOpName(node->info.type);
@ -959,7 +975,7 @@ static const char* jkInserterDataSize = "DataSize";
static bool inserterToJson(const void* obj, cJSON* json) {
const SDataInserter* inserter = (const SDataInserter*)obj;
bool res = cJSON_AddNumberToObject(json, jkInserterNumOfTables, inserter->numOfTables);
bool res = cJSON_AddNumberToObject(json, jkInserterNumOfTables, inserter->numOfTables);
if (res) {
res = cJSON_AddNumberToObject(json, jkInserterDataSize, inserter->size);
}
@ -1005,7 +1021,7 @@ static const char* jkDataSinkSchema = "Schema";
static bool dataSinkToJson(const void* obj, cJSON* json) {
const SDataSink* dsink = (const SDataSink*)obj;
bool res = cJSON_AddStringToObject(json, jkDataSinkName, dsink->info.name);
bool res = cJSON_AddStringToObject(json, jkDataSinkName, dsink->info.name);
if (res) {
res = addObject(json, dsink->info.name, specificDataSinkToJson, dsink);
}
@ -1034,7 +1050,7 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) {
const SSubplanId* id = (const SSubplanId*)obj;
char ids[40] = {0};
snprintf(ids, tListLen(ids), "%"PRIu64, id->queryId);
snprintf(ids, tListLen(ids), "%" PRIu64, id->queryId);
bool res = cJSON_AddStringToObject(jId, jkIdQueryId, ids);
if (res) {
@ -1049,9 +1065,9 @@ static bool subplanIdToJson(const void* obj, cJSON* jId) {
static bool subplanIdFromJson(const cJSON* json, void* obj) {
SSubplanId* id = (SSubplanId*)obj;
id->queryId = getBigintFromString(json, jkIdQueryId);
id->queryId = getBigintFromString(json, jkIdQueryId);
id->templateId = getNumber(json, jkIdTemplateId);
id->subplanId = getNumber(json, jkIdSubplanId);
id->subplanId = getNumber(json, jkIdSubplanId);
return true;
}
@ -1094,7 +1110,8 @@ static SSubplan* subplanFromJson(const cJSON* json) {
}
if (res) {
res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false);
res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink),
false);
}
if (!res) {
@ -1137,15 +1154,15 @@ int32_t stringToSubplan(const char* str, SSubplan** subplan) {
cJSON* qDagToJson(const SQueryDag* pDag) {
cJSON* pRoot = cJSON_CreateObject();
if(pRoot == NULL) {
if (pRoot == NULL) {
return NULL;
}
cJSON_AddNumberToObject(pRoot, "Number", pDag->numOfSubplans);
cJSON_AddNumberToObject(pRoot, "QueryId", pDag->queryId);
cJSON *pLevels = cJSON_CreateArray();
if(pLevels == NULL) {
cJSON* pLevels = cJSON_CreateArray();
if (pLevels == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
@ -1153,19 +1170,19 @@ cJSON* qDagToJson(const SQueryDag* pDag) {
cJSON_AddItemToObject(pRoot, "Subplans", pLevels);
size_t level = taosArrayGetSize(pDag->pSubplans);
for(size_t i = 0; i < level; i++) {
for (size_t i = 0; i < level; i++) {
const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i);
size_t num = taosArrayGetSize(pSubplans);
cJSON* plansOneLevel = cJSON_CreateArray();
if(plansOneLevel == NULL) {
size_t num = taosArrayGetSize(pSubplans);
cJSON* plansOneLevel = cJSON_CreateArray();
if (plansOneLevel == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
cJSON_AddItemToArray(pLevels, plansOneLevel);
for(size_t j = 0; j < num; j++) {
for (size_t j = 0; j < num; j++) {
cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j));
if(pSubplan == NULL) {
if (pSubplan == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
@ -1183,22 +1200,22 @@ char* qDagToString(const SQueryDag* pDag) {
SQueryDag* qJsonToDag(const cJSON* pRoot) {
SQueryDag* pDag = malloc(sizeof(SQueryDag));
if(pDag == NULL) {
if (pDag == NULL) {
return NULL;
}
pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "Number"));
pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "QueryId"));
pDag->pSubplans = taosArrayInit(0, sizeof(SArray));
pDag->pSubplans = taosArrayInit(0, sizeof(void*));
if (pDag->pSubplans == NULL) {
free(pDag);
return NULL;
}
cJSON* pLevels = cJSON_GetObjectItem(pRoot, "Subplans");
int level = cJSON_GetArraySize(pLevels);
for(int i = 0; i < level; i++) {
int level = cJSON_GetArraySize(pLevels);
for (int i = 0; i < level; i++) {
SArray* plansOneLevel = taosArrayInit(0, sizeof(void*));
if(plansOneLevel == NULL) {
for(int j = 0; j < i; j++) {
if (plansOneLevel == NULL) {
for (int j = 0; j < i; j++) {
taosArrayDestroy(taosArrayGetP(pDag->pSubplans, j));
}
taosArrayDestroy(pDag->pSubplans);
@ -1206,13 +1223,13 @@ SQueryDag* qJsonToDag(const cJSON* pRoot) {
return NULL;
}
cJSON* pItem = cJSON_GetArrayItem(pLevels, i);
int sz = cJSON_GetArraySize(pItem);
for(int j = 0; j < sz; j++) {
cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j);
int sz = cJSON_GetArraySize(pItem);
for (int j = 0; j < sz; j++) {
cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j);
SSubplan* pSubplan = subplanFromJson(pSubplanJson);
taosArrayPush(plansOneLevel, &pSubplan);
}
taosArrayPush(pDag->pSubplans, plansOneLevel);
taosArrayPush(pDag->pSubplans, &plansOneLevel);
}
return pDag;
}

6
source/libs/scheduler/src/scheduler.c
View File

@ -423,13 +423,13 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
if (pTask->plan->execNode.epset.numOfEps > 0) {
if (pTask->plan->execNode.epSet.numOfEps > 0) {
if (NULL == taosArrayPush(pTask->candidateAddrs, &pTask->plan->execNode)) {
SCH_TASK_ELOG("taosArrayPush execNode to candidate addrs failed, errno:%d", errno);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epset.numOfEps);
SCH_TASK_DLOG("use execNode from plan as candidate addr, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps);
return TSDB_CODE_SUCCESS;
}
@ -1061,7 +1061,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr,
isCandidateAddr = true;
}
SEpSet epSet = addr->epset;
SEpSet epSet = addr->epSet;
switch (msgType) {
case TDMT_VND_CREATE_TABLE:

14
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -104,8 +104,8 @@ void schtBuildQueryDag(SQueryDag *dag) {
scanPlan->type = QUERY_TYPE_SCAN;
scanPlan->execNode.nodeId = 1;
scanPlan->execNode.epset.inUse = 0;
addEpIntoEpSet(&scanPlan->execNode.epset, "ep0", 6030);
scanPlan->execNode.epSet.inUse = 0;
addEpIntoEpSet(&scanPlan->execNode.epSet, "ep0", 6030);
scanPlan->pChildren = NULL;
scanPlan->level = 1;
@ -118,7 +118,7 @@ void schtBuildQueryDag(SQueryDag *dag) {
mergePlan->id.subplanId = 0x5555555555;
mergePlan->type = QUERY_TYPE_MERGE;
mergePlan->level = 0;
mergePlan->execNode.epset.numOfEps = 0;
mergePlan->execNode.epSet.numOfEps = 0;
mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES);
mergePlan->pParents = NULL;
@ -157,8 +157,8 @@ void schtBuildInsertDag(SQueryDag *dag) {
insertPlan[0].level = 0;
insertPlan[0].execNode.nodeId = 1;
insertPlan[0].execNode.epset.inUse = 0;
addEpIntoEpSet(&insertPlan[0].execNode.epset, "ep0", 6030);
insertPlan[0].execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan[0].execNode.epSet, "ep0", 6030);
insertPlan[0].pChildren = NULL;
insertPlan[0].pParents = NULL;
@ -173,8 +173,8 @@ void schtBuildInsertDag(SQueryDag *dag) {
insertPlan[1].level = 0;
insertPlan[1].execNode.nodeId = 1;
insertPlan[1].execNode.epset.inUse = 0;
addEpIntoEpSet(&insertPlan[1].execNode.epset, "ep0", 6030);
insertPlan[1].execNode.epSet.inUse = 0;
addEpIntoEpSet(&insertPlan[1].execNode.epSet, "ep0", 6030);
insertPlan[1].pChildren = NULL;
insertPlan[1].pParents = NULL;

49
source/libs/sync/inc/syncIndexMgr.h Normal file
View File

@ -0,0 +1,49 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_INDEX_MGR_H
#define _TD_LIBS_SYNC_INDEX_MGR_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncInt.h"
#include "taosdef.h"
// SIndexMgr -----------------------------
typedef struct SSyncIndexMgr {
SRaftId (*replicas)[TSDB_MAX_REPLICA];
SyncIndex index[TSDB_MAX_REPLICA];
int32_t replicaNum;
SSyncNode *pSyncNode;
} SSyncIndexMgr;
SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode);
void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr);
void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr);
void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index);
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId);
cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr);
char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_INDEX_MGR_H*/

29
source/libs/sync/inc/syncInt.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "cJSON.h"
#include "sync.h"
#include "taosdef.h"
#include "tglobal.h"
@ -99,8 +100,11 @@ typedef struct SRaftStore SRaftStore;
struct SVotesGranted;
typedef struct SVotesGranted SVotesGranted;
struct SVotesResponded;
typedef struct SVotesResponded SVotesResponded;
struct SVotesRespond;
typedef struct SVotesRespond SVotesRespond;
struct SSyncIndexMgr;
typedef struct SSyncIndexMgr SSyncIndexMgr;
typedef struct SRaftId {
SyncNodeId addr; // typedef uint64_t SyncNodeId;
@ -112,17 +116,18 @@ typedef struct SSyncNode {
SyncGroupId vgId;
SSyncCfg syncCfg;
char path[TSDB_FILENAME_LEN];
char walPath[TSDB_FILENAME_LEN];
void* rpcClient;
int32_t (*FpSendMsg)(void* rpcClient, const SEpSet* pEpSet, SRpcMsg* pMsg);
void* queue;
int32_t (*FpEqMsg)(void* queue, SRpcMsg* pMsg);
// init internal
SNodeInfo me;
SRaftId raftId;
SNodeInfo myNodeInfo;
SRaftId myRaftId;
int32_t peersNum;
SNodeInfo peers[TSDB_MAX_REPLICA];
SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA];
SRaftId peersId[TSDB_MAX_REPLICA];
int32_t replicaNum;
@ -142,18 +147,18 @@ typedef struct SSyncNode {
SRaftStore* pRaftStore;
// tla+ candidate vars
SVotesGranted* pVotesGranted;
SVotesResponded* pVotesResponded;
SVotesGranted* pVotesGranted;
SVotesRespond* pVotesRespond;
// tla+ leader vars
SHashObj* pNextIndex;
SHashObj* pMatchIndex;
SSyncIndexMgr* pNextIndex;
SSyncIndexMgr* pMatchIndex;
// tla+ log vars
SSyncLogStore* pLogStore;
SyncIndex commitIndex;
// timer
// ping timer
tmr_h pPingTimer;
int32_t pingTimerMS;
uint64_t pingTimerLogicClock;
@ -161,6 +166,7 @@ typedef struct SSyncNode {
TAOS_TMR_CALLBACK FpPingTimer; // Timer Fp
uint64_t pingTimerCounter;
// elect timer
tmr_h pElectTimer;
int32_t electTimerMS;
uint64_t electTimerLogicClock;
@ -168,6 +174,7 @@ typedef struct SSyncNode {
TAOS_TMR_CALLBACK FpElectTimer; // Timer Fp
uint64_t electTimerCounter;
// heartbeat timer
tmr_h pHeartbeatTimer;
int32_t heartbeatTimerMS;
uint64_t heartbeatTimerLogicClock;
@ -188,6 +195,8 @@ typedef struct SSyncNode {
SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
void syncNodeClose(SSyncNode* pSyncNode);
cJSON* syncNode2Json(const SSyncNode* pSyncNode);
char* syncNode2Str(const SSyncNode* pSyncNode);
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);

1
source/libs/sync/inc/syncReplication.h
View File

@ -53,6 +53,7 @@ extern "C" {
//
int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode);
int32_t syncNodeReplicate(SSyncNode* pSyncNode);
int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg);
#ifdef __cplusplus

9
source/libs/sync/inc/syncUtil.h
View File

@ -42,8 +42,13 @@ void syncUtilbufCopy(const SSyncBuffer* src, SSyncBuffer* dest);
void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest);
// ---- misc ----
int32_t syncUtilRand(int32_t max);
int32_t syncUtilElectRandomMS();
int32_t syncUtilRand(int32_t max);
int32_t syncUtilElectRandomMS();
int32_t syncUtilQuorum(int32_t replicaNum);
cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p);
cJSON* syncUtilRaftId2Json(const SRaftId* p);
char* syncUtilRaftId2Str(const SRaftId* p);
const char* syncUtilState2String(ESyncState state);
#ifdef __cplusplus
}

14
source/libs/sync/inc/syncVoteMgr.h
View File

@ -28,10 +28,14 @@ extern "C" {
#include "syncUtil.h"
#include "taosdef.h"
// SVotesGranted -----------------------------
typedef struct SVotesGranted {
SRaftId (*replicas)[TSDB_MAX_REPLICA];
bool isGranted[TSDB_MAX_REPLICA];
int32_t replicaNum;
int32_t votes;
SyncTerm term;
int32_t quorum;
int32_t votes;
bool toLeader;
SSyncNode *pSyncNode;
} SVotesGranted;
@ -41,7 +45,10 @@ void voteGrantedDestroy(SVotesGranted *pVotesGranted);
bool voteGrantedMajority(SVotesGranted *pVotesGranted);
void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg);
void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term);
cJSON * voteGranted2Json(SVotesGranted *pVotesGranted);
char * voteGranted2Str(SVotesGranted *pVotesGranted);
// SVotesRespond -----------------------------
typedef struct SVotesRespond {
SRaftId (*replicas)[TSDB_MAX_REPLICA];
bool isRespond[TSDB_MAX_REPLICA];
@ -51,9 +58,12 @@ typedef struct SVotesRespond {
} SVotesRespond;
SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode);
void votesRespondDestory(SVotesRespond *pVotesRespond);
bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId);
void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg);
void Reset(SVotesRespond *pVotesRespond, SyncTerm term);
void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term);
cJSON * votesRespond2Json(SVotesRespond *pVotesRespond);
char * votesRespond2Str(SVotesRespond *pVotesRespond);
#ifdef __cplusplus
}

25
source/libs/sync/src/syncElection.c
View File

@ -15,6 +15,7 @@
#include "syncElection.h"
#include "syncMessage.h"
#include "syncRaftStore.h"
// TLA+ Spec
// RequestVote(i, j) ==
@ -28,11 +29,31 @@
// mdest |-> j])
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
//
int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {}
int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
int32_t ret = 0;
for (int i = 0; i < pSyncNode->peersNum; ++i) {
SyncRequestVote* pMsg = syncRequestVoteBuild();
pMsg->srcId = pSyncNode->myRaftId;
pMsg->destId = pSyncNode->peersId[i];
pMsg->currentTerm = pSyncNode->pRaftStore->currentTerm;
pMsg->lastLogIndex = pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore);
pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore);
ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg);
assert(ret == 0);
syncRequestVoteDestroy(pMsg);
}
return ret;
}
int32_t syncNodeElect(SSyncNode* pSyncNode) {
assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE);
// start election
syncNodeRequestVotePeers(pSyncNode);
int32_t ret = syncNodeRequestVotePeers(pSyncNode);
return ret;
}
int32_t syncNodeRequestVote(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncRequestVote* pMsg) {

100
source/libs/sync/src/syncIndexMgr.c Normal file
View File

@ -0,0 +1,100 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncIndexMgr.h"
#include "syncUtil.h"
// SMatchIndex -----------------------------
SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) {
SSyncIndexMgr *pSyncIndexMgr = malloc(sizeof(SSyncIndexMgr));
assert(pSyncIndexMgr != NULL);
memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr));
pSyncIndexMgr->replicas = &(pSyncNode->replicasId);
pSyncIndexMgr->replicaNum = pSyncNode->replicaNum;
pSyncIndexMgr->pSyncNode = pSyncNode;
syncIndexMgrClear(pSyncIndexMgr);
return pSyncIndexMgr;
}
void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) {
if (pSyncIndexMgr != NULL) {
free(pSyncIndexMgr);
}
}
void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) {
memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index));
/*
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
pSyncIndexMgr->index[i] = 0;
}
*/
}
void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
(pSyncIndexMgr->index)[i] = index;
return;
}
}
assert(0);
}
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
SyncIndex idx = (pSyncIndexMgr->index)[i];
return idx;
}
}
assert(0);
}
cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
char u64buf[128];
cJSON *pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncIndexMgr->replicaNum);
cJSON *pReplicas = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i]));
}
int respondNum = 0;
int *arr = (int *)malloc(sizeof(int) * pSyncIndexMgr->replicaNum);
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
arr[i] = pSyncIndexMgr->index[i];
}
cJSON *pIndex = cJSON_CreateIntArray(arr, pSyncIndexMgr->replicaNum);
free(arr);
cJSON_AddItemToObject(pRoot, "index", pIndex);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
cJSON *pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "pSyncIndexMgr", pRoot);
return pJson;
}
char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) {
cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr);
char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}

220
source/libs/sync/src/syncMain.c
View File

@ -20,10 +20,12 @@
#include "syncEnv.h"
#include "syncInt.h"
#include "syncRaft.h"
#include "syncRaftStore.h"
#include "syncRequestVote.h"
#include "syncRequestVoteReply.h"
#include "syncTimeout.h"
#include "syncUtil.h"
#include "syncVoteMgr.h"
static int32_t tsNodeRefId = -1;
@ -35,6 +37,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId);
static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
static int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term);
static void syncNodeBecomeFollower(SSyncNode* pSyncNode);
static void syncNodeBecomeLeader(SSyncNode* pSyncNode);
static void syncNodeFollower2Candidate(SSyncNode* pSyncNode);
@ -71,29 +74,60 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
assert(pSyncNode != NULL);
memset(pSyncNode, 0, sizeof(SSyncNode));
// init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId;
pSyncNode->syncCfg = pSyncInfo->syncCfg;
memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path));
pSyncNode->pFsm = pSyncInfo->pFsm;
memcpy(pSyncNode->walPath, pSyncInfo->walPath, sizeof(pSyncNode->walPath));
pSyncNode->rpcClient = pSyncInfo->rpcClient;
pSyncNode->FpSendMsg = pSyncInfo->FpSendMsg;
pSyncNode->queue = pSyncInfo->queue;
pSyncNode->FpEqMsg = pSyncInfo->FpEqMsg;
pSyncNode->me = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex];
pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1;
// init internal
pSyncNode->myNodeInfo = pSyncInfo->syncCfg.nodeInfo[pSyncInfo->syncCfg.myIndex];
syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncInfo->vgId, &pSyncNode->myRaftId);
// init peersNum, peers, peersId
pSyncNode->peersNum = pSyncInfo->syncCfg.replicaNum - 1;
int j = 0;
for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) {
if (i != pSyncInfo->syncCfg.myIndex) {
pSyncNode->peers[j] = pSyncInfo->syncCfg.nodeInfo[i];
pSyncNode->peersNodeInfo[j] = pSyncInfo->syncCfg.nodeInfo[i];
j++;
}
}
for (int i = 0; i < pSyncNode->peersNum; ++i) {
syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncInfo->vgId, &pSyncNode->peersId[i]);
}
// init replicaNum, replicasId
pSyncNode->replicaNum = pSyncInfo->syncCfg.replicaNum;
for (int i = 0; i < pSyncInfo->syncCfg.replicaNum; ++i) {
syncUtilnodeInfo2raftId(&pSyncInfo->syncCfg.nodeInfo[i], pSyncInfo->vgId, &pSyncNode->replicasId[i]);
}
// init raft algorithm
pSyncNode->pFsm = pSyncInfo->pFsm;
pSyncNode->quorum = syncUtilQuorum(pSyncInfo->syncCfg.replicaNum);
pSyncNode->leaderCache = EMPTY_RAFT_ID;
// init life cycle
// init server vars
pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER;
syncUtilnodeInfo2raftId(&pSyncNode->me, pSyncNode->vgId, &pSyncNode->raftId);
pSyncNode->pRaftStore = raftStoreOpen(pSyncInfo->walPath);
assert(pSyncNode->pRaftStore != NULL);
// init candidate vars
pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode);
assert(pSyncNode->pVotesGranted != NULL);
pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode);
assert(pSyncNode->pVotesRespond != NULL);
// init leader vars
pSyncNode->pNextIndex = NULL;
pSyncNode->pMatchIndex = NULL;
// init ping timer
pSyncNode->pPingTimer = NULL;
@ -119,6 +153,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) {
pSyncNode->FpHeartbeatTimer = syncNodeEqHeartbeatTimer;
pSyncNode->heartbeatTimerCounter = 0;
// init callback
pSyncNode->FpOnPing = syncNodeOnPingCb;
pSyncNode->FpOnPingReply = syncNodeOnPingReplyCb;
pSyncNode->FpOnRequestVote = syncNodeOnRequestVoteCb;
@ -135,6 +170,134 @@ void syncNodeClose(SSyncNode* pSyncNode) {
free(pSyncNode);
}
cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
// init by SSyncInfo
cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId);
cJSON_AddStringToObject(pRoot, "path", pSyncNode->path);
cJSON_AddStringToObject(pRoot, "walPath", pSyncNode->walPath);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->rpcClient);
cJSON_AddStringToObject(pRoot, "rpcClient", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpSendMsg);
cJSON_AddStringToObject(pRoot, "FpSendMsg", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->queue);
cJSON_AddStringToObject(pRoot, "queue", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpEqMsg);
cJSON_AddStringToObject(pRoot, "FpEqMsg", u64buf);
// init internal
cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo);
cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe);
cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId);
cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId);
cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum);
cJSON* pPeers = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "peersNodeInfo", pPeers);
for (int i = 0; i < pSyncNode->peersNum; ++i) {
cJSON_AddItemToArray(pPeers, syncUtilNodeInfo2Json(&pSyncNode->peersNodeInfo[i]));
}
cJSON* pPeersId = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "peersId", pPeersId);
for (int i = 0; i < pSyncNode->peersNum; ++i) {
cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i]));
}
cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum);
cJSON* pReplicasId = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId);
for (int i = 0; i < pSyncNode->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i]));
}
// raft algorithm
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm);
cJSON_AddStringToObject(pRoot, "pFsm", u64buf);
cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum);
cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache);
cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache);
// tla+ server vars
cJSON_AddNumberToObject(pRoot, "state", pSyncNode->state);
cJSON_AddStringToObject(pRoot, "state_str", syncUtilState2String(pSyncNode->state));
// tla+ candidate vars
// tla+ leader vars
// tla+ log vars
// ping timer
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pPingTimer);
cJSON_AddStringToObject(pRoot, "pPingTimer", u64buf);
cJSON_AddNumberToObject(pRoot, "pingTimerMS", pSyncNode->pingTimerMS);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClock);
cJSON_AddStringToObject(pRoot, "pingTimerLogicClock", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerLogicClockUser);
cJSON_AddStringToObject(pRoot, "pingTimerLogicClockUser", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpPingTimer);
cJSON_AddStringToObject(pRoot, "FpPingTimer", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->pingTimerCounter);
cJSON_AddStringToObject(pRoot, "pingTimerCounter", u64buf);
// elect timer
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pElectTimer);
cJSON_AddStringToObject(pRoot, "pElectTimer", u64buf);
cJSON_AddNumberToObject(pRoot, "electTimerMS", pSyncNode->electTimerMS);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClock);
cJSON_AddStringToObject(pRoot, "electTimerLogicClock", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerLogicClockUser);
cJSON_AddStringToObject(pRoot, "electTimerLogicClockUser", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpElectTimer);
cJSON_AddStringToObject(pRoot, "FpElectTimer", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->electTimerCounter);
cJSON_AddStringToObject(pRoot, "electTimerCounter", u64buf);
// heartbeat timer
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pHeartbeatTimer);
cJSON_AddStringToObject(pRoot, "pHeartbeatTimer", u64buf);
cJSON_AddNumberToObject(pRoot, "heartbeatTimerMS", pSyncNode->heartbeatTimerMS);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClock);
cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClock", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerLogicClockUser);
cJSON_AddStringToObject(pRoot, "heartbeatTimerLogicClockUser", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpHeartbeatTimer);
cJSON_AddStringToObject(pRoot, "FpHeartbeatTimer", u64buf);
snprintf(u64buf, sizeof(u64buf), "%lu", pSyncNode->heartbeatTimerCounter);
cJSON_AddStringToObject(pRoot, "heartbeatTimerCounter", u64buf);
// callback
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPing);
cJSON_AddStringToObject(pRoot, "FpOnPing", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnPingReply);
cJSON_AddStringToObject(pRoot, "FpOnPingReply", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVote);
cJSON_AddStringToObject(pRoot, "FpOnRequestVote", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnRequestVoteReply);
cJSON_AddStringToObject(pRoot, "FpOnRequestVoteReply", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntries);
cJSON_AddStringToObject(pRoot, "FpOnAppendEntries", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnAppendEntriesReply);
cJSON_AddStringToObject(pRoot, "FpOnAppendEntriesReply", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->FpOnTimeout);
cJSON_AddStringToObject(pRoot, "FpOnTimeout", u64buf);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SSyncNode", pRoot);
return pJson;
}
char* syncNode2Str(const SSyncNode* pSyncNode) {
cJSON* pJson = syncNode2Json(pSyncNode);
char* serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
SEpSet epSet;
syncUtilraftId2EpSet(destRaftId, &epSet);
@ -183,7 +346,7 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode) {
for (int i = 0; i < pSyncNode->syncCfg.replicaNum; ++i) {
SRaftId destId;
syncUtilnodeInfo2raftId(&pSyncNode->syncCfg.nodeInfo[i], pSyncNode->vgId, &destId);
SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId);
SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
ret = syncNodePing(pSyncNode, &destId, pMsg);
assert(ret == 0);
syncPingDestroy(pMsg);
@ -194,8 +357,8 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
int32_t ret = 0;
for (int i = 0; i < pSyncNode->peersNum; ++i) {
SRaftId destId;
syncUtilnodeInfo2raftId(&pSyncNode->peers[i], pSyncNode->vgId, &destId);
SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &destId);
syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &destId);
SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &destId);
ret = syncNodePing(pSyncNode, &destId, pMsg);
assert(ret == 0);
syncPingDestroy(pMsg);
@ -204,7 +367,7 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
int32_t syncNodePingSelf(SSyncNode* pSyncNode) {
int32_t ret;
SyncPing* pMsg = syncPingBuild3(&pSyncNode->raftId, &pSyncNode->raftId);
SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg);
assert(ret == 0);
syncPingDestroy(pMsg);
@ -285,7 +448,7 @@ static int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg) {
cJSON_Delete(pJson);
}
SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->raftId, &pMsg->srcId);
SyncPingReply* pMsgReply = syncPingReplyBuild3(&ths->myRaftId, &pMsg->srcId);
SRpcMsg rpcMsg;
syncPingReply2RpcMsg(pMsgReply, &rpcMsg);
syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg);
@ -351,7 +514,38 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) {
}
}
static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {}
static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) {
SSyncNode* pSyncNode = (SSyncNode*)param;
if (atomic_load_64(&pSyncNode->heartbeatTimerLogicClockUser) <=
atomic_load_64(&pSyncNode->heartbeatTimerLogicClock)) {
SyncTimeout* pSyncMsg =
syncTimeoutBuild2(SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pSyncNode->heartbeatTimerLogicClock),
pSyncNode->heartbeatTimerMS, pSyncNode);
SRpcMsg rpcMsg;
syncTimeout2RpcMsg(pSyncMsg, &rpcMsg);
pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);
syncTimeoutDestroy(pSyncMsg);
// reset timer ms
// pSyncNode->heartbeatTimerMS += 100;
taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, &gSyncEnv->pTimerManager,
&pSyncNode->pHeartbeatTimer);
} else {
sTrace("syncNodeEqHeartbeatTimer: heartbeatTimerLogicClock:%lu, heartbeatTimerLogicClockUser:%lu",
pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser);
}
}
static void UpdateTerm(SSyncNode* pSyncNode, SyncTerm term) {
if (term > pSyncNode->pRaftStore->currentTerm) {
pSyncNode->pRaftStore->currentTerm = term;
pSyncNode->pRaftStore->voteFor = EMPTY_RAFT_ID;
raftStorePersist(pSyncNode->pRaftStore);
syncNodeBecomeFollower(pSyncNode);
}
}
static void syncNodeBecomeFollower(SSyncNode* pSyncNode) {
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
@ -383,7 +577,7 @@ static void syncNodeBecomeFollower(SSyncNode* pSyncNode) {
//
static void syncNodeBecomeLeader(SSyncNode* pSyncNode) {
pSyncNode->state = TAOS_SYNC_STATE_LEADER;
pSyncNode->leaderCache = pSyncNode->raftId;
pSyncNode->leaderCache = pSyncNode->myRaftId;
// next Index +=1
// match Index = 0;

11
source/libs/sync/src/syncReplication.c
View File

@ -41,7 +41,16 @@
// mdest |-> j])
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
//
int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {}
int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
int32_t ret = 0;
return ret;
}
int32_t syncNodeReplicate(SSyncNode* pSyncNode) {
// start replicate
int32_t ret = syncNodeAppendEntriesPeers(pSyncNode);
return ret;
}
int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg) {
sTrace("syncNodeAppendEntries pSyncNode:%p ", pSyncNode);

2
source/libs/sync/src/syncTimeout.c
View File

@ -44,7 +44,7 @@ int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg) {
} else if (pMsg->timeoutType == SYNC_TIMEOUT_HEARTBEAT) {
if (atomic_load_64(&ths->heartbeatTimerLogicClockUser) <= pMsg->logicClock) {
++(ths->heartbeatTimerCounter);
syncNodeAppendEntriesPeers(ths);
syncNodeReplicate(ths);
}
} else {
sTrace("unknown timeoutType:%d", pMsg->timeoutType);

53
source/libs/sync/src/syncUtil.c
View File

@ -97,4 +97,55 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) {
int32_t syncUtilRand(int32_t max) { return rand() % max; }
int32_t syncUtilElectRandomMS() { ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); }
int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; }
cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
cJSON_AddStringToObject(pRoot, "nodeFqdn", p->nodeFqdn);
cJSON_AddNumberToObject(pRoot, "nodePort", p->nodePort);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SNodeInfo", pRoot);
return pJson;
}
cJSON* syncUtilRaftId2Json(const SRaftId* p) {
char u64buf[128];
cJSON* pRoot = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%lu", p->addr);
cJSON_AddStringToObject(pRoot, "addr", u64buf);
char host[128];
uint16_t port;
syncUtilU642Addr(p->addr, host, sizeof(host), &port);
cJSON_AddStringToObject(pRoot, "host", host);
cJSON_AddNumberToObject(pRoot, "port", port);
cJSON_AddNumberToObject(pRoot, "vgId", p->vgId);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SRaftId", pRoot);
return pJson;
}
char* syncUtilRaftId2Str(const SRaftId* p) {
cJSON* pJson = syncUtilRaftId2Json(p);
char* serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
const char* syncUtilState2String(ESyncState state) {
if (state == TAOS_SYNC_STATE_FOLLOWER) {
return "TAOS_SYNC_STATE_FOLLOWER";
} else if (state == TAOS_SYNC_STATE_CANDIDATE) {
return "TAOS_SYNC_STATE_CANDIDATE";
} else if (state == TAOS_SYNC_STATE_LEADER) {
return "TAOS_SYNC_STATE_LEADER";
} else {
return "TAOS_SYNC_STATE_UNKNOWN";
}
}

137
source/libs/sync/src/syncVoteMgr.c
View File

@ -14,15 +14,25 @@
*/
#include "syncVoteMgr.h"
#include "syncUtil.h"
// SVotesGranted -----------------------------
static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) {
memset(pVotesGranted->isGranted, 0, sizeof(pVotesGranted->isGranted));
pVotesGranted->votes = 0;
}
SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) {
SVotesGranted *pVotesGranted = malloc(sizeof(SVotesGranted));
assert(pVotesGranted != NULL);
memset(pVotesGranted, 0, sizeof(SVotesGranted));
pVotesGranted->quorum = pSyncNode->quorum;
pVotesGranted->replicas = &(pSyncNode->replicasId);
pVotesGranted->replicaNum = pSyncNode->replicaNum;
voteGrantedClearVotes(pVotesGranted);
pVotesGranted->term = 0;
pVotesGranted->votes = 0;
pVotesGranted->quorum = pSyncNode->quorum;
pVotesGranted->toLeader = false;
pVotesGranted->pSyncNode = pSyncNode;
@ -43,15 +53,73 @@ bool voteGrantedMajority(SVotesGranted *pVotesGranted) {
void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) {
assert(pMsg->voteGranted == true);
assert(pMsg->term == pVotesGranted->term);
pVotesGranted->votes++;
assert(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId));
int j = -1;
for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
if (syncUtilSameId(&((*(pVotesGranted->replicas))[i]), &(pMsg->srcId))) {
j = i;
break;
}
}
assert(j != -1);
assert(j >= 0 && j < pVotesGranted->replicaNum);
if (pVotesGranted->isGranted[j] != true) {
++(pVotesGranted->votes);
pVotesGranted->isGranted[j] = true;
}
assert(pVotesGranted->votes <= pVotesGranted->replicaNum);
}
void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) {
pVotesGranted->term = term;
pVotesGranted->votes = 0;
voteGrantedClearVotes(pVotesGranted);
pVotesGranted->toLeader = false;
}
cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
char u64buf[128];
cJSON *pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesGranted->replicaNum);
cJSON *pReplicas = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i]));
}
int *arr = (int *)malloc(sizeof(int) * pVotesGranted->replicaNum);
for (int i = 0; i < pVotesGranted->replicaNum; ++i) {
arr[i] = pVotesGranted->isGranted[i];
}
cJSON *pIsGranted = cJSON_CreateIntArray(arr, pVotesGranted->replicaNum);
free(arr);
cJSON_AddItemToObject(pRoot, "isGranted", pIsGranted);
cJSON_AddNumberToObject(pRoot, "votes", pVotesGranted->votes);
snprintf(u64buf, sizeof(u64buf), "%lu", pVotesGranted->term);
cJSON_AddStringToObject(pRoot, "term", u64buf);
cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum);
cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader);
snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
bool majority = voteGrantedMajority(pVotesGranted);
cJSON_AddNumberToObject(pRoot, "majority", majority);
cJSON *pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SVotesGranted", pRoot);
return pJson;
}
char *voteGranted2Str(SVotesGranted *pVotesGranted) {
cJSON *pJson = voteGranted2Json(pVotesGranted);
char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
// SVotesRespond -----------------------------
SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) {
SVotesRespond *pVotesRespond = malloc(sizeof(SVotesRespond));
assert(pVotesRespond != NULL);
@ -65,6 +133,12 @@ SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) {
return pVotesRespond;
}
void votesRespondDestory(SVotesRespond *pVotesRespond) {
if (pVotesRespond != NULL) {
free(pVotesRespond);
}
}
bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) {
bool ret = false;
for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
@ -79,8 +153,8 @@ bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) {
void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg) {
assert(pVotesRespond->term == pMsg->term);
for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
if (syncUtilSameId(&(*pVotesRespond->replicas)[i], &pMsg->srcId)) {
assert(pVotesRespond->isRespond[i] == false);
if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) {
// assert(pVotesRespond->isRespond[i] == false);
pVotesRespond->isRespond[i] = true;
return;
}
@ -88,9 +162,52 @@ void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *p
assert(0);
}
void Reset(SVotesRespond *pVotesRespond, SyncTerm term) {
void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) {
pVotesRespond->term = term;
for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
pVotesRespond->isRespond[i] = false;
}
memset(pVotesRespond->isRespond, 0, sizeof(pVotesRespond->isRespond));
/*
for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
pVotesRespond->isRespond[i] = false;
}
*/
}
cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) {
char u64buf[128];
cJSON *pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "replicaNum", pVotesRespond->replicaNum);
cJSON *pReplicas = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i]));
}
int respondNum = 0;
int *arr = (int *)malloc(sizeof(int) * pVotesRespond->replicaNum);
for (int i = 0; i < pVotesRespond->replicaNum; ++i) {
arr[i] = pVotesRespond->isRespond[i];
if (pVotesRespond->isRespond[i]) {
respondNum++;
}
}
cJSON *pIsRespond = cJSON_CreateIntArray(arr, pVotesRespond->replicaNum);
free(arr);
cJSON_AddItemToObject(pRoot, "isRespond", pIsRespond);
cJSON_AddNumberToObject(pRoot, "respondNum", respondNum);
snprintf(u64buf, sizeof(u64buf), "%lu", pVotesRespond->term);
cJSON_AddStringToObject(pRoot, "term", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
cJSON *pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SVotesRespond", pRoot);
return pJson;
}
char *votesRespond2Str(SVotesRespond *pVotesRespond) {
cJSON *pJson = votesRespond2Json(pVotesRespond);
char * serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}

70
source/libs/sync/test/CMakeLists.txt
View File

@ -10,6 +10,11 @@ add_executable(syncIOSendMsgServerTest "")
add_executable(syncRaftStoreTest "")
add_executable(syncEnqTest "")
add_executable(syncIndexTest "")
add_executable(syncInitTest "")
add_executable(syncUtilTest "")
add_executable(syncVotesGrantedTest "")
add_executable(syncVotesRespondTest "")
add_executable(syncIndexMgrTest "")
target_sources(syncTest
@ -60,6 +65,26 @@ target_sources(syncIndexTest
PRIVATE
"syncIndexTest.cpp"
)
target_sources(syncInitTest
PRIVATE
"syncInitTest.cpp"
)
target_sources(syncUtilTest
PRIVATE
"syncUtilTest.cpp"
)
target_sources(syncVotesGrantedTest
PRIVATE
"syncVotesGrantedTest.cpp"
)
target_sources(syncVotesRespondTest
PRIVATE
"syncVotesRespondTest.cpp"
)
target_sources(syncIndexMgrTest
PRIVATE
"syncIndexMgrTest.cpp"
)
target_include_directories(syncTest
@ -122,6 +147,31 @@ target_include_directories(syncIndexTest
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncInitTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncUtilTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncVotesGrantedTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncVotesRespondTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_include_directories(syncIndexMgrTest
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries(syncTest
@ -172,6 +222,26 @@ target_link_libraries(syncIndexTest
sync
gtest_main
)
target_link_libraries(syncInitTest
sync
gtest_main
)
target_link_libraries(syncUtilTest
sync
gtest_main
)
target_link_libraries(syncVotesGrantedTest
sync
gtest_main
)
target_link_libraries(syncVotesRespondTest
sync
gtest_main
)
target_link_libraries(syncIndexMgrTest
sync
gtest_main
)
enable_testing()

2
source/libs/sync/test/syncEnqTest.cpp
View File

@ -84,7 +84,7 @@ int main(int argc, char** argv) {
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
for (int i = 0; i < 10; ++i) {
SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->raftId, &pSyncNode->raftId);
SyncPingReply* pSyncMsg = syncPingReplyBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId);
SRpcMsg rpcMsg;
syncPingReply2RpcMsg(pSyncMsg, &rpcMsg);
pSyncNode->FpEqMsg(pSyncNode->queue, &rpcMsg);

141
source/libs/sync/test/syncIndexMgrTest.cpp Normal file
View File

@ -0,0 +1,141 @@
#include "syncIndexMgr.h"
//#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
#include "syncVoteMgr.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 3;
int32_t myIndex = 0;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM* pFsm;
SSyncNode* pSyncNode;
SSyncNode* syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = replicaNum;
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
SSyncNode* syncInitTest() { return syncNodeInit(); }
void initRaftId(SSyncNode* pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char* s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
int main(int argc, char** argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
SSyncNode* pSyncNode = syncInitTest();
assert(pSyncNode != NULL);
char* serialized = syncNode2Str(pSyncNode);
printf("%s\n", serialized);
free(serialized);
initRaftId(pSyncNode);
SSyncIndexMgr* pSyncIndexMgr = syncIndexMgrCreate(pSyncNode);
assert(pSyncIndexMgr != NULL);
printf("---------------------------------------\n");
{
char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
syncIndexMgrSetIndex(pSyncIndexMgr, &ids[0], 100);
syncIndexMgrSetIndex(pSyncIndexMgr, &ids[1], 200);
syncIndexMgrSetIndex(pSyncIndexMgr, &ids[2], 300);
printf("---------------------------------------\n");
{
char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
printf("---------------------------------------\n");
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
SyncIndex idx = syncIndexMgrGetIndex(pSyncIndexMgr, &ids[i]);
printf("index %d : %lu \n", i, idx);
}
syncIndexMgrClear(pSyncIndexMgr);
printf("---------------------------------------\n");
{
char* serialized = syncIndexMgr2Str(pSyncIndexMgr);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
syncIndexMgrDestroy(pSyncIndexMgr);
return 0;
}

99
source/libs/sync/test/syncInitTest.cpp Normal file
View File

@ -0,0 +1,99 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 5;
int32_t myIndex = 0;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM* pFsm;
SSyncNode* syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = replicaNum;
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
SSyncNode* pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
SSyncNode* syncInitTest() { return syncNodeInit(); }
void initRaftId(SSyncNode* pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char* s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
int main(int argc, char** argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
SSyncNode* pSyncNode = syncInitTest();
assert(pSyncNode != NULL);
char* serialized = syncNode2Str(pSyncNode);
printf("%s\n", serialized);
free(serialized);
initRaftId(pSyncNode);
return 0;
}

3
source/libs/sync/test/syncPingTest.cpp
View File

@ -25,7 +25,8 @@ SSyncNode* doSync(int myIndex) {
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_sync_ping");
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./wal_path");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;

32
source/libs/sync/test/syncUtilTest.cpp Normal file
View File

@ -0,0 +1,32 @@
#include "syncUtil.h"
//#include <gtest/gtest.h>
#include <stdio.h>
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
void electRandomMSTest() {
for (int i = 0; i < 10; ++i) {
int32_t ms = syncUtilElectRandomMS();
printf("syncUtilElectRandomMS: %d \n", ms);
}
}
int main() {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
logTest();
electRandomMSTest();
return 0;
}

156
source/libs/sync/test/syncVotesGrantedTest.cpp Normal file
View File

@ -0,0 +1,156 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
#include "syncVoteMgr.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 3;
int32_t myIndex = 0;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM* pFsm;
SSyncNode* pSyncNode;
SSyncNode* syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = replicaNum;
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
SSyncNode* syncInitTest() { return syncNodeInit(); }
void initRaftId(SSyncNode* pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char* s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
int main(int argc, char** argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
SSyncNode* pSyncNode = syncInitTest();
assert(pSyncNode != NULL);
char* serialized = syncNode2Str(pSyncNode);
printf("%s\n", serialized);
free(serialized);
initRaftId(pSyncNode);
SVotesGranted* pVotesGranted = voteGrantedCreate(pSyncNode);
assert(pVotesGranted != NULL);
printf("---------------------------------------\n");
{
char* serialized = voteGranted2Str(pVotesGranted);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
SyncTerm term = 1234;
printf("---------------------------------------\n");
voteGrantedReset(pVotesGranted, term);
{
char* serialized = voteGranted2Str(pVotesGranted);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
for (int i = 0; i < replicaNum; ++i) {
SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild();
reply->destId = pSyncNode->myRaftId;
reply->srcId = ids[i];
reply->term = term;
reply->voteGranted = true;
voteGrantedVote(pVotesGranted, reply);
{
char* serialized = voteGranted2Str(pVotesGranted);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
voteGrantedVote(pVotesGranted, reply);
{
char* serialized = voteGranted2Str(pVotesGranted);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
}
printf("---------------------------------------\n");
voteGrantedReset(pVotesGranted, 123456789);
{
char* serialized = voteGranted2Str(pVotesGranted);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
voteGrantedDestroy(pVotesGranted);
return 0;
}

156
source/libs/sync/test/syncVotesRespondTest.cpp Normal file
View File

@ -0,0 +1,156 @@
#include <gtest/gtest.h>
#include <stdio.h>
#include "syncEnv.h"
#include "syncIO.h"
#include "syncInt.h"
#include "syncRaftStore.h"
#include "syncUtil.h"
#include "syncVoteMgr.h"
void logTest() {
sTrace("--- sync log test: trace");
sDebug("--- sync log test: debug");
sInfo("--- sync log test: info");
sWarn("--- sync log test: warn");
sError("--- sync log test: error");
sFatal("--- sync log test: fatal");
}
uint16_t ports[] = {7010, 7110, 7210, 7310, 7410};
int32_t replicaNum = 3;
int32_t myIndex = 0;
SRaftId ids[TSDB_MAX_REPLICA];
SSyncInfo syncInfo;
SSyncFSM* pFsm;
SSyncNode* pSyncNode;
SSyncNode* syncNodeInit() {
syncInfo.vgId = 1234;
syncInfo.rpcClient = gSyncIO->clientRpc;
syncInfo.FpSendMsg = syncIOSendMsg;
syncInfo.queue = gSyncIO->pMsgQ;
syncInfo.FpEqMsg = syncIOEqMsg;
syncInfo.pFsm = pFsm;
snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./test_path");
snprintf(syncInfo.walPath, sizeof(syncInfo.walPath), "%s", "./test_wal_path");
SSyncCfg* pCfg = &syncInfo.syncCfg;
pCfg->myIndex = myIndex;
pCfg->replicaNum = replicaNum;
for (int i = 0; i < replicaNum; ++i) {
pCfg->nodeInfo[i].nodePort = ports[i];
snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1");
// taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
}
pSyncNode = syncNodeOpen(&syncInfo);
assert(pSyncNode != NULL);
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote;
gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply;
gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries;
gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply;
gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing;
gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply;
gSyncIO->pSyncNode = pSyncNode;
return pSyncNode;
}
SSyncNode* syncInitTest() { return syncNodeInit(); }
void initRaftId(SSyncNode* pSyncNode) {
for (int i = 0; i < replicaNum; ++i) {
ids[i] = pSyncNode->replicasId[i];
char* s = syncUtilRaftId2Str(&ids[i]);
printf("raftId[%d] : %s\n", i, s);
free(s);
}
}
int main(int argc, char** argv) {
// taosInitLog((char *)"syncTest.log", 100000, 10);
tsAsyncLog = 0;
sDebugFlag = 143 + 64;
myIndex = 0;
if (argc >= 2) {
myIndex = atoi(argv[1]);
}
int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]);
assert(ret == 0);
ret = syncEnvStart();
assert(ret == 0);
SSyncNode* pSyncNode = syncInitTest();
assert(pSyncNode != NULL);
char* serialized = syncNode2Str(pSyncNode);
printf("%s\n", serialized);
free(serialized);
initRaftId(pSyncNode);
SVotesRespond* pVotesRespond = votesRespondCreate(pSyncNode);
assert(pVotesRespond != NULL);
printf("---------------------------------------\n");
{
char* serialized = votesRespond2Str(pVotesRespond);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
SyncTerm term = 1234;
printf("---------------------------------------\n");
votesRespondReset(pVotesRespond, term);
{
char* serialized = votesRespond2Str(pVotesRespond);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
for (int i = 0; i < replicaNum; ++i) {
SyncRequestVoteReply* reply = SyncRequestVoteReplyBuild();
reply->destId = pSyncNode->myRaftId;
reply->srcId = ids[i];
reply->term = term;
reply->voteGranted = true;
votesRespondAdd(pVotesRespond, reply);
{
char* serialized = votesRespond2Str(pVotesRespond);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
votesRespondAdd(pVotesRespond, reply);
{
char* serialized = votesRespond2Str(pVotesRespond);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
}
printf("---------------------------------------\n");
votesRespondReset(pVotesRespond, 123456789);
{
char* serialized = votesRespond2Str(pVotesRespond);
assert(serialized != NULL);
printf("%s\n", serialized);
free(serialized);
}
votesRespondDestory(pVotesRespond);
return 0;
}

2
source/libs/tfs/inc/tfsInt.h
View File

@ -59,7 +59,7 @@ typedef struct STfsDir {
SDiskID did;
char dirname[TSDB_FILENAME_LEN];
STfsFile tfile;
DIR *dir;
TdDirPtr pDir;
STfs *pTfs;
} STfsDir;

67
source/libs/tfs/src/tfs.c
View File

@ -192,14 +192,14 @@ void tfsBasename(const STfsFile *pFile, char *dest) {
char tname[TSDB_FILENAME_LEN] = "\0";
tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN);
tstrncpy(dest, basename(tname), TSDB_FILENAME_LEN);
tstrncpy(dest, taosDirEntryBaseName(tname), TSDB_FILENAME_LEN);
}
void tfsDirname(const STfsFile *pFile, char *dest) {
char tname[TSDB_FILENAME_LEN] = "\0";
tstrncpy(tname, pFile->aname, TSDB_FILENAME_LEN);
tstrncpy(dest, dirname(tname), TSDB_FILENAME_LEN);
tstrncpy(dest, taosDirName(tname), TSDB_FILENAME_LEN);
}
int32_t tfsRemoveFile(const STfsFile *pFile) { return taosRemoveFile(pFile->aname); }
@ -233,7 +233,7 @@ int32_t tfsMkdirRecurAt(STfs *pTfs, const char *rname, SDiskID diskId) {
// the pointer directly in this recursion.
// See
// https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/dirname.3.html
char *dir = strdup(dirname(s));
char *dir = strdup(taosDirName(s));
if (tfsMkdirRecurAt(pTfs, dir, diskId) < 0) {
free(s);
@ -324,45 +324,46 @@ STfsDir *tfsOpendir(STfs *pTfs, const char *rname) {
return pDir;
}
const STfsFile *tfsReaddir(STfsDir *pDir) {
if (pDir == NULL || pDir->dir == NULL) return NULL;
const STfsFile *tfsReaddir(STfsDir *pTfsDir) {
if (pTfsDir == NULL || pTfsDir->pDir == NULL) return NULL;
char bname[TMPNAME_LEN * 2] = "\0";
while (true) {
struct dirent *dp = NULL;
dp = readdir(pDir->dir);
if (dp != NULL) {
TdDirEntryPtr pDirEntry = NULL;
pDirEntry = taosReadDir(pTfsDir->pDir);
if (pDirEntry != NULL) {
// Skip . and ..
if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
char *name = taosGetDirEntryName(pDirEntry);
if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
if (pDir->dirname == NULL || pDir->dirname[0] == 0) {
snprintf(bname, TMPNAME_LEN * 2, "%s", dp->d_name);
if (pTfsDir->dirname == NULL || pTfsDir->dirname[0] == 0) {
snprintf(bname, TMPNAME_LEN * 2, "%s", name);
} else {
snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pDir->dirname, TD_DIRSEP, dp->d_name);
snprintf(bname, TMPNAME_LEN * 2, "%s%s%s", pTfsDir->dirname, TD_DIRSEP, name);
}
tfsInitFile(pDir->pTfs, &pDir->tfile, pDir->did, bname);
return &pDir->tfile;
tfsInitFile(pTfsDir->pTfs, &pTfsDir->tfile, pTfsDir->did, bname);
return &pTfsDir->tfile;
}
if (tfsOpendirImpl(pDir->pTfs, pDir) < 0) {
if (tfsOpendirImpl(pTfsDir->pTfs, pTfsDir) < 0) {
return NULL;
}
if (pDir->dir == NULL) {
if (pTfsDir->pDir == NULL) {
terrno = TSDB_CODE_SUCCESS;
return NULL;
}
}
}
void tfsClosedir(STfsDir *pDir) {
if (pDir) {
if (pDir->dir != NULL) {
closedir(pDir->dir);
pDir->dir = NULL;
void tfsClosedir(STfsDir *pTfsDir) {
if (pTfsDir) {
if (pTfsDir->pDir != NULL) {
taosCloseDir(pTfsDir->pDir);
pTfsDir->pDir = NULL;
}
free(pDir);
free(pTfsDir);
}
}
@ -487,29 +488,29 @@ static STfsDisk *tfsGetDiskByName(STfs *pTfs, const char *dir) {
return pDisk;
}
static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pDir) {
static int32_t tfsOpendirImpl(STfs *pTfs, STfsDir *pTfsDir) {
STfsDisk *pDisk = NULL;
char adir[TMPNAME_LEN * 2] = "\0";
if (pDir->dir != NULL) {
closedir(pDir->dir);
pDir->dir = NULL;
if (pTfsDir->pDir != NULL) {
taosCloseDir(pTfsDir->pDir);
pTfsDir->pDir = NULL;
}
while (true) {
pDisk = tfsNextDisk(pTfs, &pDir->iter);
pDisk = tfsNextDisk(pTfs, &pTfsDir->iter);
if (pDisk == NULL) return 0;
pDir->did.level = pDisk->level;
pDir->did.id = pDisk->id;
pTfsDir->did.level = pDisk->level;
pTfsDir->did.id = pDisk->id;
if (pDisk->path == NULL || pDisk->path[0] == 0) {
snprintf(adir, TMPNAME_LEN * 2, "%s", pDir->dirname);
snprintf(adir, TMPNAME_LEN * 2, "%s", pTfsDir->dirname);
} else {
snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pDir->dirname);
snprintf(adir, TMPNAME_LEN * 2, "%s%s%s", pDisk->path, TD_DIRSEP, pTfsDir->dirname);
}
pDir->dir = opendir(adir);
if (pDir->dir != NULL) break;
pTfsDir->pDir = taosOpenDir(adir);
if (pTfsDir->pDir != NULL) break;
}
return 0;

1
source/libs/transport/CMakeLists.txt
View File

@ -12,6 +12,7 @@ target_link_libraries(
PUBLIC os
PUBLIC util
PUBLIC common
PUBLIC zlib
)
if (${BUILD_WITH_UV_TRANS})
if (${BUILD_WITH_UV})

120
source/libs/transport/src/rpcTcp.c
View File

@ -24,7 +24,7 @@
#ifndef USE_UV
typedef struct SFdObj {
void * signature;
SOCKET fd; // TCP socket FD
TdSocketPtr pSocket; // TCP socket FD
void * thandle; // handle from upper layer, like TAOS
uint32_t ip;
uint16_t port;
@ -40,7 +40,7 @@ typedef struct SThreadObj {
pthread_mutex_t mutex;
uint32_t ip;
bool stop;
EpollFd pollFd;
TdEpollPtr pEpoll;
int numOfFds;
int threadId;
char label[TSDB_LABEL_LEN];
@ -56,20 +56,20 @@ typedef struct {
} SClientObj;
typedef struct {
SOCKET fd;
uint32_t ip;
uint16_t port;
int8_t stop;
int8_t reserve;
char label[TSDB_LABEL_LEN];
int numOfThreads;
void * shandle;
SThreadObj **pThreadObj;
pthread_t thread;
TdSocketServerPtr pSocketServer;
uint32_t ip;
uint16_t port;
int8_t stop;
int8_t reserve;
char label[TSDB_LABEL_LEN];
int numOfThreads;
void * shandle;
SThreadObj **pThreadObj;
pthread_t thread;
} SServerObj;
static void * taosProcessTcpData(void *param);
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd);
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket);
static void taosFreeFdObj(SFdObj *pFdObj);
static void taosReportBrokenLink(SFdObj *pFdObj);
static void * taosAcceptTcpConnection(void *arg);
@ -85,7 +85,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
return NULL;
}
pServerObj->fd = -1;
pServerObj->pSocketServer = NULL;
taosResetPthread(&pServerObj->thread);
pServerObj->ip = ip;
pServerObj->port = port;
@ -118,7 +118,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
}
pServerObj->pThreadObj[i] = pThreadObj;
pThreadObj->pollFd = -1;
pThreadObj->pEpoll = NULL;
taosResetPthread(&pThreadObj->thread);
pThreadObj->processData = fp;
tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label));
@ -135,8 +135,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
break;
}
pThreadObj->pollFd = (EpollFd)epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter
if (pThreadObj->pEpoll == NULL) {
tError("%s failed to create TCP epoll", label);
code = -1;
break;
@ -151,8 +151,8 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread
pThreadObj->threadId = i;
}
pServerObj->fd = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (pServerObj->fd < 0) code = -1;
pServerObj->pSocketServer = taosOpenTcpServerSocket(pServerObj->ip, pServerObj->port);
if (pServerObj->pSocketServer == NULL) code = -1;
if (code == 0) {
code = pthread_create(&pServerObj->thread, &thattr, taosAcceptTcpConnection, (void *)pServerObj);
@ -196,8 +196,8 @@ void taosStopTcpServer(void *handle) {
if (pServerObj == NULL) return;
pServerObj->stop = 1;
if (pServerObj->fd >= 0) {
taosShutDownSocketRD(pServerObj->fd);
if (pServerObj->pSocketServer != NULL) {
taosShutDownSocketServerRD(pServerObj->pSocketServer);
}
if (taosCheckPthreadValid(pServerObj->thread)) {
if (taosComparePthread(pServerObj->thread, pthread_self())) {
@ -227,7 +227,7 @@ void taosCleanUpTcpServer(void *handle) {
}
static void *taosAcceptTcpConnection(void *arg) {
SOCKET connFd = -1;
TdSocketPtr pSocket = NULL;
struct sockaddr_in caddr;
int threadId = 0;
SThreadObj * pThreadObj;
@ -239,13 +239,13 @@ static void *taosAcceptTcpConnection(void *arg) {
while (1) {
socklen_t addrlen = sizeof(caddr);
connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen);
pSocket = taosAcceptTcpConnectSocket(pServerObj->pSocketServer, (struct sockaddr *)&caddr, &addrlen);
if (pServerObj->stop) {
tDebug("%s TCP server stop accepting new connections", pServerObj->label);
break;
}
if (connFd == -1) {
if (pSocket == NULL) {
if (errno == EINVAL) {
tDebug("%s TCP server stop accepting new connections, exiting", pServerObj->label);
break;
@ -255,11 +255,11 @@ static void *taosAcceptTcpConnection(void *arg) {
continue;
}
taosKeepTcpAlive(connFd);
taosKeepTcpAlive(pSocket);
struct timeval to = {5, 0};
int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
int32_t ret = taosSetSockOpt(pSocket, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
if (ret != 0) {
taosCloseSocket(connFd);
taosCloseSocket(&pSocket);
tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno),
taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port));
continue;
@ -268,14 +268,14 @@ static void *taosAcceptTcpConnection(void *arg) {
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj[threadId];
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd);
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket);
if (pFdObj) {
pFdObj->ip = caddr.sin_addr.s_addr;
pFdObj->port = htons(caddr.sin_port);
tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label,
taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds);
tDebug("%s new TCP connection from %s:%hu, FD:%p numOfFds:%d", pServerObj->label,
taosInetNtoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds);
} else {
taosCloseSocket(connFd);
taosCloseSocket(&pSocket);
tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno),
taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port));
}
@ -285,7 +285,7 @@ static void *taosAcceptTcpConnection(void *arg) {
threadId = threadId % pServerObj->numOfThreads;
}
taosCloseSocket(pServerObj->fd);
taosCloseSocketServer(&pServerObj->pSocketServer);
return NULL;
}
@ -339,8 +339,8 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread
break;
}
pThreadObj->pollFd = (int64_t)epoll_create(10); // size does not matter
if (pThreadObj->pollFd < 0) {
pThreadObj->pEpoll = taosCreateEpoll(10); // size does not matter
if (pThreadObj->pEpoll == NULL) {
tError("%s failed to create TCP epoll", label);
code = -1;
break;
@ -388,21 +388,17 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
atomic_store_32(&pClientObj->index, index + 1);
SThreadObj *pThreadObj = pClientObj->pThreadObj[index];
SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
if (fd == (SOCKET)-1) return NULL;
#else
if (fd <= 0) return NULL;
#endif
TdSocketPtr pSocket = taosOpenTcpClientSocket(ip, port, pThreadObj->ip);
if (pSocket == NULL) return NULL;
struct sockaddr_in sin;
uint16_t localPort = 0;
unsigned int addrlen = sizeof(sin);
if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
if (taosGetSocketName(pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) {
localPort = (uint16_t)ntohs(sin.sin_port);
}
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd);
SFdObj *pFdObj = taosMallocFdObj(pThreadObj, pSocket);
if (pFdObj) {
pFdObj->thandle = thandle;
@ -415,7 +411,7 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin
ipport, localPort, pFdObj, pThreadObj->numOfFds);
} else {
tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno));
taosCloseSocket(fd);
taosCloseSocket(&pSocket);
}
return pFdObj;
@ -430,7 +426,7 @@ void taosCloseTcpConnection(void *chandle) {
// pFdObj->thandle = NULL;
pFdObj->closedByApp = 1;
taosShutDownSocketWR(pFdObj->fd);
taosShutDownSocketWR(pFdObj->pSocket);
}
int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) {
@ -438,8 +434,8 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand
if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1;
SThreadObj *pThreadObj = pFdObj->pThreadObj;
int ret = taosWriteMsg(pFdObj->fd, data, len);
tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret);
int ret = taosWriteMsg(pFdObj->pSocket, data, len);
tTrace("%s %p TCP data is sent, FD:%p bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, ret);
return ret;
}
@ -449,7 +445,7 @@ static void taosReportBrokenLink(SFdObj *pFdObj) {
// notify the upper layer, so it will clean the associated context
if (pFdObj->closedByApp == 0) {
taosShutDownSocketWR(pFdObj->fd);
taosShutDownSocketWR(pFdObj->pSocket);
SRecvInfo recvInfo;
recvInfo.msg = NULL;
@ -473,7 +469,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
SThreadObj *pThreadObj = pFdObj->pThreadObj;
headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead));
headLen = taosReadMsg(pFdObj->pSocket, &rpcHead, sizeof(SRpcHead));
if (headLen != sizeof(SRpcHead)) {
tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen);
return -1;
@ -486,13 +482,12 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) {
tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen);
return -1;
} else {
tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd,
buffer);
tTrace("%s %p read data, FD:%p TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, buffer);
}
msg = buffer + tsRpcOverhead;
leftLen = msgLen - headLen;
retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen);
retLen = taosReadMsg(pFdObj->pSocket, msg + headLen, leftLen);
if (leftLen != retLen) {
tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj);
@ -532,7 +527,7 @@ static void *taosProcessTcpData(void *param) {
setThreadName(name);
while (1) {
int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
int fdNum = taosWaitEpoll(pThreadObj->pEpoll, events, maxEvents, TAOS_EPOLL_WAIT_TIME);
if (pThreadObj->stop) {
tDebug("%s TCP thread get stop event, exiting...", pThreadObj->label);
break;
@ -561,7 +556,7 @@ static void *taosProcessTcpData(void *param) {
}
if (taosReadTcpData(pFdObj, &recvInfo) < 0) {
shutdown(pFdObj->fd, SHUT_WR);
taosShutDownSocketWR(pFdObj->pSocket);
continue;
}
@ -572,9 +567,9 @@ static void *taosProcessTcpData(void *param) {
if (pThreadObj->stop) break;
}
if (pThreadObj->pollFd >= 0) {
EpollClose(pThreadObj->pollFd);
pThreadObj->pollFd = -1;
if (pThreadObj->pEpoll != NULL) {
taosCloseEpoll(&pThreadObj->pEpoll);
pThreadObj->pEpoll = NULL;
}
while (pThreadObj->pHead) {
@ -590,7 +585,7 @@ static void *taosProcessTcpData(void *param) {
return NULL;
}
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, TdSocketPtr pSocket) {
struct epoll_event event;
SFdObj *pFdObj = (SFdObj *)calloc(sizeof(SFdObj), 1);
@ -599,13 +594,13 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) {
}
pFdObj->closedByApp = 0;
pFdObj->fd = fd;
pFdObj->pSocket = pSocket;
pFdObj->pThreadObj = pThreadObj;
pFdObj->signature = pFdObj;
event.events = EPOLLIN | EPOLLRDHUP;
event.data.ptr = pFdObj;
if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
if (taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_ADD, pSocket, &event) < 0) {
tfree(pFdObj);
terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
@ -635,8 +630,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
}
pFdObj->signature = NULL;
epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_DEL, pFdObj->fd, NULL);
taosCloseSocket(pFdObj->fd);
taosCtlEpoll(pThreadObj->pEpoll, EPOLL_CTL_DEL, pFdObj->pSocket, NULL);
taosCloseSocket(&pFdObj->pSocket);
pThreadObj->numOfFds--;
if (pThreadObj->numOfFds < 0)
@ -655,8 +650,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) {
pthread_mutex_unlock(&pThreadObj->mutex);
tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj,
pFdObj->fd, pThreadObj->numOfFds);
tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds);
tfree(pFdObj);
}

45
source/libs/transport/src/rpcUdp.c
View File

@ -30,17 +30,17 @@
#define RPC_MAX_UDP_SIZE 65480
typedef struct {
int index;
SOCKET fd;
uint16_t port; // peer port
uint16_t localPort; // local port
char label[TSDB_LABEL_LEN]; // copy from udpConnSet;
pthread_t thread;
void * hash;
void * shandle; // handle passed by upper layer during server initialization
void * pSet;
void *(*processData)(SRecvInfo *pRecv);
char *buffer; // buffer to receive data
int index;
TdSocketPtr pSocket;
uint16_t port; // peer port
uint16_t localPort; // local port
char label[TSDB_LABEL_LEN]; // copy from udpConnSet;
pthread_t thread;
void *hash;
void *shandle; // handle passed by upper layer during server initialization
void *pSet;
void *(*processData)(SRecvInfo *pRecv);
char *buffer; // buffer to receive data
} SUdpConn;
typedef struct {
@ -86,8 +86,8 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
for (i = 0; i < threads; ++i) {
pConn = pSet->udpConn + i;
ownPort = (port ? port + i : 0);
pConn->fd = taosOpenUdpSocket(ip, ownPort);
if (pConn->fd < 0) {
pConn->pSocket = taosOpenUdpSocket(ip, ownPort);
if (pConn->pSocket == NULL) {
tError("%s failed to open UDP socket %x:%hu", label, ip, port);
break;
}
@ -100,7 +100,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads
struct sockaddr_in sin;
unsigned int addrlen = sizeof(sin);
if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET &&
if (taosGetSocketName(pConn->pSocket, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET &&
addrlen == sizeof(sin)) {
pConn->localPort = (uint16_t)ntohs(sin.sin_port);
}
@ -138,9 +138,9 @@ void taosStopUdpConnection(void *handle) {
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR);
if (pConn->fd >= 0) taosCloseSocket(pConn->fd);
pConn->fd = -1;
if (pConn->pSocket != NULL) taosShutDownSocketRDWR(pConn->pSocket);
if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket);
pConn->pSocket = NULL;
}
for (int i = 0; i < pSet->threads; ++i) {
@ -163,7 +163,7 @@ void taosCleanUpUdpConnection(void *handle) {
for (int i = 0; i < pSet->threads; ++i) {
pConn = pSet->udpConn + i;
if (pConn->fd >= 0) taosCloseSocket(pConn->fd);
if (pConn->pSocket != NULL) taosCloseSocket(&pConn->pSocket);
}
tDebug("%s UDP is cleaned up", pSet->label);
@ -199,13 +199,12 @@ static void *taosRecvUdpData(void *param) {
setThreadName("recvUdpData");
while (1) {
dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
dataLen = taosReadFromSocket(pConn->pSocket, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen);
if (dataLen <= 0) {
tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d fd:%d", pConn->label, strerror(errno), (int32_t)dataLen,
pConn->fd);
tDebug("%s UDP socket was closed, exiting(%s), dataLen:%d", pConn->label, strerror(errno), (int32_t)dataLen);
// for windows usage, remote shutdown also returns - 1 in windows client
if (pConn->fd == -1) {
if (pConn->pSocket == NULL) {
break;
} else {
continue;
@ -255,7 +254,7 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c
destAdd.sin_addr.s_addr = ip;
destAdd.sin_port = htons(port);
int ret = (int)taosSendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
int ret = taosSendto(pConn->pSocket, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
return ret;
}

17
source/libs/transport/src/thttp.c
View File

@ -173,7 +173,7 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
#else
int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) {
int32_t code = -1;
SOCKET fd = 0;
TdSocketPtr pSocket = NULL;
uint32_t ip = taosGetIpv4FromFqdn(server);
if (ip == 0xffffffff) {
@ -182,8 +182,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
goto SEND_OVER;
}
fd = taosOpenTcpClientSocket(ip, port, 0);
if (fd < 0) {
pSocket = taosOpenTcpClientSocket(ip, port, 0);
if (pSocket == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to create http socket to %s:%u since %s", server, port, terrstr());
goto SEND_OVER;
@ -200,21 +200,20 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
char header[1024] = {0};
int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag);
if (taosWriteSocket(fd, header, headLen) < 0) {
if (taosWriteMsg(pSocket, header, headLen) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to send http header to %s:%u since %s", server, port, terrstr());
goto SEND_OVER;
}
if (taosWriteSocket(fd, (void*)pCont, contLen) < 0) {
if (taosWriteMsg(pSocket, (void*)pCont, contLen) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to send http content to %s:%u since %s", server, port, terrstr());
goto SEND_OVER;
}
// read something to avoid nginx error 499
if (taosReadSocket(fd, header, 10) < 0) {
if (taosWriteMsg(pSocket, header, 10) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
uError("failed to receive response from %s:%u since %s", server, port, terrstr());
goto SEND_OVER;
@ -223,8 +222,8 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32
code = 0;
SEND_OVER:
if (fd != 0) {
taosCloseSocket(fd);
if (pSocket != NULL) {
taosCloseSocket(&pSocket);
}
return code;

24
source/libs/wal/src/walMeta.c
View File

@ -130,16 +130,16 @@ int walCheckAndRepairMeta(SWal* pWal) {
regcomp(&logRegPattern, logPattern, REG_EXTENDED);
regcomp(&idxRegPattern, idxPattern, REG_EXTENDED);
DIR* dir = opendir(pWal->path);
if (dir == NULL) {
TdDirPtr pDir = taosOpenDir(pWal->path);
if (pDir == NULL) {
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1;
}
// scan log files and build new meta
struct dirent* ent;
while ((ent = readdir(dir)) != NULL) {
char* name = basename(ent->d_name);
TdDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry));
int code = regexec(&logRegPattern, name, 0, NULL, 0);
if (code == 0) {
SWalFileInfo fileInfo;
@ -149,7 +149,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
}
}
closedir(dir);
taosCloseDir(pDir);
regfree(&logRegPattern);
regfree(&idxRegPattern);
@ -337,25 +337,25 @@ static int walFindCurMetaVer(SWal* pWal) {
regex_t walMetaRegexPattern;
regcomp(&walMetaRegexPattern, pattern, REG_EXTENDED);
DIR* dir = opendir(pWal->path);
if (dir == NULL) {
TdDirPtr pDir = taosOpenDir(pWal->path);
if (pDir == NULL) {
wError("vgId:%d, path:%s, failed to open since %s", pWal->cfg.vgId, pWal->path, strerror(errno));
return -1;
}
struct dirent* ent;
TdDirEntryPtr pDirEntry;
// find existing meta-ver[x].json
int metaVer = -1;
while ((ent = readdir(dir)) != NULL) {
char* name = basename(ent->d_name);
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char* name = taosDirEntryBaseName(taosGetDirEntryName(pDirEntry));
int code = regexec(&walMetaRegexPattern, name, 0, NULL, 0);
if (code == 0) {
sscanf(name, "meta-ver%d", &metaVer);
break;
}
}
closedir(dir);
taosCloseDir(pDir);
regfree(&walMetaRegexPattern);
return metaVer;
}

48
source/os/src/osDir.c
View File

@ -14,6 +14,7 @@
*/
#define _DEFAULT_SOURCE
#define ALLOW_FORBID_FUNC
#include "os.h"
#include "osString.h"
@ -36,6 +37,10 @@
#include <unistd.h>
#include <wordexp.h>
typedef struct dirent dirent;
typedef struct DIR TdDir;
typedef struct dirent TdDirent;
void taosRemoveDir(const char *dirname) {
DIR *dir = opendir(dirname);
if (dir == NULL) return;
@ -149,4 +154,47 @@ bool taosIsDir(const char *dirname) {
return false;
}
char* taosDirName(char *name) {
return dirname(name);
}
char* taosDirEntryBaseName(char *name) {
return basename(name);
}
TdDirPtr taosOpenDir(const char *dirname) {
if (dirname == NULL) {
return NULL;
}
return (TdDirPtr)opendir(dirname);
}
TdDirEntryPtr taosReadDir(TdDirPtr pDir) {
if (pDir == NULL) {
return NULL;
}
return (TdDirEntryPtr)readdir((DIR*)pDir);
}
bool taosDirEntryIsDir(TdDirEntryPtr pDirEntry) {
if (pDirEntry == NULL) {
return false;
}
return (((dirent*)pDirEntry)->d_type & DT_DIR) != 0;
}
char* taosGetDirEntryName(TdDirEntryPtr pDirEntry) {
if (pDirEntry == NULL) {
return NULL;
}
return ((dirent*)pDirEntry)->d_name;
}
int32_t taosCloseDir(TdDirPtr pDir) {
if (pDir == NULL) {
return -1;
}
return closedir((DIR*)pDir);
}
#endif

56
source/os/src/osSemaphore.c
View File

@ -99,7 +99,7 @@ static void *sem_thread_routine(void *arg) {
sem_port = mach_task_self();
kern_return_t ret = semaphore_create(sem_port, &sem_exit, SYNC_POLICY_FIFO, 0);
if (ret != KERN_SUCCESS) {
fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", basename(__FILE__), __LINE__, __func__);
fprintf(stderr, "==%s[%d]%s()==failed to create sem_exit\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
sem_inited = -1;
return NULL;
}
@ -112,7 +112,7 @@ static void once_init(void) {
int r = 0;
r = pthread_create(&sem_thread, NULL, sem_thread_routine, NULL);
if (r) {
fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", basename(__FILE__), __LINE__, __func__);
fprintf(stderr, "==%s[%d]%s()==failed to create thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
return;
}
while (sem_inited == 0) {
@ -139,14 +139,14 @@ struct tsem_s {
};
int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
// fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", basename(__FILE__), __LINE__, __func__, sem);
// fprintf(stderr, "==%s[%d]%s():[%p]==creating\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
if (*sem) {
fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==already initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
struct tsem_s *p = (struct tsem_s *)calloc(1, sizeof(*p));
if (!p) {
fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==out of memory\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
@ -162,7 +162,7 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
p->val = value;
} while (0);
if (r) {
fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
#elif defined(SEM_USE_POSIX)
@ -181,27 +181,27 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
int e = errno;
if (e == EEXIST) continue;
if (e == EINTR) continue;
fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e,
fprintf(stderr, "==%s[%d]%s():[%p]==not created[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e,
strerror(e));
abort();
} while (p->sem == SEM_FAILED);
#elif defined(SEM_USE_SEM)
pthread_once(&sem_once, once_init);
if (sem_inited != 1) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal resource init failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
errno = ENOMEM;
return -1;
}
kern_return_t ret = semaphore_create(sem_port, &p->sem, SYNC_POLICY_FIFO, value);
if (ret != KERN_SUCCESS) {
fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==semophore_create failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
// we fail-fast here, because we have less-doc about semaphore_create for the moment
abort();
}
#else // SEM_USE_PTHREAD
p->sem = dispatch_semaphore_create(value);
if (p->sem == NULL) {
fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==not created\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
#endif // SEM_USE_PTHREAD
@ -215,28 +215,28 @@ int tsem_init(tsem_t *sem, int pshared, unsigned int value) {
int tsem_wait(tsem_t *sem) {
if (!*sem) {
fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
struct tsem_s *p = *sem;
if (!p->valid) {
fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
#ifdef SEM_USE_PTHREAD
if (pthread_mutex_lock(&p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
p->val -= 1;
if (p->val < 0) {
if (pthread_cond_wait(&p->cond, &p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
}
if (pthread_mutex_unlock(&p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
return 0;
@ -251,28 +251,28 @@ int tsem_wait(tsem_t *sem) {
int tsem_post(tsem_t *sem) {
if (!*sem) {
fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
struct tsem_s *p = *sem;
if (!p->valid) {
fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
#ifdef SEM_USE_PTHREAD
if (pthread_mutex_lock(&p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
p->val += 1;
if (p->val <= 0) {
if (pthread_cond_signal(&p->cond)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
}
if (pthread_mutex_unlock(&p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
return 0;
@ -286,34 +286,34 @@ int tsem_post(tsem_t *sem) {
}
int tsem_destroy(tsem_t *sem) {
// fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", basename(__FILE__), __LINE__, __func__, sem);
// fprintf(stderr, "==%s[%d]%s():[%p]==destroying\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
if (!*sem) {
// fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", basename(__FILE__), __LINE__, __func__, sem);
// fprintf(stderr, "==%s[%d]%s():[%p]==not initialized\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
// abort();
return 0;
}
struct tsem_s *p = *sem;
if (!p->valid) {
// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", basename(__FILE__), __LINE__, __func__, sem);
// fprintf(stderr, "==%s[%d]%s():[%p]==already destroyed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
// abort();
return 0;
}
#ifdef SEM_USE_PTHREAD
if (pthread_mutex_lock(&p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
p->valid = 0;
if (pthread_cond_destroy(&p->cond)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
if (pthread_mutex_unlock(&p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
if (pthread_mutex_destroy(&p->lock)) {
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", basename(__FILE__), __LINE__, __func__, sem);
fprintf(stderr, "==%s[%d]%s():[%p]==internal logic error\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem);
abort();
}
#elif defined(SEM_USE_POSIX)
@ -322,7 +322,7 @@ int tsem_destroy(tsem_t *sem) {
int r = sem_unlink(name);
if (r) {
int e = errno;
fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", basename(__FILE__), __LINE__, __func__, sem, e,
fprintf(stderr, "==%s[%d]%s():[%p]==unlink failed[%d]%s\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__, sem, e,
strerror(e));
abort();
}

890
source/os/src/osSocket.c
View File

File diff suppressed because it is too large Load Diff

9
source/os/src/osSysinfo.c
View File

@ -16,6 +16,15 @@
#define _DEFAULT_SOURCE
#include "os.h"
bool taosCheckSystemIsSmallEnd() {
union check{
int16_t i;
char ch[2];
}c;
c.i=1;
return c.ch[0]==1;
}
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
/*

8
source/os/src/osTimer.c
View File

@ -79,7 +79,7 @@ static void* timer_routine(void* arg) {
struct kevent64_s kev[10] = {0};
r = kevent64(timer_kq, NULL, 0, kev, sizeof(kev) / sizeof(kev[0]), 0, &to);
if (r != 0) {
fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", basename(__FILE__), __LINE__, __func__);
fprintf(stderr, "==%s[%d]%s()==kevent64 failed\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
abort();
}
timer_callback(SIGALRM); // just mock
@ -97,14 +97,14 @@ int taosInitTimer(void (*callback)(int), int ms) {
timer_kq = kqueue();
if (timer_kq == -1) {
fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", basename(__FILE__), __LINE__, __func__);
fprintf(stderr, "==%s[%d]%s()==failed to create timer kq\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
// since no caller of this func checks the return value for the moment
abort();
}
r = pthread_create(&timer_thread, NULL, timer_routine, NULL);
if (r) {
fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", basename(__FILE__), __LINE__, __func__);
fprintf(stderr, "==%s[%d]%s()==failed to create timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
// since no caller of this func checks the return value for the moment
abort();
}
@ -116,7 +116,7 @@ void taosUninitTimer() {
timer_stop = 1;
r = pthread_join(timer_thread, NULL);
if (r) {
fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", basename(__FILE__), __LINE__, __func__);
fprintf(stderr, "==%s[%d]%s()==failed to join timer thread\n", taosDirEntryBaseName(__FILE__), __LINE__, __func__);
// since no caller of this func checks the return value for the moment
abort();
}

2
source/util/CMakeLists.txt
View File

@ -10,7 +10,7 @@ target_link_libraries(
util
PRIVATE os
PUBLIC lz4_static
PUBLIC api cjson zlib
PUBLIC api cjson
)
if(${BUILD_TEST})

67
tests/test-all.sh
View File

@ -1,5 +1,7 @@
#!/bin/bash
# set -x
# Color setting
RED='\033[0;31m'
GREEN='\033[1;32m'
@ -12,19 +14,19 @@ tests_dir=`pwd`
IN_TDINTERNAL="community"
function stopTaosd {
echo "Stop taosd"
echo "Stop taosd"
sudo systemctl stop taosd || echo 'no sudo or systemctl or stop fail'
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
while [ -n "$PID" ]
do
while [ -n "$PID" ]
do
pkill -TERM -x taosd
sleep 1
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'`
done
}
function dohavecore(){
corefile=`find $corepath -mmin 1`
[ -d $corepath ] && corefile=`find $corepath -mmin 1` || return 1
if [ -n "$corefile" ];then
core_file=`echo $corefile|cut -d " " -f2`
proc=`file $core_file|awk -F "execfn:" '/execfn:/{print $2}'|tr -d \' |awk '{print $1}'|tr -d \,`
@ -56,20 +58,20 @@ function dohavecore(){
function runSimCaseOneByOne {
while read -r line; do
if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then
case=`echo $line | grep sim$ |awk '{print $NF}'`
case=`echo $line | grep sim$ |awk '{print $NF}'`
start_time=`date +%s`
date +%F\ %T | tee -a out.log
if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
echo -n $case
./test.sh -f $case > /dev/null 2>&1 && \
( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
./test.sh -f $case > /dev/null 2>&1 \
&& ([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log ) \
|| ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
echo -e "${RED} failed${NC}" | tee -a out.log
else
echo -n $case
./test.sh -f $case > /dev/null 2>&1 && \
( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
( grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
([ -f ../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
echo -e "${RED} failed${NC}" | tee -a out.log
fi
out_log=`tail -1 out.log `
@ -89,22 +91,22 @@ function runSimCaseOneByOnefq {
for ((i=$start;i<=$end;i++)) ; do
line=`sed -n "$i"p jenkins/basic.txt`
if [[ $line =~ ^./test.sh* ]] || [[ $line =~ ^run* ]]; then
case=`echo $line | grep sim$ |awk '{print $NF}'`
case=`echo $line | grep sim$ |awk '{print $NF}'`
start_time=`date +%s`
date +%F\ %T | tee -a out.log
if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
echo -n $case
./test.sh -f $case > case.log 2>&1 && \
( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
./test.sh -f $case > case.log 2>&1 \
&& \
([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
([ -f ../../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log )
else
pwd
echo -n $case
./test.sh -f $case > ../../sim/case.log 2>&1 && \
( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \
([ -f ../../sim/tsim/log/taoslog0.0 ] && grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \
( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && pwd && cat ../../sim/case.log )
fi
@ -134,7 +136,6 @@ function runPyCaseOneByOne {
while read -r line; do
if [[ $line =~ ^python.* ]]; then
if [[ $line != *sleep* ]]; then
if [[ $line =~ '-r' ]];then
case=`echo $line|awk '{print $4}'`
else
@ -172,7 +173,6 @@ function runPyCaseOneByOnefq() {
line=`sed -n "$i"p fulltest.sh`
if [[ $line =~ ^python.* ]]; then
if [[ $line != *sleep* ]]; then
if [[ $line =~ '-r' ]];then
case=`echo $line|awk '{print $4}'`
else
@ -239,7 +239,9 @@ totalExampleFailed=0
if [ "${OS}" == "Linux" ]; then
corepath=`grep -oP '.*(?=core_)' /proc/sys/kernel/core_pattern||grep -oP '.*(?=core-)' /proc/sys/kernel/core_pattern`
if [ -z "$corepath" ];then
echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied"
[ -d /coredump ] || mkdir /coredump || echo -e "failed to mdkir /coredump"
[ -d /coredump ] \
&& echo "/coredump/core_%e_%p_%t" > /proc/sys/kernel/core_pattern || echo "Permission denied"
corepath="/coredump/"
fi
fi
@ -248,6 +250,7 @@ if [ "$2" != "jdbc" ] && [ "$2" != "python" ] && [ "$2" != "unit" ] && [ "$2" !
echo "### run TSIM test case ###"
cd $tests_dir/script
[ -d ../../sim ] || mkdir ../../sim || echo -e "failed to mkdir ../../sim"
[ -f out.log ] && rm -f out.log
if [ "$1" == "cron" ]; then
echo "### run TSIM regression test ###"
@ -368,16 +371,20 @@ if [ "$2" != "sim" ] && [ "$2" != "jdbc" ] && [ "$2" != "unit" ] && [ "$2" != "
echo "### run Python smoke test ###"
runPyCaseOneByOne smoketest.sh
fi
totalPySuccess=`grep 'success' pytest-out.log | wc -l`
if [ "$totalPySuccess" -gt "0" ]; then
echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}"
fi
if [ -f pytest-out.log ]; then
totalPySuccess=`grep 'success' pytest-out.log | wc -l`
if [ "$totalPySuccess" -gt "0" ]; then
echo -e "\n${GREEN} ### Total $totalPySuccess python case(s) succeed! ### ${NC}"
fi
totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
if [ "$totalPyFailed" -ne "0" ]; then
echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}"
# exit $totalPyFailed
totalPyFailed=`grep 'failed\|fault' pytest-out.log | wc -l`
if [ "$totalPyFailed" -ne "0" ]; then
echo -e "\n${RED} ### Total $totalPyFailed python case(s) failed! ### ${NC}"
# exit $totalPyFailed
fi
else
echo -e "pytest-out.log not found!"
fi
fi

23
tests/test/c/tmqDemo.c
View File

@ -192,11 +192,11 @@ static void msg_process(tmq_message_t* message) { tmqShowMsg(message); }
// calc dir size (not include itself 4096Byte)
int64_t getDirectorySize(char *dir)
{
DIR *dp;
struct dirent *entry;
TdDirPtr pDir;
TdDirEntryPtr pDirEntry;
int64_t totalSize=0;
if ((dp = opendir(dir)) == NULL) {
if ((pDir = taosOpenDir(dir)) == NULL) {
fprintf(stderr, "Cannot open dir: %s\n", dir);
return -1;
}
@ -204,26 +204,27 @@ int64_t getDirectorySize(char *dir)
//lstat(dir, &statbuf);
//totalSize+=statbuf.st_size;
while ((entry = readdir(dp)) != NULL) {
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char subdir[1024];
sprintf(subdir, "%s/%s", dir, entry->d_name);
char* fileName = taosGetDirEntryName(pDirEntry);
sprintf(subdir, "%s/%s", dir, fileName);
//printf("===d_name: %s\n", entry->d_name);
if (taosIsDir(subdir)) {
if (strcmp(".", entry->d_name) == 0 || strcmp("..", entry->d_name) == 0) {
if (strcmp(".", fileName) == 0 || strcmp("..", fileName) == 0) {
continue;
}
int64_t subDirSize = getDirectorySize(subdir);
totalSize+=subDirSize;
} else if (0 == strcmp(strchr(entry->d_name, '.'), ".log")) { // only calc .log file size, and not include .idx file
} else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file
int64_t file_size = 0;
taosStatFile(subdir, &file_size, NULL);
totalSize+=file_size;
}
}
closedir(dp);
taosCloseDir(pDir);
return totalSize;
}
@ -352,7 +353,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
int32_t cnt = 0;
/*clock_t startTime = clock();*/
while (running) {
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
if (tmqmessage) {
cnt++;
msg_process(tmqmessage);
@ -383,7 +384,7 @@ void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) {
}
while (running) {
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
if (tmqmessage) {
msg_process(tmqmessage);
tmq_message_destroy(tmqmessage);
@ -411,7 +412,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog
int32_t skipLogNum = 0;
int64_t startTime = taosGetTimestampUs();
while (running) {
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500);
tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 1);
if (tmqmessage) {
batchCnt++;
skipLogNum += tmqGetSkipLogNum(tmqmessage);