xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin' into enh/TD-21108

This commit is contained in:
dapan1121 2023-01-04 08:57:22 +08:00
parent 38d1e8688e 89425dc488
commit ac7d250a7f
119 changed files with 1623 additions and 2146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmake/cmake.define
View File

@ -117,12 +117,18 @@ ELSE ()
IF (${BUILD_SANITIZER})
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
MESSAGE(STATUS "Will compile with Address Sanitizer!")
MESSAGE(STATUS "Compile with Address Sanitizer!")
ELSE ()
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=2 -Wno-format-nonliteral -Wno-format-truncation -Wno-format-y2k")
ENDIF ()
# disable all assert
IF ((${DISABLE_ASSERT} MATCHES "true") OR (${DISABLE_ASSERTS} MATCHES "true"))
ADD_DEFINITIONS(-DDISABLE_ASSERT)
MESSAGE(STATUS "Disable all asserts")
ENDIF()
INCLUDE(CheckCCompilerFlag)
IF (TD_ARM_64 OR TD_ARM_32)
SET(COMPILER_SUPPORT_SSE42 false)
@ -155,7 +161,7 @@ ELSE ()
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
ENDIF()
MESSAGE(STATUS "SIMD instructions (AVX/AVX2) is ACTIVATED")
MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED")
ENDIF()
ENDIF ()

2
cmake/cmake.version
View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "3.0.2.1")
SET(TD_VER_NUMBER "3.0.2.2")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)

2
cmake/taosadapter_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG 5662a6d
GIT_TAG a2e9920
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

4
include/common/tmsgcb.h
View File

@ -39,6 +39,7 @@ typedef enum {
QUEUE_MAX,
} EQueueType;
typedef int32_t (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port);
typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
@ -48,6 +49,7 @@ typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
typedef void (*ReportStartup)(const char* name, const char* desc);
typedef struct {
void* data;
void* mgmt;
void* clientRpc;
PutToQueueFp putToQueueFp;
@ -57,6 +59,7 @@ typedef struct {
RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
ReleaseHandleFp releaseHandleFp;
ReportStartup reportStartupFp;
UpdateDnodeInfoFp updateDnodeInfoFp;
} SMsgCb;
void tmsgSetDefault(const SMsgCb* msgcb);
@ -67,6 +70,7 @@ void tmsgSendRsp(SRpcMsg* pMsg);
void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg);
void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type);
void tmsgReportStartup(const char* name, const char* desc);
int32_t tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port);
#ifdef __cplusplus
}

1
include/common/trow.h
View File

@ -20,7 +20,6 @@
#include "talgo.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tbuffer.h"
#include "tdataformat.h"
#include "tdef.h"
#include "ttypes.h"

1
include/libs/function/function.h
View File

@ -20,7 +20,6 @@
extern "C" {
#endif
#include "tbuffer.h"
#include "tcommon.h"
#include "tvariant.h"

2
include/libs/sync/sync.h
View File

@ -78,6 +78,8 @@ typedef enum {
} ESyncState;
typedef struct SNodeInfo {
int64_t clusterId;
int32_t nodeId;
uint16_t nodePort;
char nodeFqdn[TSDB_FQDN_LEN];
} SNodeInfo;

1
include/util/taoserror.h
View File

@ -345,6 +345,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TOPIC_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x03EB)
#define TSDB_CODE_MND_CGROUP_USED TAOS_DEF_ERROR_CODE(0, 0x03EC)
#define TSDB_CODE_MND_TOPIC_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03ED)
#define TSDB_CODE_MND_INVALID_SUB_OPTION TAOS_DEF_ERROR_CODE(0, 0x03EE)
#define TSDB_CODE_MND_IN_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x03EF)
// mnode-stream

168
include/util/tbuffer.h
View File

@ -1,168 +0,0 @@
/*
* Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_BUFFER_H_
#define _TD_UTIL_BUFFER_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
////////////////////////////////////////////////////////////////////////////////
// usage example
/*
#include <stdio.h>
#include "texception.h"
int32_t main( int32_t argc, char** argv ) {
SBufferWriter bw = tbufInitWriter( NULL, false );
TRY( 1 ) {
//--------------------- write ------------------------
// reserve 1024 bytes for the buffer to improve performance
tbufEnsureCapacity( &bw, 1024 );
// reserve space for the interger count
size_t pos = tbufReserve( &bw, sizeof(int32_t) );
// write 5 integers to the buffer
for( int32_t i = 0; i < 5; i++) {
tbufWriteInt32( &bw, i );
}
// write the integer count to buffer at reserved position
tbufWriteInt32At( &bw, pos, 5 );
// write a string to the buffer
tbufWriteString( &bw, "this is a string.\n" );
// acquire the result and close the write buffer
size_t size = tbufTell( &bw );
char* data = tbufGetData( &bw, false );
//------------------------ read -----------------------
SBufferReader br = tbufInitReader( data, size, false );
// read & print out all integers
int32_t count = tbufReadInt32( &br );
for( int32_t i = 0; i < count; i++ ) {
printf( "%d\n", tbufReadInt32(&br) );
}
// read & print out a string
puts( tbufReadString(&br, NULL) );
// try read another integer, this result in an error as there no this integer
tbufReadInt32( &br );
printf( "you should not see this message.\n" );
} CATCH( code ) {
printf( "exception code is: %d, you will see this message after print out 5 integers and a string.\n", code );
} END_TRY
tbufCloseWriter( &bw );
return 0;
}
*/
typedef struct SBufferReader {
bool endian;
const char* data;
size_t pos;
size_t size;
} SBufferReader;
typedef struct SBufferWriter {
bool endian;
char* data;
size_t pos;
size_t size;
void* (*allocator)(void*, size_t);
} SBufferWriter;
// common functions & macros for both reader & writer
#define tbufTell(buf) ((buf)->pos)
/* ------------------------ BUFFER WRITER FUNCTIONS AND MACROS ------------------------ */
// *Allocator*, function to allocate memory, will use 'realloc' if NULL
// *Endian*, if true, writer functions of primitive types will do 'hton' automatically
#define tbufInitWriter(Allocator, Endian) \
{ .endian = (Endian), .data = NULL, .pos = 0, .size = 0, .allocator = ((Allocator) == NULL ? realloc : (Allocator)) }
void tbufCloseWriter(SBufferWriter* buf);
void tbufEnsureCapacity(SBufferWriter* buf, size_t size);
size_t tbufReserve(SBufferWriter* buf, size_t size);
char* tbufGetData(SBufferWriter* buf, bool takeOver);
void tbufWrite(SBufferWriter* buf, const void* data, size_t size);
void tbufWriteAt(SBufferWriter* buf, size_t pos, const void* data, size_t size);
void tbufWriteStringLen(SBufferWriter* buf, const char* str, size_t len);
void tbufWriteString(SBufferWriter* buf, const char* str);
// the prototype of tbufWriteBinary and tbufWrite are identical
// the difference is: tbufWriteBinary writes the length of the data to the buffer
// first, then the actual data, which means the reader don't need to know data
// size before read. Write only write the data itself, which means the reader
// need to know data size before read.
void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len);
void tbufWriteBool(SBufferWriter* buf, bool data);
void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data);
void tbufWriteChar(SBufferWriter* buf, char data);
void tbufWriteCharAt(SBufferWriter* buf, size_t pos, char data);
void tbufWriteInt8(SBufferWriter* buf, int8_t data);
void tbufWriteInt8At(SBufferWriter* buf, size_t pos, int8_t data);
void tbufWriteUint8(SBufferWriter* buf, uint8_t data);
void tbufWriteUint8At(SBufferWriter* buf, size_t pos, uint8_t data);
void tbufWriteInt16(SBufferWriter* buf, int16_t data);
void tbufWriteInt16At(SBufferWriter* buf, size_t pos, int16_t data);
void tbufWriteUint16(SBufferWriter* buf, uint16_t data);
void tbufWriteUint16At(SBufferWriter* buf, size_t pos, uint16_t data);
void tbufWriteInt32(SBufferWriter* buf, int32_t data);
void tbufWriteInt32At(SBufferWriter* buf, size_t pos, int32_t data);
void tbufWriteUint32(SBufferWriter* buf, uint32_t data);
void tbufWriteUint32At(SBufferWriter* buf, size_t pos, uint32_t data);
void tbufWriteInt64(SBufferWriter* buf, int64_t data);
void tbufWriteInt64At(SBufferWriter* buf, size_t pos, int64_t data);
void tbufWriteUint64(SBufferWriter* buf, uint64_t data);
void tbufWriteUint64At(SBufferWriter* buf, size_t pos, uint64_t data);
void tbufWriteFloat(SBufferWriter* buf, float data);
void tbufWriteFloatAt(SBufferWriter* buf, size_t pos, float data);
void tbufWriteDouble(SBufferWriter* buf, double data);
void tbufWriteDoubleAt(SBufferWriter* buf, size_t pos, double data);
/* ------------------------ BUFFER READER FUNCTIONS AND MACROS ------------------------ */
// *Endian*, if true, reader functions of primitive types will do 'ntoh' automatically
#define tbufInitReader(Data, Size, Endian) \
{ .endian = (Endian), .data = (Data), .pos = 0, .size = ((Data) == NULL ? 0 : (Size)) }
size_t tbufSkip(SBufferReader* buf, size_t size);
const char* tbufRead(SBufferReader* buf, size_t size);
void tbufReadToBuffer(SBufferReader* buf, void* dst, size_t size);
const char* tbufReadString(SBufferReader* buf, size_t* len);
size_t tbufReadToString(SBufferReader* buf, char* dst, size_t size);
const char* tbufReadBinary(SBufferReader* buf, size_t* len);
size_t tbufReadToBinary(SBufferReader* buf, void* dst, size_t size);
bool tbufReadBool(SBufferReader* buf);
char tbufReadChar(SBufferReader* buf);
int8_t tbufReadInt8(SBufferReader* buf);
uint8_t tbufReadUint8(SBufferReader* buf);
int16_t tbufReadInt16(SBufferReader* buf);
uint16_t tbufReadUint16(SBufferReader* buf);
int32_t tbufReadInt32(SBufferReader* buf);
uint32_t tbufReadUint32(SBufferReader* buf);
int64_t tbufReadInt64(SBufferReader* buf);
uint64_t tbufReadUint64(SBufferReader* buf);
float tbufReadFloat(SBufferReader* buf);
double tbufReadDouble(SBufferReader* buf);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_BUFFER_H_*/

21
include/util/tjson.h
View File

@ -30,6 +30,27 @@ extern "C" {
val = _tmp; \
} while (0)
#define tjsonGetInt32ValueFromDouble(pJson, pName, val, code) \
do { \
double _tmp = 0; \
code = tjsonGetDoubleValue(pJson, pName, &_tmp); \
val = (int32_t)_tmp; \
} while (0)
#define tjsonGetInt8ValueFromDouble(pJson, pName, val, code) \
do { \
double _tmp = 0; \
code = tjsonGetDoubleValue(pJson, pName, &_tmp); \
val = (int8_t)_tmp; \
} while (0)
#define tjsonGetUInt16ValueFromDouble(pJson, pName, val, code) \
do { \
double _tmp = 0; \
code = tjsonGetDoubleValue(pJson, pName, &_tmp); \
val = (uint16_t)_tmp; \
} while (0)
typedef void SJson;
SJson* tjsonCreateObject();

7
include/util/tlog.h
View File

@ -85,12 +85,19 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...);
bool taosAssertRelease(bool condition);
// Disable all asserts that may compromise the performance.
#if defined DISABLE_ASSERT
#define ASSERT(condition)
#define ASSERTS(condition, ...)
#else
#define ASSERTS(condition, ...) taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__)
#ifdef NDEBUG
#define ASSERT(condition) taosAssertRelease(condition)
#else
#define ASSERT(condition) taosAssertDebug(condition, __FILE__, __LINE__, "assert info not provided")
#endif
#endif
void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo);
void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd);

8
source/client/src/clientTmq.c
View File

@ -912,10 +912,12 @@ void tmqFreeImpl(void* handle) {
tmq_t* tmq = (tmq_t*)handle;
// TODO stop timer
tmqClearUnhandleMsg(tmq);
if (tmq->mqueue) taosCloseQueue(tmq->mqueue);
if (tmq->mqueue) {
tmqClearUnhandleMsg(tmq);
taosCloseQueue(tmq->mqueue);
}
if (tmq->delayedTask) taosCloseQueue(tmq->delayedTask);
if (tmq->qall) taosFreeQall(tmq->qall);
taosFreeQall(tmq->qall);
tsem_destroy(&tmq->rspSem);

2
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -180,6 +180,6 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
return -1;
}
dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy);
dDebug("succeed to write %s, deployed:%d", realfile, pOption->deploy);
return 0;
}

12
source/dnode/mgmt/mgmt_snode/src/smWorker.c
View File

@ -58,11 +58,7 @@ static void smProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
dTrace("msg:%p, get from snode-stream queue", pMsg);
int32_t code = sndProcessStreamMsg(pMgmt->pSnode, pMsg);
if (code < 0) {
if (pMsg) {
dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code));
} else {
dGError("snd, msg:%p failed to process stream empty msg since %s", pMsg, terrstr(code));
}
dGError("snd, msg:%p failed to process stream msg %s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr(code));
smSendRsp(pMsg, terrno);
}
@ -161,8 +157,10 @@ int32_t smPutMsgToQueue(SSnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
smPutNodeMsgToWriteQueue(pMgmt, pMsg);
break;
default:
ASSERTS(0, "msg:%p failed to put into snode queue since %s, type:%s qtype:%d", pMsg, terrstr(),
TMSG_INFO(pMsg->msgType), qtype);
terrno = TSDB_CODE_INVALID_PARA;
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
return -1;
}
return 0;
}

2
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -215,7 +215,7 @@ _OVER:
if (code != 0) return -1;
dInfo("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
dInfo("succeed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
code = taosRenameFile(file, realfile);
if (code != 0) {

13
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -132,10 +132,12 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->syncCfg.myIndex = pCreate->selfIndex;
pCfg->syncCfg.replicaNum = pCreate->replica;
memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo));
for (int i = 0; i < pCreate->replica; ++i) {
for (int32_t i = 0; i < pCreate->replica; ++i) {
SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
pNode->nodeId = pCreate->replicas[i].id;
pNode->nodePort = pCreate->replicas[i].port;
tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
}
}
@ -188,8 +190,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix,
req.hashSuffix, req.replica, req.selfIndex, req.strict);
for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d id:%d fqdn:%s port:%u", req.vgId, i, req.replicas[i].id, req.replicas[i].fqdn,
req.replicas[i].port);
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
req.replicas[i].id);
}
SReplica *pReplica = &req.replicas[req.selfIndex];
@ -286,7 +288,8 @@ int32_t vmProcessAlterVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo("vgId:%d, start to alter vnode, replica:%d selfIndex:%d strict:%d", alterReq.vgId, alterReq.replica,
alterReq.selfIndex, alterReq.strict);
for (int32_t i = 0; i < alterReq.replica; ++i) {
dInfo("vgId:%d, replica:%d ep:%s:%u", alterReq.vgId, i, alterReq.replicas[i].fqdn, alterReq.replicas[i].port);
SReplica *pReplica = &alterReq.replicas[i];
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", alterReq.vgId, i, pReplica->fqdn, pReplica->port, pReplica->port);
}
if (alterReq.replica <= 0 || alterReq.selfIndex < 0 || alterReq.selfIndex >= alterReq.replica) {

16
source/dnode/mgmt/mgmt_vnode/src/vmWorker.c
View File

@ -86,12 +86,8 @@ static void vmProcessStreamQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo);
if (code != 0) {
if (terrno != 0) code = terrno;
if (pMsg) {
dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType),
terrstr(code));
} else {
dGError("vgId:%d, msg:%p failed to process stream empty msg since %s", pVnode->vgId, pMsg, terrstr(code));
}
dGError("vgId:%d, msg:%p failed to process stream msg %s since %s", pVnode->vgId, pMsg, TMSG_INFO(pMsg->msgType),
terrstr(code));
vmSendRsp(pMsg, code);
}
@ -146,16 +142,16 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
return -1;
}
SMsgHead *pHead = pMsg->pCont;
int32_t code = 0;
SMsgHead *pHead = pMsg->pCont;
int32_t code = 0;
pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId);
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
if (pVnode == NULL) {
dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, terrstr(),
TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg,
terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen);
return terrno != 0 ? terrno : -1;
}

2
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -345,6 +345,8 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) {
.registerBrokenLinkArgFp = dmRegisterBrokenLinkArg,
.releaseHandleFp = dmReleaseHandle,
.reportStartupFp = dmReportStartup,
.updateDnodeInfoFp = dmUpdateDnodeInfo,
.data = &pDnode->data,
};
return msgCb;
}

1
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -167,6 +167,7 @@ void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps);
void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet);
void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
int32_t dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port);
#ifdef __cplusplus
}

82
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -182,22 +182,25 @@ _OVER:
}
int32_t dmWriteEps(SDnodeData *pData) {
int32_t code = -1;
char *content = NULL;
TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
dError("failed to write %s since %s", file, strerror(errno));
dError("failed to open %s since %s", file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
goto _OVER;
}
int32_t len = 0;
int32_t maxLen = 256 * 1024;
char *content = taosMemoryCalloc(1, maxLen + 1);
content = taosMemoryCalloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pData->dnodeId);
@ -221,20 +224,39 @@ int32_t dmWriteEps(SDnodeData *pData) {
}
len += snprintf(content + len, maxLen - len, "}\n");
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
if (taosWriteFile(pFile, content, len) != len) {
dError("failed to write %s since %s", file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
dError("failed to fsync %s since %s", file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
taosMemoryFree(content);
taosMemoryFreeClear(content);
if (taosRenameFile(file, realfile) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to rename %s since %s", file, terrstr());
return -1;
goto _OVER;
}
code = 0;
pData->updateTime = taosGetTimestampMs();
dDebug("successed to write %s, dnodeVer:%" PRId64, realfile, pData->dnodeVer);
return 0;
dInfo("succeed to write %s, dnodeVer:%" PRId64, realfile, pData->dnodeVer);
_OVER:
if (content != NULL) taosMemoryFreeClear(content);
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to write file %s since %s", realfile, terrstr());
}
return code;
}
void dmUpdateEps(SDnodeData *pData, SArray *eps) {
@ -332,3 +354,41 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) {
dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
}
}
int32_t dmUpdateDnodeInfo(void *data, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port) {
SDnodeData *pData = data;
int32_t ret = -1;
taosThreadRwlockRdlock(&pData->lock);
if (*dnodeId <= 0) {
for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pData->dnodeEps); ++i) {
SDnodeEp *pDnodeEp = taosArrayGet(pData->dnodeEps, i);
if (strcmp(pDnodeEp->ep.fqdn, fqdn) == 0 && pDnodeEp->ep.port == *port) {
dInfo("dnode:%s:%u, update dnodeId from %d to %d", fqdn, *port, *dnodeId, pDnodeEp->id);
*dnodeId = pDnodeEp->id;
*clusterId = pData->clusterId;
ret = 0;
}
}
if (ret != 0) {
dInfo("dnode:%s:%u, failed to update dnodeId:%d", fqdn, *port, *dnodeId);
}
} else {
SDnodeEp *pDnodeEp = taosHashGet(pData->dnodeHash, dnodeId, sizeof(int32_t));
if (pDnodeEp) {
if (strcmp(pDnodeEp->ep.fqdn, fqdn) != 0) {
dInfo("dnode:%d, update port from %s to %s", *dnodeId, fqdn, pDnodeEp->ep.fqdn);
tstrncpy(fqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN);
}
if (pDnodeEp->ep.port != *port) {
dInfo("dnode:%d, update port from %u to %u", *dnodeId, *port, pDnodeEp->ep.port);
*port = pDnodeEp->ep.port;
}
*clusterId = pData->clusterId;
ret = 0;
} else {
dInfo("dnode:%d, failed to update dnode info", *dnodeId);
}
}
taosThreadRwlockUnlock(&pData->lock);
return ret;
}

2
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -105,7 +105,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) {
return -1;
}
dInfo("successed to write %s, deployed:%d", realfile, deployed);
dInfo("succeed to write %s, deployed:%d", realfile, deployed);
code = 0;
_OVER:

2
source/dnode/mnode/impl/src/mndDb.c
View File

@ -112,7 +112,6 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.hashMethod, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.numOfRetensions, _OVER)
for (int32_t i = 0; i < pDb->cfg.numOfRetensions; ++i) {
ASSERT(taosArrayGetSize(pDb->cfg.pRetensions) == pDb->cfg.numOfRetensions);
SRetention *pRetension = taosArrayGet(pDb->cfg.pRetensions, i);
SDB_SET_INT64(pRaw, dataPos, pRetension->freq, _OVER)
SDB_SET_INT64(pRaw, dataPos, pRetension->keep, _OVER)
@ -364,6 +363,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
terrno = 0;
return terrno;

2
source/dnode/mnode/impl/src/mndDef.c
View File

@ -489,7 +489,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
tlen += tEncodeSMqConsumerEp(buf, pConsumerEp);
cnt++;
}
ASSERT(cnt == sz);
if(cnt != sz) return -1;
tlen += taosEncodeArray(buf, pSub->unassignedVgs, (FEncode)tEncodeSMqVgEp);
tlen += taosEncodeString(buf, pSub->dbName);
return tlen;

11
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -21,6 +21,7 @@
#include "mndSync.h"
#include "mndTrans.h"
#include "tmisce.h"
#include "mndCluster.h"
#define MNODE_VER_NUMBER 1
#define MNODE_RESERVE_SIZE 64
@ -743,8 +744,12 @@ static void mndReloadSyncConfig(SMnode *pMnode) {
if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) {
SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum];
tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn));
pNode->nodeId = pObj->pDnode->id;
pNode->clusterId = mndGetClusterId(pMnode);
pNode->nodePort = pObj->pDnode->port;
tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId);
if (pObj->pDnode->id == pMnode->selfDnodeId) {
cfg.myIndex = cfg.replicaNum;
}
@ -758,7 +763,6 @@ static void mndReloadSyncConfig(SMnode *pMnode) {
mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes);
return;
}
// ASSERT(0);
if (cfg.myIndex == -1) {
#if 1
@ -775,7 +779,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) {
mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
for (int32_t i = 0; i < cfg.replicaNum; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId,
pNode->clusterId);
}
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -111,7 +111,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) {
type = TSDB_MGMT_TABLE_PRIVILEGES;
} else {
// ASSERT(0);
mError("invalid show name:%s len:%d", name, len);
}
return type;

18
source/dnode/mnode/impl/src/mndSma.c
View File

@ -488,7 +488,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
memcpy(smaObj.db, pDb->name, TSDB_DB_FNAME_LEN);
smaObj.createdTime = taosGetTimestampMs();
smaObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
ASSERT(smaObj.uid != 0);
char resultTbName[TSDB_TABLE_FNAME_LEN + 16] = {0};
snprintf(resultTbName, TSDB_TABLE_FNAME_LEN + 16, "%s_td_tsma_rst_tb", pCreate->name);
memcpy(smaObj.dstTbName, resultTbName, TSDB_TABLE_FNAME_LEN);
@ -558,13 +558,15 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
SNode *pAst = NULL;
if (nodesStringToNode(streamObj.ast, &pAst) < 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_INVALID_SMA_OPTION;
mError("sma:%s, failed to create since parse ast error", smaObj.name);
return -1;
}
// extract output schema from ast
if (qExtractResultSchema(pAst, (int32_t *)&streamObj.outputSchema.nCols, &streamObj.outputSchema.pSchema) != 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_INVALID_SMA_OPTION;
mError("sma:%s, failed to create since extract result schema error", smaObj.name);
return -1;
}
@ -579,15 +581,18 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
};
if (qCreateQueryPlan(&cxt, &pPlan, NULL) < 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_INVALID_SMA_OPTION;
mError("sma:%s, failed to create since create query plan error", smaObj.name);
return -1;
}
// save physcial plan
if (nodesNodeToString((SNode *)pPlan, false, &streamObj.physicalPlan, NULL) != 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_INVALID_SMA_OPTION;
mError("sma:%s, failed to create since save physcial plan error", smaObj.name);
return -1;
}
if (pAst != NULL) nodesDestroyNode(pAst);
nodesDestroyNode((SNode *)pPlan);
@ -826,14 +831,13 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p
if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
ASSERT(0);
goto _OVER;
}
// drop stream
if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
mError("stream:%s, failed to drop log since %s", pStream->name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
ASSERT(0);
goto _OVER;
}
}

16
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1177,7 +1177,9 @@ static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName,
SNode *pAst = NULL;
if (nodesStringToNode(pTopic->ast, &pAst) != 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_FIELD_CONFLICT_WITH_TOPIC;
mError("topic:%s, create ast error", pTopic->name);
sdbRelease(pSdb, pTopic);
return -1;
}
@ -1222,7 +1224,9 @@ static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName
SNode *pAst = NULL;
if (nodesStringToNode(pStream->ast, &pAst) != 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION;
mError("stream:%s, create ast error", pStream->name);
sdbRelease(pSdb, pStream);
return -1;
}
@ -2094,7 +2098,9 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName,
SNode *pAst = NULL;
if (nodesStringToNode(pTopic->ast, &pAst) != 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
mError("topic:%s, create ast error", pTopic->name);
sdbRelease(pSdb, pTopic);
return -1;
}
@ -2141,7 +2147,9 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
SNode *pAst = NULL;
if (nodesStringToNode(pStream->ast, &pAst) != 0) {
ASSERT(0);
terrno = TSDB_CODE_MND_INVALID_STREAM_OPTION;
mError("stream:%s, create ast error", pStream->name);
sdbRelease(pSdb, pStream);
return -1;
}

90
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -96,18 +96,12 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
pSub->subType = pTopic->subType;
pSub->withMeta = pTopic->withMeta;
ASSERT(pSub->unassignedVgs->size == 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
if (mndSchedInitSubEp(pMnode, pTopic, pSub) < 0) {
tDeleteSubscribeObj(pSub);
taosMemoryFree(pSub);
return NULL;
}
ASSERT(pSub->unassignedVgs->size > 0);
ASSERT(taosHashGetSize(pSub->consumerHash) == 0);
return pSub;
}
@ -144,7 +138,10 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SMqSubscribeObj *pSub,
const SMqRebOutputVg *pRebVg) {
ASSERT(pRebVg->oldConsumerId != pRebVg->newConsumerId);
if (pRebVg->oldConsumerId == pRebVg->newConsumerId) {
terrno = TSDB_CODE_MND_INVALID_SUB_OPTION;
return -1;
}
void *buf;
int32_t tlen;
@ -155,8 +152,8 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM
int32_t vgId = pRebVg->pVgEp->vgId;
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
if (pVgObj == NULL) {
ASSERT(0);
taosMemoryFree(buf);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
@ -206,9 +203,9 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) {
}
static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) {
int32_t totalVgNum = pOutput->pSub->vgNum;
mInfo("mq rebalance: subscription: %s, vgNum: %d", pOutput->pSub->key, pOutput->pSub->vgNum);
int32_t totalVgNum = pOutput->pSub->vgNum;
const char *sub = pOutput->pSub->key;
mInfo("sub:%s, mq rebalance vgNum:%d", sub, pOutput->pSub->vgNum);
// 1. build temporary hash(vgId -> SMqRebOutputVg) to store modified vg
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
@ -218,11 +215,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
int32_t actualRemoved = 0;
for (int32_t i = 0; i < removedNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i);
ASSERT(consumerId > 0);
SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
ASSERT(pConsumerEp);
if (pConsumerEp) {
ASSERT(consumerId == pConsumerEp->consumerId);
actualRemoved++;
int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
for (int32_t j = 0; j < consumerVgNum; j++) {
@ -233,7 +229,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp,
};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64, pVgEp->vgId, consumerId);
mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64, sub, pVgEp->vgId, consumerId);
}
taosArrayDestroy(pConsumerEp->vgs);
taosHashRemove(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t));
@ -241,7 +237,10 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
taosArrayPush(pOutput->removedConsumers, &consumerId);
}
}
ASSERT(removedNum == actualRemoved);
if (removedNum != actualRemoved) {
mError("sub:%s, mq rebalance removedNum:%d not matched with actual:%d", sub, removedNum, actualRemoved);
}
// if previously no consumer, there are vgs not assigned
{
@ -254,7 +253,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp,
};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &rebOutput, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vgId:%d from unassigned", pVgEp->vgId);
mInfo("sub:%s, mq rebalance remove vgId:%d from unassigned", sub, pVgEp->vgId);
}
}
@ -268,8 +267,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
minVgCnt = totalVgNum / afterRebConsumerNum;
imbConsumerNum = totalVgNum % afterRebConsumerNum;
}
mInfo("mq rebalance: %d consumer after rebalance, at least %d vg each, %d consumer has more vg", afterRebConsumerNum,
minVgCnt, imbConsumerNum);
mInfo("sub:%s, mq rebalance %d consumer after rebalance, at least %d vg each, %d consumer has more vg", sub,
afterRebConsumerNum, minVgCnt, imbConsumerNum);
// 4. first scan: remove consumer more than wanted, put to remove hash
int32_t imbCnt = 0;
@ -278,7 +277,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) break;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
ASSERT(pConsumerEp->consumerId > 0);
int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs);
// all old consumers still existing are touched
// TODO optimize: touch only consumer whose vgs changed
@ -298,7 +297,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp,
};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId,
mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
pConsumerEp->consumerId);
}
imbCnt++;
@ -313,7 +312,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
.pVgEp = pVgEp,
};
taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg));
mInfo("mq rebalance: remove vgId:%d from consumer:%" PRId64 ",(first scan)", pVgEp->vgId,
mInfo("sub:%s, mq rebalance remove vgId:%d from consumer:%" PRId64 ",(first scan)", sub, pVgEp->vgId,
pConsumerEp->consumerId);
}
}
@ -325,13 +324,13 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
int32_t consumerNum = taosArrayGetSize(pInput->pRebInfo->newConsumers);
for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i);
ASSERT(consumerId > 0);
SMqConsumerEp newConsumerEp;
newConsumerEp.consumerId = consumerId;
newConsumerEp.vgs = taosArrayInit(0, sizeof(void *));
taosHashPut(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t), &newConsumerEp, sizeof(SMqConsumerEp));
taosArrayPush(pOutput->newConsumers, &consumerId);
mInfo("mq rebalance: add new consumer:%" PRId64, consumerId);
mInfo("sub:%s, mq rebalance add new consumer:%" PRId64, sub, consumerId);
}
}
@ -344,13 +343,16 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
if (pIter == NULL) break;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
ASSERT(pConsumerEp->consumerId > 0);
// push until equal minVg
while (taosArrayGetSize(pConsumerEp->vgs) < minVgCnt) {
// iter hash and find one vg
pRemovedIter = taosHashIterate(pHash, pRemovedIter);
ASSERT(pRemovedIter);
if (pRemovedIter == NULL) {
mError("sub:%s, removed iter is null", sub);
continue;
}
pRebVg = (SMqRebOutputVg *)pRemovedIter;
// push
taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp);
@ -361,7 +363,6 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
}
}
ASSERT(pIter == NULL);
// 7. handle unassigned vg
if (taosHashGetSize(pOutput->pSub->consumerHash) != 0) {
// if has consumer, assign all left vg
@ -377,9 +378,8 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
}
while (1) {
pIter = taosHashIterate(pOutput->pSub->consumerHash, pIter);
ASSERT(pIter);
pConsumerEp = (SMqConsumerEp *)pIter;
ASSERT(pConsumerEp->consumerId > 0);
if (taosArrayGetSize(pConsumerEp->vgs) == minVgCnt) {
break;
}
@ -404,19 +404,19 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
pIter = taosHashIterate(pHash, pIter);
if (pIter == NULL) break;
pRebOutput = (SMqRebOutputVg *)pIter;
ASSERT(pRebOutput->newConsumerId == -1);
taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp);
taosArrayPush(pOutput->rebVgs, pRebOutput);
mInfo("mq rebalance: unassign vgId:%d (second scan)", pRebOutput->pVgEp->vgId);
mInfo("sub:%s, mq rebalance unassign vgId:%d (second scan)", sub, pRebOutput->pVgEp->vgId);
}
}
// 8. generate logs
mInfo("mq rebalance: calculation completed, rebalanced vg:");
mInfo("sub:%s, mq rebalance calculation completed, rebalanced vg", sub);
for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) {
SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i);
mInfo("mq rebalance: vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, pOutputRebVg->pVgEp->vgId,
pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
mInfo("sub:%s, mq rebalance vgId:%d, moved from consumer:%" PRId64 ", to consumer:%" PRId64, sub,
pOutputRebVg->pVgEp->vgId, pOutputRebVg->oldConsumerId, pOutputRebVg->newConsumerId);
}
{
void *pIter = NULL;
@ -425,10 +425,11 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR
if (pIter == NULL) break;
SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter;
int32_t sz = taosArrayGetSize(pConsumerEp->vgs);
mInfo("mq rebalance: final cfg: consumer %" PRId64 " has %d vg", pConsumerEp->consumerId, sz);
mInfo("sub:%s, mq rebalance final cfg: consumer %" PRId64 " has %d vg", sub, pConsumerEp->consumerId, sz);
for (int32_t i = 0; i < sz; i++) {
SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i);
mInfo("mq rebalance: final cfg: vg %d to consumer %" PRId64 "", pVgEp->vgId, pConsumerEp->consumerId);
mInfo("sub:%s, mq rebalance final cfg: vg %d to consumer %" PRId64 "", sub, pVgEp->vgId,
pConsumerEp->consumerId);
}
}
}
@ -487,7 +488,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
consumerNum = taosArrayGetSize(pOutput->newConsumers);
for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->newConsumers, i);
ASSERT(consumerId > 0);
SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
pConsumerNew->updateType = CONSUMER_UPDATE__ADD;
@ -497,7 +498,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
taosArrayPush(pConsumerNew->rebNewTopics, &topic);
mndReleaseConsumer(pMnode, pConsumerOld);
if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
ASSERT(0);
tDeleteSMqConsumerObj(pConsumerNew);
taosMemoryFree(pConsumerNew);
goto REB_FAIL;
@ -510,7 +510,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
consumerNum = taosArrayGetSize(pOutput->removedConsumers);
for (int32_t i = 0; i < consumerNum; i++) {
int64_t consumerId = *(int64_t *)taosArrayGet(pOutput->removedConsumers, i);
ASSERT(consumerId > 0);
SMqConsumerObj *pConsumerOld = mndAcquireConsumer(pMnode, consumerId);
SMqConsumerObj *pConsumerNew = tNewSMqConsumerObj(pConsumerOld->consumerId, pConsumerOld->cgroup);
pConsumerNew->updateType = CONSUMER_UPDATE__REMOVE;
@ -520,7 +520,6 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
taosArrayPush(pConsumerNew->rebRemovedTopics, &topic);
mndReleaseConsumer(pMnode, pConsumerOld);
if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) {
ASSERT(0);
tDeleteSMqConsumerObj(pConsumerNew);
taosMemoryFree(pConsumerNew);
goto REB_FAIL;
@ -577,7 +576,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
char cgroup[TSDB_CGROUP_LEN];
mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic);
/*ASSERT(pTopic);*/
if (pTopic == NULL) {
mError("mq rebalance %s failed since topic %s not exist, abort", pRebInfo->key, topic);
continue;
@ -585,8 +583,14 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
taosRLockLatch(&pTopic->lock);
rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key);
if (rebOutput.pSub == NULL) {
mError("mq rebalance %s failed create sub since %s, abort", pRebInfo->key, terrstr());
taosRUnLockLatch(&pTopic->lock);
mndReleaseTopic(pMnode, pTopic);
continue;
}
memcpy(rebOutput.pSub->dbName, pTopic->db, TSDB_DB_FNAME_LEN);
ASSERT(taosHashGetSize(rebOutput.pSub->consumerHash) == 0);
taosRUnLockLatch(&pTopic->lock);
mndReleaseTopic(pMnode, pTopic);
@ -606,7 +610,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) {
// if add more consumer to balanced subscribe,
// possibly no vg is changed
/*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) {
mError("mq rebalance persist rebalance output error, possibly vnode splitted or dropped");
@ -693,6 +696,7 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
void *buf = NULL;
int32_t tlen = tEncodeSubscribeObj(NULL, pSub);
if (tlen <= 0) goto SUB_ENCODE_OVER;
int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size);

8
source/dnode/mnode/impl/src/mndSync.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "mndSync.h"
#include "mndCluster.h"
#include "mndTrans.h"
static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
@ -297,9 +298,12 @@ int32_t mndInitSync(SMnode *pMnode) {
pCfg->myIndex = pMgmt->selfIndex;
for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) {
SNodeInfo *pNode = &pCfg->nodeInfo[i];
tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
pNode->nodeId = pMgmt->replicas[i].id;
pNode->nodePort = pMgmt->replicas[i].port;
mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort);
tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId,
pNode->clusterId);
}
tsem_init(&pMgmt->syncSem, 0, 0);

1
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -17,7 +17,6 @@
#include "mndTelem.h"
#include "mndCluster.h"
#include "mndSync.h"
#include "tbuffer.h"
#include "thttp.h"
#include "tjson.h"

8
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -384,7 +384,11 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
topicObj.subType = pCreate->subType;
topicObj.withMeta = pCreate->withMeta;
if (topicObj.withMeta) {
ASSERT(topicObj.subType != TOPIC_SUB_TYPE__COLUMN);
if (topicObj.subType == TOPIC_SUB_TYPE__COLUMN) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
return -1;
}
}
if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) {
@ -499,7 +503,6 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
if (code < 0) {
sdbRelease(pSdb, pVgroup);
mndTransDrop(pTrans);
ASSERT(0);
return -1;
}
void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len);
@ -723,7 +726,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
// TODO check if rebalancing
if (mndDropSubByTopic(pMnode, pTrans, dropReq.name) < 0) {
/*ASSERT(0);*/
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
mndTransDrop(pTrans);
mndReleaseTopic(pMnode, pTopic);

4
source/dnode/vnode/src/inc/tsdb.h
View File

@ -772,8 +772,8 @@ static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) {
return 0;
}
#define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
// #define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
// #define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
if (pIter == NULL) return NULL;

3
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -284,8 +284,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) {
handle.pRef = walOpenRef(pTq->pVnode->pWal);
if (handle.pRef == NULL) {
ASSERT(0);
return -1;
continue;
}
walRefVer(handle.pRef, handle.snapshotVer);

28
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -46,21 +46,37 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
}
int32_t size = htonl(head.size);
void* memBuf = taosMemoryCalloc(1, size);
if (memBuf == NULL) {
return -1;
}
if ((code = taosReadFile(pFile, memBuf, size)) != size) {
ASSERT(0);
// TODO handle error
taosMemoryFree(memBuf);
return -1;
}
STqOffset offset;
SDecoder decoder;
tDecoderInit(&decoder, memBuf, size);
if (tDecodeSTqOffset(&decoder, &offset) < 0) {
ASSERT(0);
taosMemoryFree(memBuf);
tDecoderClear(&decoder);
return -1;
}
tDecoderClear(&decoder);
if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) {
ASSERT(0);
// TODO
return -1;
}
if (offset.val.type == TMQ_OFFSET__LOG) {
STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey));
if (pHandle) {
if (walRefVer(pHandle->pRef, offset.val.version) < 0) {
tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId,
pHandle->subKey, offset.val.version);
}
}
}
taosMemoryFree(memBuf);
}
@ -124,7 +140,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) {
const char* sysErrStr = strerror(errno);
tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname,
sysErrStr);
ASSERT(0);
taosMemoryFree(fname);
return -1;
}
taosMemoryFree(fname);

2
source/dnode/vnode/src/tq/tqSink.c
View File

@ -64,7 +64,7 @@ int32_t tqBuildDeleteReq(SVnode* pVnode, const char* stbFullName, const SSDataBl
.startTs = startTs,
.endTs = endTs,
};
strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN);
strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1);
taosMemoryFree(name);
/*tqDebug("stream delete msg, active: vgId:%d, ts:%" PRId64 " name:%s", pVnode->config.vgId, ts, name);*/
taosArrayPush(deleteReq->deleteReqs, &req);

4
source/dnode/vnode/src/tq/tqSnapshot.c
View File

@ -175,6 +175,8 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
if (code) goto _err;
}
int vgId = TD_VID(pWriter->pTq->pVnode);
taosMemoryFree(pWriter);
*ppWriter = NULL;
@ -186,7 +188,7 @@ int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
return code;
_err:
tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code));
tqError("vgId:%d, tq snapshot writer close failed since %s", vgId, tstrerror(code));
return code;
}

97
source/dnode/vnode/src/tsdb/tsdbMemTable.c
View File

@ -19,9 +19,13 @@
#define SL_MAX_LEVEL 5
// sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l) * 2
#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4))
#define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
#define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4))
#define SL_NODE_FORWARD(n, l) ((n)->forwards[l])
#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
#define SL_GET_NODE_FORWARD(n, l) ((SMemSkipListNode *)atomic_load_64((int64_t *)&SL_NODE_FORWARD(n, l)))
#define SL_GET_NODE_BACKWARD(n, l) ((SMemSkipListNode *)atomic_load_64((int64_t *)&SL_NODE_BACKWARD(n, l)))
#define SL_SET_NODE_FORWARD(n, l, p) atomic_store_64((int64_t *)&SL_NODE_FORWARD(n, l), (int64_t)(p))
#define SL_SET_NODE_BACKWARD(n, l, p) atomic_store_64((int64_t *)&SL_NODE_BACKWARD(n, l), (int64_t)(p))
#define SL_MOVE_BACKWARD 0x1
#define SL_MOVE_FROM_POS 0x2
@ -246,18 +250,18 @@ void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDa
if (pFrom == NULL) {
// create from head or tail
if (backward) {
pIter->pNode = SL_NODE_BACKWARD(pTbData->sl.pTail, 0);
pIter->pNode = SL_GET_NODE_BACKWARD(pTbData->sl.pTail, 0);
} else {
pIter->pNode = SL_NODE_FORWARD(pTbData->sl.pHead, 0);
pIter->pNode = SL_GET_NODE_FORWARD(pTbData->sl.pHead, 0);
}
} else {
// create from a key
if (backward) {
tbDataMovePosTo(pTbData, pos, pFrom, SL_MOVE_BACKWARD);
pIter->pNode = SL_NODE_BACKWARD(pos[0], 0);
pIter->pNode = SL_GET_NODE_BACKWARD(pos[0], 0);
} else {
tbDataMovePosTo(pTbData, pos, pFrom, 0);
pIter->pNode = SL_NODE_FORWARD(pos[0], 0);
pIter->pNode = SL_GET_NODE_FORWARD(pos[0], 0);
}
}
}
@ -271,7 +275,7 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) {
return false;
}
pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0);
pIter->pNode = SL_GET_NODE_BACKWARD(pIter->pNode, 0);
if (pIter->pNode == pIter->pTbData->sl.pHead) {
return false;
}
@ -282,7 +286,7 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) {
return false;
}
pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0);
pIter->pNode = SL_GET_NODE_FORWARD(pIter->pNode, 0);
if (pIter->pNode == pIter->pTbData->sl.pTail) {
return false;
}
@ -335,7 +339,7 @@ static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid
int8_t maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
ASSERT(pPool != NULL);
pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
pTbData = vnodeBufPoolMallocAligned(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
if (pTbData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
@ -408,7 +412,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p
if (fromPos) px = pos[pTbData->sl.level - 1];
for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) {
pn = SL_NODE_BACKWARD(px, iLevel);
pn = SL_GET_NODE_BACKWARD(px, iLevel);
while (pn != pTbData->sl.pHead) {
tKey.version = pn->version;
tKey.ts = pn->pTSRow->ts;
@ -418,7 +422,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p
break;
} else {
px = pn;
pn = SL_NODE_BACKWARD(px, iLevel);
pn = SL_GET_NODE_BACKWARD(px, iLevel);
}
}
@ -438,7 +442,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p
if (fromPos) px = pos[pTbData->sl.level - 1];
for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) {
pn = SL_NODE_FORWARD(px, iLevel);
pn = SL_GET_NODE_FORWARD(px, iLevel);
while (pn != pTbData->sl.pTail) {
tKey.version = pn->version;
tKey.ts = pn->pTSRow->ts;
@ -448,7 +452,7 @@ static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *p
break;
} else {
px = pn;
pn = SL_NODE_FORWARD(px, iLevel);
pn = SL_GET_NODE_FORWARD(px, iLevel);
}
}
@ -474,58 +478,53 @@ static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListN
int8_t level;
SMemSkipListNode *pNode;
SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
int64_t nSize;
// node
// create node
level = tsdbMemSkipListRandLevel(&pTbData->sl);
ASSERT(pPool != NULL);
pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level));
nSize = SL_NODE_SIZE(level);
pNode = (SMemSkipListNode *)vnodeBufPoolMallocAligned(pPool, nSize + pRow->len);
if (pNode == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pNode->level = level;
pNode->version = version;
pNode->pTSRow = vnodeBufPoolMalloc(pPool, pRow->len);
if (NULL == pNode->pTSRow) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
pNode->pTSRow = (STSRow *)((char *)pNode + nSize);
memcpy(pNode->pTSRow, pRow, pRow->len);
for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) {
SMemSkipListNode *pn = pos[iLevel];
SMemSkipListNode *px;
if (forward) {
px = SL_NODE_FORWARD(pn, iLevel);
SL_NODE_BACKWARD(pNode, iLevel) = pn;
SL_NODE_FORWARD(pNode, iLevel) = px;
} else {
px = SL_NODE_BACKWARD(pn, iLevel);
SL_NODE_BACKWARD(pNode, iLevel) = px;
SL_NODE_FORWARD(pNode, iLevel) = pn;
// set node
if (forward) {
for (int8_t iLevel = 0; iLevel < level; iLevel++) {
SL_NODE_FORWARD(pNode, iLevel) = SL_NODE_FORWARD(pos[iLevel], iLevel);
SL_NODE_BACKWARD(pNode, iLevel) = pos[iLevel];
}
} else {
for (int8_t iLevel = 0; iLevel < level; iLevel++) {
SL_NODE_FORWARD(pNode, iLevel) = pos[iLevel];
SL_NODE_BACKWARD(pNode, iLevel) = SL_NODE_BACKWARD(pos[iLevel], iLevel);
}
}
for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) {
SMemSkipListNode *pn = pos[iLevel];
SMemSkipListNode *px;
// set forward and backward
if (forward) {
for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) {
SMemSkipListNode *pNext = pos[iLevel]->forwards[iLevel];
if (forward) {
px = SL_NODE_FORWARD(pn, iLevel);
SL_SET_NODE_FORWARD(pos[iLevel], iLevel, pNode);
SL_SET_NODE_BACKWARD(pNext, iLevel, pNode);
SL_NODE_FORWARD(pn, iLevel) = pNode;
SL_NODE_BACKWARD(px, iLevel) = pNode;
} else {
px = SL_NODE_BACKWARD(pn, iLevel);
SL_NODE_FORWARD(px, iLevel) = pNode;
SL_NODE_BACKWARD(pn, iLevel) = pNode;
pos[iLevel] = pNode;
}
} else {
for (int8_t iLevel = level - 1; iLevel >= 0; iLevel--) {
SMemSkipListNode *pPrev = pos[iLevel]->forwards[pos[iLevel]->level + iLevel];
pos[iLevel] = pNode;
SL_SET_NODE_FORWARD(pPrev, iLevel, pNode);
SL_SET_NODE_BACKWARD(pos[iLevel], iLevel, pNode);
pos[iLevel] = pNode;
}
}
pTbData->sl.size++;

107
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -76,6 +76,7 @@ static int32_t tFDataIterCmprFn(const SRBTreeNode* pNode1, const SRBTreeNode* pN
static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
int32_t code = 0;
int32_t lino = 0;
SDFileSet dFileSet = {.fid = pReader->fid};
SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
@ -83,7 +84,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
pReader->fid = pSet->fid;
code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
pReader->pIter = NULL;
tRBTreeCreate(&pReader->rbt, tFDataIterCmprFn);
@ -93,13 +94,13 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
pIter->type = SNAP_DATA_FILE_ITER;
code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) {
pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx);
code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) {
SDataBlk dataBlk;
@ -108,7 +109,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue;
code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid);
ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid);
@ -139,7 +140,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
pIter->iStt = iStt;
code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) {
SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
@ -148,7 +149,7 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
if (pSttBlk->maxVer < pReader->sver) continue;
code = tsdbReadSttBlockEx(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
@ -169,13 +170,13 @@ static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
pIter++;
}
tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode),
pReader->pTsdb->path, pReader->fid);
return code;
_err:
tsdbError("vgId:%d, vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode),
tstrerror(code));
_exit:
if (code) {
tsdbError("vgId:%d, %s failed since %s", TD_VID(pReader->pTsdb->pVnode), __func__, tstrerror(code));
} else {
tsdbInfo("vgId:%d, %s done, path:%s, fid:%d", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->pTsdb->path,
pReader->fid);
}
return code;
}
@ -318,12 +319,14 @@ _exit:
static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
int32_t code = 0;
STsdb* pTsdb = pReader->pTsdb;
int32_t lino = 0;
STsdb* pTsdb = pReader->pTsdb;
while (true) {
if (pReader->pDataFReader == NULL) {
code = tsdbSnapReadOpenFile(pReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
if (pReader->pDataFReader == NULL) break;
@ -338,17 +341,17 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
SBlockData* pBlockData = &pReader->bData;
code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataInit(pBlockData, &id, pReader->skmTable.pTSchema, NULL, 0);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) {
code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSnapNextRow(pReader);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbSnapGetRow(pReader);
if (pRowInfo == NULL) {
@ -360,21 +363,22 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
}
code = tsdbSnapCmprData(pReader, ppData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
break;
}
return code;
_err:
tsdbError("vgId:%d, vnode snapshot tsdb read data for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
_exit:
if (code) {
tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code), pTsdb->path);
}
return code;
}
static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
int32_t code = 0;
int32_t code = 0;
int32_t lino = 0;
STsdb* pTsdb = pReader->pTsdb;
SDelFile* pDelFile = pReader->fs.pDelFile;
@ -385,11 +389,11 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
// open
code = tsdbDelFReaderOpen(&pReader->pDelFReader, pDelFile, pTsdb);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
// read index
code = tsdbReadDelIdx(pReader->pDelFReader, pReader->aDelIdx);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
pReader->iDelIdx = 0;
}
@ -405,7 +409,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
pReader->iDelIdx++;
code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData);
if (code) goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
int32_t size = 0;
for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) {
@ -422,7 +426,7 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
if (*ppData == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
}
SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
@ -449,11 +453,9 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
}
_exit:
return code;
_err:
tsdbError("vgId:%d, vnode snapshot tsdb read del for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
if (code) {
tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pTsdb->pVnode), __func__, tstrerror(code), pTsdb->path);
}
return code;
}
@ -591,44 +593,39 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) {
int32_t code = 0;
int32_t lino = 0;
*ppData = NULL;
// read data file
if (!pReader->dataDone) {
code = tsdbSnapReadData(pReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->dataDone = 1;
}
pReader->dataDone = 1;
}
}
// read del file
if (!pReader->delDone) {
code = tsdbSnapReadDel(pReader, ppData);
if (code) {
goto _err;
TSDB_CHECK_CODE(code, lino, _exit);
if (*ppData) {
goto _exit;
} else {
if (*ppData) {
goto _exit;
} else {
pReader->delDone = 1;
}
pReader->delDone = 1;
}
}
_exit:
tsdbDebug("vgId:%d, vnode snapshot tsdb read for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path);
return code;
_err:
tsdbError("vgId:%d, vnode snapshot tsdb read for %s failed since %s", TD_VID(pReader->pTsdb->pVnode),
pReader->pTsdb->path, tstrerror(code));
if (code) {
tsdbError("vgId:%d, %s failed since %s, path:%s", TD_VID(pReader->pTsdb->pVnode), __func__, tstrerror(code),
pReader->pTsdb->path);
} else {
tsdbDebug("vgId:%d, %s done, path:%s", TD_VID(pReader->pTsdb->pVnode), __func__, pReader->pTsdb->path);
}
return code;
}

44
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -125,13 +125,22 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries) < 0) return -1;
SJson *pNodeInfoArr = tjsonCreateArray();
tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", pNodeInfoArr);
SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1;
vDebug("vgId:%d, encode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum,
pCfg->syncCfg.myIndex);
for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) {
SJson *pNodeInfo = tjsonCreateObject();
tjsonAddIntegerToObject(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
tjsonAddStringToObject(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
tjsonAddItemToArray(pNodeInfoArr, pNodeInfo);
SJson *info = tjsonCreateObject();
SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1;
if (tjsonAddIntegerToObject(info, "nodePort", pNode->nodePort) < 0) return -1;
if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1;
if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1;
if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1;
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId);
}
return 0;
@ -240,15 +249,24 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code);
if (code < 0) return -1;
SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(pNodeInfoArr);
assert(arraySize == pCfg->syncCfg.replicaNum);
SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(nodeInfo);
if (arraySize != pCfg->syncCfg.replicaNum) return -1;
vDebug("vgId:%d, decode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum,
pCfg->syncCfg.myIndex);
for (int i = 0; i < arraySize; ++i) {
SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i);
assert(pNodeInfo != NULL);
tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort, code);
tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
SJson *info = tjsonGetArrayItem(nodeInfo, i);
SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
if (info == NULL) return -1;
tjsonGetNumberValue(info, "nodePort", pNode->nodePort, code);
if (code < 0) return -1;
tjsonGetStringValue(info, "nodeFqdn", pNode->nodeFqdn);
if (code < 0) return -1;
tjsonGetNumberValue(info, "nodeId", pNode->nodeId, code);
tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code);
vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId);
}
tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code);

6
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -105,8 +105,8 @@ int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) {
// free info binary
taosMemoryFree(data);
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d", pInfo->config.vgId, fname,
pInfo->config.syncCfg.replicaNum);
vInfo("vgId:%d, vnode info is saved, fname:%s replica:%d selfIndex:%d", pInfo->config.vgId, fname,
pInfo->config.syncCfg.replicaNum, pInfo->config.syncCfg.myIndex);
return 0;
@ -206,6 +206,8 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
} else {
snprintf(dir, TSDB_FILENAME_LEN, "%s", pVnode->path);
}
vDebug("vgId:%d, save config while prepare commit", TD_VID(pVnode));
if (vnodeSaveInfo(dir, &pInfo->info) < 0) {
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);

7
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -48,6 +48,7 @@ int32_t vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
info.state.applied = -1;
info.state.commitID = 0;
vInfo("vgId:%d, save config while create", pCfg->vgId);
if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) {
vError("vgId:%d, failed to save vnode config since %s", pCfg ? pCfg->vgId : 0, tstrerror(terrno));
return -1;
@ -79,12 +80,14 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) {
pCfg->replicaNum = pReq->replica;
memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo));
vInfo("vgId:%d, save config, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex);
vInfo("vgId:%d, save config while alter, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex);
for (int i = 0; i < pReq->replica; ++i) {
SNodeInfo *pNode = &pCfg->nodeInfo[i];
pNode->nodeId = pReq->replicas[i].id;
pNode->nodePort = pReq->replicas[i].port;
tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
vInfo("vgId:%d, save config, replica:%d ep:%s:%u", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort);
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId);
}
info.config.syncCfg = *pCfg;

4
source/dnode/vnode/src/vnd/vnodeSnapshot.c
View File

@ -405,6 +405,10 @@ static int32_t vnodeSnapWriteInfo(SVSnapWriter *pWriter, uint8_t *pData, uint32_
} else {
snprintf(dir, TSDB_FILENAME_LEN, "%s", pWriter->pVnode->path);
}
SVnode *pVnode = pWriter->pVnode;
pWriter->info.config = pVnode->config;
vDebug("vgId:%d, save config while write snapshot", pWriter->pVnode->config.vgId);
if (vnodeSaveInfo(dir, &pWriter->info) < 0) {
code = terrno;
goto _exit;

3
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -578,7 +578,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
vInfo("vgId:%d, start to open sync, replica:%d selfIndex:%d", pVnode->config.vgId, pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SNodeInfo *pNode = &pCfg->nodeInfo[i];
vInfo("vgId:%d, index:%d ep:%s:%u", pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort);
vInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId, pNode->clusterId);
}
pVnode->sync = syncOpen(&syncInfo);

5
source/libs/executor/inc/executil.h
View File

@ -18,7 +18,6 @@
#include "function.h"
#include "nodes.h"
#include "plannodes.h"
#include "tbuffer.h"
#include "tcommon.h"
#include "tpagedbuf.h"
#include "tsimplehash.h"
@ -116,6 +115,10 @@ struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t i
static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) {
SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId);
if (NULL == bufPage) {
return NULL;
}
if (forUpdate) {
setBufPageDirty(bufPage, true);
}

13
source/libs/executor/src/dataInserter.c
View File

@ -41,6 +41,7 @@ typedef struct SDataInserterHandle {
SHashObj* pCols;
int32_t status;
bool queryEnd;
bool fullOrderColList;
uint64_t useconds;
uint64_t cachedSize;
TdThreadMutex mutex;
@ -125,7 +126,6 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
int64_t uid = pInserter->pNode->tableId;
int64_t suid = pInserter->pNode->stableId;
int32_t vgId = pInserter->pNode->vgId;
bool fullCol = (pInserter->pNode->pCols->length == pTSchema->numOfCols);
SSubmitReq* ret = NULL;
int32_t sz = taosArrayGetSize(pBlocks);
@ -176,7 +176,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
const STColumn* pColumn = &pTSchema->columns[k];
SColumnInfoData* pColData = NULL;
int16_t colIdx = k;
if (!fullCol) {
if (!pInserter->fullOrderColList) {
int16_t* slotId = taosHashGet(pInserter->pCols, &pColumn->colId, sizeof(pColumn->colId));
if (NULL == slotId) {
continue;
@ -212,7 +212,7 @@ int32_t dataBlockToSubmit(SDataInserterHandle* pInserter, SSubmitReq** pReq) {
tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k);
}
}
if (!fullCol) {
if (!pInserter->fullOrderColList) {
rb.hasNone = true;
}
tdSRowEnd(&rb);
@ -346,12 +346,19 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat
return TSDB_CODE_OUT_OF_MEMORY;
}
inserter->fullOrderColList = pInserterNode->pCols->length == inserter->pSchema->numOfCols;
inserter->pCols = taosHashInit(pInserterNode->pCols->length, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT),
false, HASH_NO_LOCK);
SNode* pNode = NULL;
int32_t i = 0;
FOREACH(pNode, pInserterNode->pCols) {
SColumnNode* pCol = (SColumnNode*)pNode;
taosHashPut(inserter->pCols, &pCol->colId, sizeof(pCol->colId), &pCol->slotId, sizeof(pCol->slotId));
if (inserter->fullOrderColList && pCol->colId != inserter->pSchema->columns[i].colId) {
inserter->fullOrderColList = false;
}
++i;
}
tsem_init(&inserter->ready, 0, 0);

6
source/libs/executor/src/executil.c
View File

@ -1726,8 +1726,10 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI
return w;
}
w = getResultRowByPos(pBuf, &pResultRowInfo->cur, false)->win;
SResultRow* pRow = getResultRowByPos(pBuf, &pResultRowInfo->cur, false);
if (pRow) {
w = pRow->win;
}
// in case of typical time window, we can calculate time window directly.
if (w.skey > ts || w.ekey < ts) {
w = doCalculateTimeWindow(ts, pInterval);

36
source/libs/executor/src/executorimpl.c
View File

@ -150,6 +150,11 @@ SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, i
pData->num = sizeof(SFilePage);
} else {
pData = getBufPage(pResultBuf, *currentPageId);
if (pData == NULL) {
qError("failed to get buffer, code:%s", tstrerror(terrno));
return NULL;
}
pageId = *currentPageId;
if (pData->num + interBufSize > getBufPageSize(pResultBuf)) {
@ -200,6 +205,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
if (isIntervalQuery) {
if (p1 != NULL) { // the *p1 may be NULL in case of sliding+offset exists.
pResult = getResultRowByPos(pResultBuf, p1, true);
if (NULL == pResult) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset);
}
} else {
@ -208,6 +217,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
if (p1 != NULL) {
// todo
pResult = getResultRowByPos(pResultBuf, p1, true);
if (NULL == pResult) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset);
}
}
@ -216,6 +229,10 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId))) {
SResultRowPosition pos = pResultRowInfo->cur;
SFilePage* pPage = getBufPage(pResultBuf, pos.pageId);
if (pPage == NULL) {
qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo));
T_LONG_JMP(pTaskInfo->env, terrno);
}
releaseBufPage(pResultBuf, pPage);
}
@ -223,6 +240,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
if (pResult == NULL) {
ASSERT(pSup->resultRowSize > 0);
pResult = getNewResultRow(pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
if (pResult == NULL) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
// add a new result set for a new group
SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
@ -260,6 +280,11 @@ static int32_t addNewWindowResultBuf(SResultRow* pWindowRes, SDiskbasedBuf* pRes
} else {
SPageInfo* pi = getLastPageInfo(list);
pData = getBufPage(pResultBuf, getPageId(pi));
if (pData == NULL) {
qError("failed to get buffer, code:%s", tstrerror(terrno));
return terrno;
}
pageId = getPageId(pi);
if (pData->num + size > getBufPageSize(pResultBuf)) {
@ -912,7 +937,7 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin
if (pResultRow->pageId == -1) {
int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->aggSup.pResultBuf, pAggInfo->binfo.pRes->info.rowSize);
if (ret != TSDB_CODE_SUCCESS) {
return;
T_LONG_JMP(pTaskInfo->env, terrno);
}
}
@ -993,6 +1018,11 @@ static void doCopyResultToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SR
int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup,
SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId);
if (page == NULL) {
qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo));
T_LONG_JMP(pTaskInfo->env, terrno);
}
SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset);
SqlFunctionCtx* pCtx = pSup->pCtx;
@ -1036,6 +1066,10 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) {
SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
SFilePage* page = getBufPage(pBuf, pPos->pos.pageId);
if (page == NULL) {
qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo));
T_LONG_JMP(pTaskInfo->env, terrno);
}
SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset);

19
source/libs/executor/src/groupoperator.c
View File

@ -492,13 +492,17 @@ _error:
static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
SPartitionOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j);
int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
SDataGroupInfo* pGroupInfo = NULL;
void* pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len);
if (pPage == NULL) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
pGroupInfo->numOfRows += 1;
@ -595,6 +599,10 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
} else {
int32_t* curId = taosArrayGetLast(p->pPageList);
pPage = getBufPage(pInfo->pBuf, *curId);
if (pPage == NULL) {
qError("failed to get buffer, code:%s", tstrerror(terrno));
return pPage;
}
int32_t* rows = (int32_t*)pPage;
if (*rows >= pInfo->rowCapacity) {
@ -674,7 +682,8 @@ static int compareDataGroupInfo(const void* group1, const void* group2) {
static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
SPartitionOperatorInfo* pInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SDataGroupInfo* pGroupInfo =
(pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL;
if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) {
@ -692,7 +701,11 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex);
void* page = getBufPage(pInfo->pBuf, *pageId);
if (page == NULL) {
qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo));
T_LONG_JMP(pTaskInfo->env, terrno);
}
blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity);
blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity);

4
source/libs/executor/src/scanoperator.c
View File

@ -170,6 +170,10 @@ static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t gro
}
*pPage = getBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, p1->pageId);
if (NULL == *pPage) {
return NULL;
}
return (SResultRow*)((char*)(*pPage) + p1->offset);
}

14
source/libs/executor/src/timewindowoperator.c
View File

@ -636,6 +636,10 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
}
SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1, false);
if (NULL == pr) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId);
if (pr->closed) {
@ -1315,6 +1319,10 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type
static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) {
SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false);
if (NULL == pResult) {
return;
}
SqlFunctionCtx* pCtx = pSup->pCtx;
for (int32_t i = 0; i < numOfOutput; ++i) {
pCtx[i].resultInfo = getResultEntryInfo(pResult, i, pSup->rowEntryInfoOffset);
@ -1328,6 +1336,9 @@ static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf,
}
}
SFilePage* bufPage = getBufPage(pResultBuf, p1->pageId);
if (NULL == bufPage) {
return;
}
setBufPageDirty(bufPage, true);
releaseBufPage(pResultBuf, bufPage);
}
@ -4114,6 +4125,9 @@ void destroyMAIOperatorInfo(void* param) {
static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) {
SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize);
if (NULL == pResult) {
return pResult;
}
pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset};
return pResult;
}

9
source/libs/executor/src/tsort.c
View File

@ -270,6 +270,10 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32
int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
void* pPage = getBufPage(pHandle->pBuf, *pPgId);
if (NULL == pPage) {
return terrno;
}
code = blockDataFromBuf(pSource->src.pBlock, pPage);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -337,6 +341,11 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
void* pPage = getBufPage(pHandle->pBuf, *pPgId);
if (pPage == NULL) {
qError("failed to get buffer, code:%s", tstrerror(terrno));
return terrno;
}
int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage);
if (code != TSDB_CODE_SUCCESS) {
return code;

2
source/libs/function/src/builtinsimpl.c
View File

@ -784,7 +784,7 @@ int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
pEntryInfo->isNullRes = (pEntryInfo->numOfRes == 0) ? 1 : 0;
if (pCol->info.type == TSDB_DATA_TYPE_FLOAT) {
float v = *(float*)&pRes->v;
float v = GET_FLOAT_VAL(&pRes->v);
colDataAppend(pCol, currentRow, (const char*)&v, pEntryInfo->isNullRes);
} else {
colDataAppend(pCol, currentRow, (const char*)&pRes->v, pEntryInfo->isNullRes);

15
source/libs/function/src/detail/tminmax.c
View File

@ -737,7 +737,12 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
}
if (!pBuf->assign) {
pBuf->v = *(int64_t*)tval;
if (type == TSDB_DATA_TYPE_FLOAT) {
GET_FLOAT_VAL(&pBuf->v) = GET_DOUBLE_VAL(tval);
} else {
pBuf->v = GET_INT64_VAL(tval);
}
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
@ -751,7 +756,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
int64_t val = GET_INT64_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(int64_t*)&pBuf->v = val;
GET_INT64_VAL(&pBuf->v) = val;
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
@ -765,7 +770,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
uint64_t val = GET_UINT64_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(uint64_t*)&pBuf->v = val;
GET_UINT64_VAL(&pBuf->v) = val;
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
@ -779,7 +784,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
double val = GET_DOUBLE_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(double*)&pBuf->v = val;
GET_DOUBLE_VAL(&pBuf->v) = val;
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
@ -793,7 +798,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
float val = GET_DOUBLE_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(float*)&pBuf->v = val;
GET_FLOAT_VAL(&pBuf->v) = val;
}
if (pCtx->subsidiaries.num > 0) {

1
source/libs/function/src/tfunctionInt.c
View File

@ -20,7 +20,6 @@
#include "ttypes.h"
#include "function.h"
#include "tbuffer.h"
#include "tcompression.h"
#include "tdatablock.h"
#include "tfunctionInt.h"

5
source/libs/function/src/udfd.c
View File

@ -460,13 +460,14 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
#else
snprintf(path, sizeof(path), "%s/lib%s.so", tsTempDir, pFuncInfo->name);
#endif
TdFilePtr file =
taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (file == NULL) {
fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno));
msgInfo->code = TSDB_CODE_FILE_CORRUPTED;
goto _return;
}
int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
if (count != pFuncInfo->codeSize) {
fnError("udfd write udf shared library failed");

1
source/libs/index/inc/indexUtil.h
View File

@ -36,7 +36,6 @@ extern "C" {
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(type) == sizeof(c)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)

4
source/libs/index/src/index.c
View File

@ -226,7 +226,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
indexDebug("w suid:%" PRIu64 ", colName:%s, colType:%d", key.suid, key.colName, key.colType);
IndexCache** cache = taosHashGet(index->colObj, buf, sz);
assert(*cache != NULL);
ASSERTS(*cache != NULL, "index-cache already release");
if (*cache == NULL) return -1;
int ret = idxCachePut(*cache, p, uid);
if (ret != 0) {
return ret;

5
source/libs/index/src/indexComm.c
View File

@ -170,7 +170,6 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d
}
return tDoCompare(func, cmptype, &va, &vb);
}
assert(0);
return BREAK;
#endif
}
@ -367,7 +366,7 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) {
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
default:
ASSERT(0);
ASSERTS(0, "index invalid input type");
break;
}
*dst = (char*)*dst - tlen;
@ -459,7 +458,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
*dst = (char*)*dst - tlen;
break;
default:
ASSERT(0);
ASSERTS(0, "index invalid input type");
break;
}
return tlen;

4
source/libs/index/src/indexFilter.c
View File

@ -206,7 +206,9 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) {
static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
SOperatorNode *nd = (SOperatorNode *)node;
assert(nodeType(node) == QUERY_NODE_OPERATOR);
if (nodeType(node) != QUERY_NODE_OPERATOR) {
return -1;
}
SColumnNode *l = (SColumnNode *)nd->pLeft;
SValueNode *r = (SValueNode *)nd->pRight;

60
source/libs/index/src/indexFst.c
View File

@ -65,10 +65,7 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) {
taosArrayPush(nodes->stack, &un);
}
FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) {
assert(taosArrayGetSize(nodes->stack) == 1);
FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
assert(un->last == NULL);
return un->node;
}
@ -82,7 +79,6 @@ FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledA
FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) {
FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
assert(un->last == NULL);
return un->node;
}
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) {
@ -102,7 +98,8 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output
}
int32_t sz = taosArrayGetSize(nodes->stack) - 1;
FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz);
assert(un->last == NULL);
ASSERTS(un->last == NULL, "index-fst meet unexpected node");
if (un->last != NULL) return;
// FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition));
// trn->inp = s->data[s->start];
@ -247,7 +244,6 @@ void fstStateCompileForOneTrans(IdxFstFile* w, CompiledAddr addr, FstTransition*
}
void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode* node) {
int32_t sz = taosArrayGetSize(node->trans);
assert(sz <= 256);
uint8_t tSize = 0;
uint8_t oSize = packSize(node->finalOutput);
@ -322,7 +318,7 @@ void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode
// set_comm_input
void fstStateSetCommInput(FstState* s, uint8_t inp) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
uint8_t val;
COMMON_INDEX(inp, 0b111111, val);
@ -331,7 +327,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) {
// comm_input
uint8_t fstStateCommInput(FstState* s, bool* null) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
uint8_t v = s->val & 0b00111111;
if (v == 0) {
*null = true;
@ -344,7 +340,7 @@ uint8_t fstStateCommInput(FstState* s, bool* null) {
// input_len
uint64_t fstStateInputLen(FstState* s) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
bool null = false;
fstStateCommInput(s, &null);
return null ? 1 : 0;
@ -352,11 +348,11 @@ uint64_t fstStateInputLen(FstState* s) {
// end_addr
uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) {
assert(s->state == OneTransNext);
ASSERT(s->state == OneTransNext);
return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s);
}
uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) {
assert(s->state == OneTrans);
ASSERT(s->state == OneTrans);
return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size
- FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes);
}
@ -370,7 +366,7 @@ uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice*
}
// input
uint8_t fstStateInput(FstState* s, FstNode* node) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
FstSlice* slice = &node->data;
bool null = false;
uint8_t inp = fstStateCommInput(s, &null);
@ -378,7 +374,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) {
return null == false ? inp : data[node->start - 1];
}
uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
FstSlice* slice = &node->data;
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
@ -390,7 +386,7 @@ uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
// trans_addr
CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
FstSlice* slice = &node->data;
if (s->state == OneTransNext) {
return (CompiledAddr)(node->end) - 1;
@ -406,7 +402,7 @@ CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) {
}
}
CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
FstSlice* slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
@ -418,7 +414,7 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i
// sizes
PackSizes fstStateSizes(FstState* s, FstSlice* slice) {
assert(s->state == OneTrans || s->state == AnyTrans);
ASSERT(s->state == OneTrans || s->state == AnyTrans);
uint64_t i;
if (s->state == OneTrans) {
i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1;
@ -431,7 +427,7 @@ PackSizes fstStateSizes(FstState* s, FstSlice* slice) {
}
// Output
Output fstStateOutput(FstState* s, FstNode* node) {
assert(s->state == OneTrans);
ASSERT(s->state == OneTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) {
@ -445,7 +441,7 @@ Output fstStateOutput(FstState* s, FstNode* node) {
return unpackUint64(data + i, oSizes);
}
Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) {
@ -462,19 +458,19 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
// anyTrans specify function
void fstStateSetFinalState(FstState* s, bool yes) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
if (yes) {
s->val |= 0b01000000;
}
return;
}
bool fstStateIsFinalState(FstState* s) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
return (s->val & 0b01000000) == 0b01000000;
}
void fstStateSetStateNtrans(FstState* s, uint8_t n) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
if (n <= 0b00111111) {
s->val = (s->val & 0b11000000) | n;
}
@ -482,7 +478,7 @@ void fstStateSetStateNtrans(FstState* s, uint8_t n) {
}
// state_ntrans
uint8_t fstStateStateNtrans(FstState* s, bool* null) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
*null = false;
uint8_t n = s->val & 0b00111111;
@ -492,16 +488,16 @@ uint8_t fstStateStateNtrans(FstState* s, bool* null) {
return n;
}
uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans);
return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize;
}
uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0;
}
uint64_t fstStateNtransLen(FstState* s) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
bool null = false;
fstStateStateNtrans(s, &null);
return null == true ? 1 : 0;
@ -530,7 +526,7 @@ Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackS
return unpackUint64(data + at, (uint8_t)oSizes);
}
uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
FstSlice* slice = &node->data;
if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) {
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
@ -676,17 +672,17 @@ bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) {
bool s = true;
FstState* st = &node->state;
if (st->state == OneTransNext) {
assert(i == 0);
ASSERT(i == 0);
fstStateTransAddr(st, node);
} else if (st->state == OneTrans) {
assert(i == 0);
ASSERT(i == 0);
fstStateTransAddr(st, node);
} else if (st->state == AnyTrans) {
fstStateTransAddrForAnyTrans(st, node, i);
} else if (FST_STATE_EMPTY_FINAL(node)) {
s = false;
} else {
assert(0);
ASSERT(0);
}
return s;
}
@ -722,7 +718,7 @@ bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) {
bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) {
int32_t sz = taosArrayGetSize(builderNode->trans);
assert(sz < 256);
ASSERT(sz < 256);
if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) {
return true;
} else if (sz != 1 || builderNode->isFinal) {
@ -804,7 +800,7 @@ void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) {
uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out);
if (prefixLen == FST_SLICE_LEN(s)) {
assert(out == 0);
ASSERT(out == 0);
return;
}
@ -848,7 +844,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) {
addr = fstBuilderCompile(b, bn);
fstBuilderNodeDestroy(bn);
assert(addr != NONE_ADDRESS);
ASSERT(addr != NONE_ADDRESS);
}
fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
return;

5
source/libs/index/src/indexFstDfa.c
View File

@ -104,8 +104,9 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet
DfaState *t = taosArrayGet(builder->dfa->states, state);
for (int i = 0; i < taosArrayGetSize(t->insts); i++) {
int32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
bool succ = sparSetAdd(cur, ip, NULL);
assert(succ == true);
bool succ = sparSetAdd(cur, ip, NULL);
if (succ == false) return false;
}
dfaRun(builder->dfa, cur, next, byte);

14
source/libs/index/src/indexFstFile.c
View File

@ -100,7 +100,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
do {
char key[1024] = {0};
assert(strlen(ctx->file.buf) + 1 + 64 < sizeof(key));
ASSERT(strlen(ctx->file.buf) + 1 + 64 < sizeof(key));
idxGenLRUKey(key, ctx->file.buf, blkId);
LRUHandle* h = taosLRUCacheLookup(ctx->lru, key, strlen(key));
@ -114,7 +114,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
if (left < kBlockSize) {
nread = TMIN(left, len);
int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset);
assert(bytes == nread);
ASSERTS(bytes == nread, "index read incomplete data");
if (bytes != nread) break;
total += bytes;
return total;
@ -124,7 +125,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize);
blk->blockId = blkId;
blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize);
assert(blk->nread <= kBlockSize);
ASSERTS(blk->nread <= kBlockSize, "index read incomplete data");
if (blk->nread > kBlockSize) break;
if (blk->nread < kBlockSize && blk->nread < len) {
taosMemoryFree(blk);
@ -275,7 +277,10 @@ int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) {
// update checksum
IFileCtx* ctx = write->wrt;
int nWrite = ctx->write(ctx, buf, len);
assert(nWrite == len);
ASSERTS(nWrite == len, "index write incomplete data");
if (nWrite != len) {
return -1;
}
write->count += len;
write->summer = taosCalcChecksum(write->summer, buf, len);
@ -302,7 +307,6 @@ int idxFileFlush(IdxFstFile* write) {
}
void idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes) {
assert(1 <= nBytes && nBytes <= 8);
uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t));
for (uint8_t i = 0; i < nBytes; i++) {
buf[i] = (uint8_t)n;

4
source/libs/index/src/indexFstRegister.c
View File

@ -57,8 +57,8 @@ static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) {
if (start >= sz && end >= sz) {
return;
}
assert(start >= end);
ASSERTS(start >= end, "index-fst start lower than end");
if (start < end) return;
int32_t s = (int32_t)start;
int32_t e = (int32_t)end;

1
source/libs/index/src/indexFstUtil.c
View File

@ -101,7 +101,6 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) {
int32_t slen;
uint8_t* data = fstSliceData(s, &slen);
assert(tlen <= slen);
uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen);
memcpy(buf, data + start, tlen);

16
source/libs/index/src/indexTfile.c
View File

@ -122,7 +122,6 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) {
char buf[128] = {0};
int32_t sz = idxSerialCacheKey(&key, buf);
assert(sz < sizeof(buf));
taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
tfileReaderRef(reader);
}
@ -151,9 +150,8 @@ void tfileCacheDestroy(TFileCache* tcache) {
}
TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
char buf[128] = {0};
int32_t sz = idxSerialCacheKey(key, buf);
assert(sz < sizeof(buf));
char buf[128] = {0};
int32_t sz = idxSerialCacheKey(key, buf);
TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
if (reader == NULL || *reader == NULL) {
return NULL;
@ -877,7 +875,7 @@ static int tfileWriteFooter(TFileWriter* write) {
int nwrite = write->ctx->write(write->ctx, buf, (int32_t)strlen(buf));
indexInfo("tfile write footer size: %d", write->ctx->size(write->ctx));
assert(nwrite == sizeof(FILE_MAGIC_NUMBER));
ASSERTS(nwrite == sizeof(FILE_MAGIC_NUMBER), "index write incomplete data");
return nwrite;
}
static int tfileReaderLoadHeader(TFileReader* reader) {
@ -892,7 +890,6 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
} else {
indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf);
}
// assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf));
return 0;
@ -914,7 +911,10 @@ static int tfileReaderLoadFst(TFileReader* reader) {
indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %d, time cost: %" PRId64 "us", nread,
reader->header.fstOffset, fstSize, ctx->file.buf, size, cost);
// we assuse fst size less than FST_MAX_SIZE
assert(nread > 0 && nread <= fstSize);
ASSERTS(nread > 0 && nread <= fstSize, "index read incomplete fst");
if (nread <= 0 || nread > fstSize) {
return -1;
}
FstSlice st = fstSliceCreate((uint8_t*)buf, nread);
reader->fst = fstCreate(&st);
@ -929,7 +929,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
// add block cache
char block[4096] = {0};
int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset);
assert(nread >= sizeof(uint32_t));
ASSERT(nread >= sizeof(uint32_t));
char* p = block;
int32_t nid = *(int32_t*)p;

1
source/libs/stream/src/streamMeta.c
View File

@ -294,6 +294,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) {
tdbTbcClose(pCur);
return -1;
}
pTask->taskStatus = TASK_STATUS__NORMAL;
}
tdbFree(pKey);

2
source/libs/stream/src/streamState.c
View File

@ -107,7 +107,7 @@ static inline int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2,
}
SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages) {
szPage = szPage < 0 ? (16 * 1024) : szPage;
szPage = szPage < 0 ? 4096 : szPage;
pages = pages < 0 ? 256 : pages;
SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState));
if (pState == NULL) {

16
source/libs/sync/inc/syncInt.h
View File

@ -53,6 +53,18 @@ typedef struct SyncPreSnapshot SyncPreSnapshot;
typedef struct SSyncLogBuffer SSyncLogBuffer;
typedef struct SSyncLogReplMgr SSyncLogReplMgr;
#define MAX_CONFIG_INDEX_COUNT 256
typedef struct SRaftCfg {
SSyncCfg cfg;
int32_t batchSize;
int8_t isStandBy;
int8_t snapshotStrategy;
SyncIndex lastConfigIndex;
int32_t configIndexCount;
SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT];
} SRaftCfg;
typedef struct SRaftId {
SyncNodeId addr;
SyncGroupId vgId;
@ -93,7 +105,7 @@ typedef struct SPeerState {
typedef struct SSyncNode {
// init by SSyncInfo
SyncGroupId vgId;
SRaftCfg* pRaftCfg;
SRaftCfg raftCfg;
char path[TSDB_FILENAME_LEN];
char raftStorePath[TSDB_FILENAME_LEN * 2];
char configPath[TSDB_FILENAME_LEN * 2];
@ -112,6 +124,7 @@ typedef struct SSyncNode {
int32_t peersNum;
SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA];
SEpSet peersEpset[TSDB_MAX_REPLICA];
SRaftId peersId[TSDB_MAX_REPLICA];
int32_t replicaNum;
@ -245,7 +258,6 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode);
// utils --------------
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode);
int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h);
bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode);

61
source/libs/sync/inc/syncRaftCfg.h
View File

@ -22,64 +22,9 @@ extern "C" {
#include "syncInt.h"
#define CONFIG_FILE_LEN 2048
#define MAX_CONFIG_INDEX_COUNT 256
typedef struct SRaftCfgIndex {
TdFilePtr pFile;
char path[TSDB_FILENAME_LEN * 2];
SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT];
int32_t configIndexCount;
} SRaftCfgIndex;
SRaftCfgIndex *raftCfgIndexOpen(const char *path);
int32_t raftCfgIndexClose(SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexPersist(SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexAddConfigIndex(SRaftCfgIndex *pRaftCfgIndex, SyncIndex configIndex);
cJSON *raftCfgIndex2Json(SRaftCfgIndex *pRaftCfgIndex);
char *raftCfgIndex2Str(SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexFromJson(const cJSON *pRoot, SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexFromStr(const char *s, SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexCreateFile(const char *path);
typedef struct SRaftCfg {
SSyncCfg cfg;
TdFilePtr pFile;
char path[TSDB_FILENAME_LEN * 2];
int8_t isStandBy;
int32_t batchSize;
int8_t snapshotStrategy;
SyncIndex lastConfigIndex;
SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT];
int32_t configIndexCount;
} SRaftCfg;
SRaftCfg *raftCfgOpen(const char *path);
int32_t raftCfgClose(SRaftCfg *pRaftCfg);
int32_t raftCfgPersist(SRaftCfg *pRaftCfg);
int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex);
void syncCfg2SimpleStr(const SSyncCfg *pCfg, char *str, int32_t bufLen);
cJSON *syncCfg2Json(SSyncCfg *pSyncCfg);
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg);
cJSON *raftCfg2Json(SRaftCfg *pRaftCfg);
char *raftCfg2Str(SRaftCfg *pRaftCfg);
int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg);
int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg);
typedef struct SRaftCfgMeta {
int8_t isStandBy;
int32_t batchSize;
int8_t snapshotStrategy;
SyncIndex lastConfigIndex;
} SRaftCfgMeta;
int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path);
int32_t syncWriteCfgFile(SSyncNode *pNode);
int32_t syncReadCfgFile(SSyncNode *pNode);
int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex);
#ifdef __cplusplus
}

19
source/libs/sync/inc/syncUtil.h
View File

@ -62,22 +62,19 @@ extern "C" {
// clang-format on
uint64_t syncUtilAddr2U64(const char* host, uint16_t port);
void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port);
void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet);
void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet);
bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId);
bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2);
bool syncUtilEmptyId(const SRaftId* pId);
#define CID(pRaftId) (int32_t)(((pRaftId)->addr) >> 32)
#define DID(pRaftId) (int32_t)((pRaftId)->addr)
#define SYNC_ADDR(pInfo) (int64_t)(((pInfo)->clusterId << 32) | (pInfo)->nodeId)
void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet);
bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId);
bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2);
bool syncUtilEmptyId(const SRaftId* pId);
int32_t syncUtilElectRandomMS(int32_t min, int32_t max);
int32_t syncUtilQuorum(int32_t replicaNum);
cJSON* syncUtilRaftId2Json(const SRaftId* p);
const char* syncStr(ESyncState state);
char* syncUtilPrintBin(char* ptr, uint32_t len);
char* syncUtilPrintBin2(char* ptr, uint32_t len);
void syncUtilMsgHtoN(void* msg);
void syncUtilMsgNtoH(void* msg);
bool syncUtilUserPreCommit(tmsg_t msgType);
bool syncUtilUserRollback(tmsg_t msgType);

2
source/libs/sync/src/syncElection.c
View File

@ -94,7 +94,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode);
votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum);
syncNodeCandidate2Leader(pSyncNode);
pSyncNode->pVotesGranted->toLeader = true;

2
source/libs/sync/src/syncEnv.c
View File

@ -115,7 +115,7 @@ void syncHbTimerDataRemove(int64_t rid) { taosRemoveRef(gHbDataRefId, rid); }
SSyncHbTimerData *syncHbTimerDataAcquire(int64_t rid) {
SSyncHbTimerData *pData = taosAcquireRef(gHbDataRefId, rid);
if (pData == NULL) {
sError("failed to acquire hb-timer-data from refId:%" PRId64, rid);
sInfo("failed to acquire hb-timer-data from refId:%" PRId64, rid);
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
}

54
source/libs/sync/src/syncIndexMgr.c
View File

@ -64,10 +64,8 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, Sync
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr set index:%" PRId64 " for %s:%d failed", pIndexMgr->pNode->vgId, index, host, port);
sError("vgId:%d, indexmgr set index:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, index,
DID(pRaftId), CID(pRaftId));
}
SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) {
@ -77,10 +75,7 @@ SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) {
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr get replmgr from %s:%d failed", pNode->vgId, host, port);
sError("vgId:%d, indexmgr get replmgr from dnode:%d cluster:%d failed", pNode->vgId, DID(pRaftId), CID(pRaftId));
return NULL;
}
@ -92,10 +87,8 @@ SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId)
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr get index from %s:%d failed", pIndexMgr->pNode->vgId, host, port);
sError("vgId:%d, indexmgr get index from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return SYNC_INDEX_INVALID;
}
@ -107,11 +100,8 @@ void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId,
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr set start-time:%" PRId64 " for %s:%d failed", pIndexMgr->pNode->vgId, startTime, host,
port);
sError("vgId:%d, indexmgr set start-time:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId,
startTime, DID(pRaftId), CID(pRaftId));
}
int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
@ -122,10 +112,8 @@ int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftI
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr get start-time from %s:%d failed", pIndexMgr->pNode->vgId, host, port);
sError("vgId:%d, indexmgr get start-time from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return -1;
}
@ -137,10 +125,8 @@ void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, i
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr set recv-time:%" PRId64 " for %s:%d failed", pIndexMgr->pNode->vgId, recvTime, host, port);
sError("vgId:%d, indexmgr set recv-time:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, recvTime,
DID(pRaftId), CID(pRaftId));
}
int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
@ -151,10 +137,8 @@ int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr get recv-time from %s:%d failed", pIndexMgr->pNode->vgId, host, port);
sError("vgId:%d, indexmgr get recv-time from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return -1;
}
@ -166,10 +150,8 @@ void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncT
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr set term:%" PRId64 " for %s:%d failed", pIndexMgr->pNode->vgId, term, host, port);
sError("vgId:%d, indexmgr set term:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, term,
DID(pRaftId), CID(pRaftId));
}
SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
@ -180,9 +162,7 @@ SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
}
}
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, indexmgr get term from %s:%d failed", pIndexMgr->pNode->vgId, host, port);
sError("vgId:%d, indexmgr get term from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return -1;
}

303
source/libs/sync/src/syncMain.c
View File

@ -306,15 +306,10 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) {
for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {
int64_t matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[i]));
if (lastApplyIndex > matchIndex) {
do {
char host[64];
uint16_t port;
syncUtilU642Addr(pSyncNode->peersId[i].addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"new-snapshot-index:%" PRId64 " is greater than match-index:%" PRId64
" of %s:%d, do not delete wal",
lastApplyIndex, matchIndex, host, port);
} while (0);
sNTrace(pSyncNode,
"new-snapshot-index:%" PRId64 " is greater than match-index:%" PRId64
" of dnode:%d, do not delete wal",
lastApplyIndex, matchIndex, DID(&pSyncNode->peersId[i]));
syncNodeRelease(pSyncNode);
return 0;
@ -556,7 +551,7 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) {
(void)syncBuildLeaderTransfer(&rpcMsg, pSyncNode->vgId);
SyncLeaderTransfer* pMsg = rpcMsg.pCont;
pMsg->newLeaderId.addr = syncUtilAddr2U64(newLeader.nodeFqdn, newLeader.nodePort);
pMsg->newLeaderId.addr = SYNC_ADDR(&newLeader);
pMsg->newLeaderId.vgId = pSyncNode->vgId;
pMsg->newNodeInfo = newLeader;
@ -622,9 +617,9 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) {
return -1;
}
ASSERT(rid == pSyncNode->rid);
sMeta->lastConfigIndex = pSyncNode->pRaftCfg->lastConfigIndex;
sMeta->lastConfigIndex = pSyncNode->raftCfg.lastConfigIndex;
sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->pRaftCfg->lastConfigIndex);
sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->raftCfg.lastConfigIndex);
syncNodeRelease(pSyncNode);
return 0;
@ -637,13 +632,13 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct
}
ASSERT(rid == pSyncNode->rid);
ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0];
ASSERT(pSyncNode->raftCfg.configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0];
for (int32_t i = 0; i < pSyncNode->pRaftCfg->configIndexCount; ++i) {
if ((pSyncNode->pRaftCfg->configIndexArr)[i] > lastIndex &&
(pSyncNode->pRaftCfg->configIndexArr)[i] <= snapshotIndex) {
lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.configIndexCount; ++i) {
if ((pSyncNode->raftCfg.configIndexArr)[i] > lastIndex &&
(pSyncNode->raftCfg.configIndexArr)[i] <= snapshotIndex) {
lastIndex = (pSyncNode->raftCfg.configIndexArr)[i];
}
}
sMeta->lastConfigIndex = lastIndex;
@ -656,13 +651,13 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct
#endif
SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapshotLastApplyIndex) {
ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0];
ASSERT(pSyncNode->raftCfg.configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0];
for (int32_t i = 0; i < pSyncNode->pRaftCfg->configIndexCount; ++i) {
if ((pSyncNode->pRaftCfg->configIndexArr)[i] > lastIndex &&
(pSyncNode->pRaftCfg->configIndexArr)[i] <= snapshotLastApplyIndex) {
lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.configIndexCount; ++i) {
if ((pSyncNode->raftCfg.configIndexArr)[i] > lastIndex &&
(pSyncNode->raftCfg.configIndexArr)[i] <= snapshotLastApplyIndex) {
lastIndex = (pSyncNode->raftCfg.configIndexArr)[i];
}
}
sTrace("vgId:%d, sync get last config index, index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId,
@ -677,15 +672,15 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) {
SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) return;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
SEp* pEp = &pEpSet->eps[i];
tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN);
pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort;
tstrncpy(pEp->fqdn, pSyncNode->raftCfg.cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN);
pEp->port = (pSyncNode->raftCfg.cfg.nodeInfo)[i].nodePort;
pEpSet->numOfEps++;
sDebug("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port);
}
if (pEpSet->numOfEps > 0) {
pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps;
pEpSet->inUse = (pSyncNode->raftCfg.cfg.myIndex + 1) % pEpSet->numOfEps;
}
sInfo("vgId:%d, sync get retry epset numOfEps:%d inUse:%d", pSyncNode->vgId, pEpSet->numOfEps, pEpSet->inUse);
@ -849,60 +844,57 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
}
}
snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP);
if (!taosCheckExistFile(pSyncNode->configPath)) {
// create a new raft config file
SRaftCfgMeta meta = {0};
meta.isStandBy = pSyncInfo->isStandBy;
meta.snapshotStrategy = pSyncInfo->snapshotStrategy;
meta.lastConfigIndex = SYNC_INDEX_INVALID;
meta.batchSize = pSyncInfo->batchSize;
if (raftCfgCreateFile(&pSyncInfo->syncCfg, meta, pSyncNode->configPath) != 0) {
sError("vgId:%d, failed to create raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
if (pSyncInfo->syncCfg.replicaNum == 0) {
sInfo("vgId:%d, sync config not input", pSyncNode->vgId);
pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
}
} else {
// update syncCfg by raft_config.json
pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
if (pSyncNode->pRaftCfg == NULL) {
sError("vgId:%d, failed to open raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->pRaftCfg->cfg, &pSyncInfo->syncCfg)) {
sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId);
pSyncNode->pRaftCfg->cfg = pSyncInfo->syncCfg;
if (raftCfgPersist(pSyncNode->pRaftCfg) != 0) {
sError("vgId:%d, failed to persist raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
} else {
sInfo("vgId:%d, use sync config from raft cfg file", pSyncNode->vgId);
pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
}
raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
}
// init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId;
SSyncCfg* pCfg = &pSyncInfo->syncCfg;
sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SNodeInfo* pNode = &pCfg->nodeInfo[i];
sInfo("vgId:%d, index:%d ep:%s:%u", pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort);
}
memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path));
snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s%sraft_store.json", pSyncInfo->path,
TD_DIRSEP);
snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP);
if (!taosCheckExistFile(pSyncNode->configPath)) {
// create a new raft config file
sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId);
pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy;
pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy;
pSyncNode->raftCfg.lastConfigIndex = SYNC_INDEX_INVALID;
pSyncNode->raftCfg.batchSize = pSyncInfo->batchSize;
pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg;
pSyncNode->raftCfg.configIndexCount = 1;
pSyncNode->raftCfg.configIndexArr[0] = -1;
if (syncWriteCfgFile(pSyncNode) != 0) {
sError("vgId:%d, failed to create sync cfg file", pSyncNode->vgId);
goto _error;
}
} else {
// update syncCfg by raft_config.json
if (syncReadCfgFile(pSyncNode) != 0) {
sError("vgId:%d, failed to read sync cfg file", pSyncNode->vgId);
goto _error;
}
if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) {
sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId);
pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg;
if (syncWriteCfgFile(pSyncNode) != 0) {
sError("vgId:%d, failed to write sync cfg file", pSyncNode->vgId);
goto _error;
}
} else {
sInfo("vgId:%d, use sync config from sync cfg file", pSyncNode->vgId);
pSyncInfo->syncCfg = pSyncNode->raftCfg.cfg;
}
}
// init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId;
SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg;
sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SNodeInfo* pNode = &pCfg->nodeInfo[i];
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
sInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId, pNode->clusterId);
}
pSyncNode->pWal = pSyncInfo->pWal;
pSyncNode->msgcb = pSyncInfo->msgcb;
pSyncNode->syncSendMSg = pSyncInfo->syncSendMSg;
@ -916,26 +908,20 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
goto _error;
}
// init raft config
pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
if (pSyncNode->pRaftCfg == NULL) {
sError("vgId:%d, failed to open raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
// init internal
pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex];
pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex];
if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) {
sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId);
goto _error;
}
// init peersNum, peers, peersId
pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1;
pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1;
int32_t j = 0;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
if (i != pSyncNode->pRaftCfg->cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
if (i != pSyncNode->raftCfg.cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i];
syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]);
j++;
}
}
@ -947,9 +933,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
}
// init replicaNum, replicasId
pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
if (!syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) {
pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
if (!syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) {
sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i);
goto _error;
}
@ -958,7 +944,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
// init raft algorithm
pSyncNode->pFsm = pSyncInfo->pFsm;
pSyncInfo->pFsm = NULL;
pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum);
pSyncNode->leaderCache = EMPTY_RAFT_ID;
// init life cycle outside
@ -1293,8 +1279,6 @@ void syncNodeClose(SSyncNode* pSyncNode) {
pSyncNode->pLogStore = NULL;
syncLogBufferDestroy(pSyncNode->pLogBuf);
pSyncNode->pLogBuf = NULL;
raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
syncNodeStopPingTimer(pSyncNode);
syncNodeStopElectTimer(pSyncNode);
@ -1330,7 +1314,7 @@ void syncNodeClose(SSyncNode* pSyncNode) {
taosMemoryFree(pSyncNode);
}
ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotStrategy; }
ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg.snapshotStrategy; }
// timer control --------------
int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) {
@ -1392,7 +1376,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) {
void syncNodeResetElectTimer(SSyncNode* pSyncNode) {
int32_t electMS;
if (pSyncNode->pRaftCfg->isStandBy) {
if (pSyncNode->raftCfg.isStandBy) {
electMS = TIMER_MAX_MS;
} else {
electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine);
@ -1461,55 +1445,46 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) {
return 0;
}
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
SEpSet epSet;
syncUtilRaftId2EpSet(destRaftId, &epSet);
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pNode, SRpcMsg* pMsg) {
SEpSet* epSet = NULL;
for (int32_t i = 0; i < pNode->peersNum; ++i) {
if (destRaftId->addr == pNode->peersId[i].addr) {
epSet = &pNode->peersEpset[i];
break;
}
}
if (pSyncNode->syncSendMSg != NULL) {
if (pNode->syncSendMSg != NULL && epSet != NULL) {
syncUtilMsgHtoN(pMsg->pCont);
pMsg->info.noResp = 1;
return pSyncNode->syncSendMSg(&epSet, pMsg);
return pNode->syncSendMSg(epSet, pMsg);
} else {
sError("vgId:%d, sync send msg by id error, fp-send-msg is null", pSyncNode->vgId);
sError("vgId:%d, sync send msg by id error, fp:%p epset:%p", pNode->vgId, pNode->syncSendMSg, epSet);
rpcFreeCont(pMsg->pCont);
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return -1;
}
}
int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
SEpSet epSet;
syncUtilNodeInfo2EpSet(nodeInfo, &epSet);
if (pSyncNode->syncSendMSg != NULL) {
// htonl
syncUtilMsgHtoN(pMsg->pCont);
pMsg->info.noResp = 1;
pSyncNode->syncSendMSg(&epSet, pMsg);
} else {
sError("vgId:%d, sync send msg by info error, fp-send-msg is null", pSyncNode->vgId);
}
return 0;
}
inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) {
inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) {
bool b1 = false;
bool b2 = false;
for (int32_t i = 0; i < config->replicaNum; ++i) {
if (strcmp((config->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 &&
(config->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) {
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
if (strcmp(pCfg->nodeInfo[i].nodeFqdn, pNode->myNodeInfo.nodeFqdn) == 0 &&
pCfg->nodeInfo[i].nodePort == pNode->myNodeInfo.nodePort) {
b1 = true;
break;
}
}
for (int32_t i = 0; i < config->replicaNum; ++i) {
SRaftId raftId;
raftId.addr = syncUtilAddr2U64((config->nodeInfo)[i].nodeFqdn, (config->nodeInfo)[i].nodePort);
raftId.vgId = pSyncNode->vgId;
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SRaftId raftId = {
.addr = SYNC_ADDR(&pCfg->nodeInfo[i]),
.vgId = pNode->vgId,
};
if (syncUtilSameId(&raftId, &(pSyncNode->myRaftId))) {
if (syncUtilSameId(&raftId, &pNode->myRaftId)) {
b2 = true;
break;
}
@ -1533,14 +1508,14 @@ static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg
}
void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex) {
SSyncCfg oldConfig = pSyncNode->pRaftCfg->cfg;
SSyncCfg oldConfig = pSyncNode->raftCfg.cfg;
if (!syncIsConfigChanged(&oldConfig, pNewConfig)) {
sInfo("vgId:1, sync not reconfig since not changed");
return;
}
pSyncNode->pRaftCfg->cfg = *pNewConfig;
pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex;
pSyncNode->raftCfg.cfg = *pNewConfig;
pSyncNode->raftCfg.lastConfigIndex = lastConfigChangeIndex;
pSyncNode->configChangeNum++;
@ -1563,21 +1538,18 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// log begin config change
char oldCfgStr[1024] = {0};
char newCfgStr[1024] = {0};
syncCfg2SimpleStr(&oldConfig, oldCfgStr, sizeof(oldCfgStr));
syncCfg2SimpleStr(pNewConfig, oldCfgStr, sizeof(oldCfgStr));
sNInfo(pSyncNode, "begin do config change, from %s to %s", oldCfgStr, oldCfgStr);
sNInfo(pSyncNode, "begin do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum,
pNewConfig->replicaNum);
if (IamInNew) {
pSyncNode->pRaftCfg->isStandBy = 0; // change isStandBy to normal
pSyncNode->raftCfg.isStandBy = 0; // change isStandBy to normal
}
if (isDrop) {
pSyncNode->pRaftCfg->isStandBy = 1; // set standby
pSyncNode->raftCfg.isStandBy = 1; // set standby
}
// add last config index
raftCfgAddConfigIndex(pSyncNode->pRaftCfg, lastConfigChangeIndex);
syncAddCfgIndex(pSyncNode, lastConfigChangeIndex);
if (IamInNew) {
//-----------------------------------------
@ -1594,15 +1566,16 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// init internal
pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex];
pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex];
syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId);
// init peersNum, peers, peersId
pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1;
pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1;
int32_t j = 0;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
if (i != pSyncNode->pRaftCfg->cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
if (i != pSyncNode->raftCfg.cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i];
syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]);
j++;
}
}
@ -1611,13 +1584,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// init replicaNum, replicasId
pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
}
// update quorum first
pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum);
syncIndexMgrUpdate(pSyncNode->pNextIndex, pSyncNode);
syncIndexMgrUpdate(pSyncNode->pMatchIndex, pSyncNode);
@ -1637,11 +1610,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
bool reset = false;
for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) {
if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) {
char host[128];
uint16_t port;
syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port);
sNTrace(pSyncNode, "snapshot sender reset for: %" PRId64 ", newIndex:%d, %s:%d, %p",
(pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]);
sNTrace(pSyncNode, "snapshot sender reset for:%" PRId64 ", newIndex:%d, dnode:%d, %p",
(pSyncNode->replicasId)[i].addr, i, DID(&pSyncNode->replicasId[i]), oldSenders[j]);
pSyncNode->senders[i] = oldSenders[j];
oldSenders[j] = NULL;
@ -1651,8 +1621,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
int32_t oldreplicaIndex = pSyncNode->senders[i]->replicaIndex;
pSyncNode->senders[i]->replicaIndex = i;
sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", oldreplicaIndex,
i, host, port, pSyncNode->senders[i], reset);
sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, dnode:%d, %p, reset:%d",
oldreplicaIndex, i, DID(&pSyncNode->replicasId[i]), pSyncNode->senders[i], reset);
break;
}
@ -1684,33 +1654,29 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// persist cfg
raftCfgPersist(pSyncNode->pRaftCfg);
char tmpbuf[1024] = {0};
snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, index:%" PRId64 ", %s --> %s",
oldConfig.replicaNum, pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr);
syncWriteCfgFile(pSyncNode);
// change isStandBy to normal (election timeout)
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
syncNodeBecomeLeader(pSyncNode, tmpbuf);
syncNodeBecomeLeader(pSyncNode, "");
// Raft 3.6.2 Committing entries from previous terms
syncNodeAppendNoop(pSyncNode);
// syncMaybeAdvanceCommitIndex(pSyncNode);
} else {
syncNodeBecomeFollower(pSyncNode, tmpbuf);
syncNodeBecomeFollower(pSyncNode, "");
}
} else {
// persist cfg
raftCfgPersist(pSyncNode->pRaftCfg);
sNInfo(pSyncNode, "do not config change from %d to %d, index:%" PRId64 ", %s --> %s", oldConfig.replicaNum,
pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr);
syncWriteCfgFile(pSyncNode);
sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum);
}
_END:
// log end config change
sNInfo(pSyncNode, "end do config change, from %s to %s", oldCfgStr, newCfgStr);
sNInfo(pSyncNode, "end do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum,
pNewConfig->replicaNum);
}
// raft state change --------------
@ -2889,9 +2855,10 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p
int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg) {
for (int32_t i = 0; i < pNewCfg->replicaNum; ++i) {
SRaftId raftId;
raftId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort);
raftId.vgId = ths->vgId;
SRaftId raftId = {
.addr = SYNC_ADDR(&pNewCfg->nodeInfo[i]),
.vgId = ths->vgId,
};
if (syncUtilSameId(&(ths->myRaftId), &raftId)) {
pNewCfg->myIndex = i;

23
source/libs/sync/src/syncPipeline.c
View File

@ -652,18 +652,15 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod
SSyncLogBuffer* pBuf = pNode->pLogBuf;
SRaftId destId = pMsg->srcId;
ASSERT(pMgr->restored == false);
char host[64];
uint16_t port;
syncUtilU642Addr(destId.addr, host, sizeof(host), &port);
if (pMgr->endIndex == 0) {
ASSERT(pMgr->startIndex == 0);
ASSERT(pMgr->matchIndex == 0);
if (pMsg->matchIndex < 0) {
pMgr->restored = true;
sInfo("vgId:%d, sync log repl mgr restored. peer: %s:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
pNode->vgId, host, port, destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
return 0;
}
@ -678,21 +675,21 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod
if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) {
pMgr->matchIndex = pMsg->matchIndex;
pMgr->restored = true;
sInfo("vgId:%d, sync log repl mgr restored. peer: %s:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
pNode->vgId, host, port, destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
return 0;
}
if (pMsg->success == false && pMsg->matchIndex >= pMsg->lastSendIndex) {
sWarn("vgId:%d, failed to rollback match index. peer: %s:%d, match index: %" PRId64 ", last sent: %" PRId64,
pNode->vgId, host, port, pMsg->matchIndex, pMsg->lastSendIndex);
sWarn("vgId:%d, failed to rollback match index. peer: dnode:%d, match index: %" PRId64 ", last sent: %" PRId64,
pNode->vgId, DID(&destId), pMsg->matchIndex, pMsg->lastSendIndex);
if (syncNodeStartSnapshot(pNode, &destId) < 0) {
sError("vgId:%d, failed to start snapshot for peer %s:%d", pNode->vgId, host, port);
sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId));
return -1;
}
sInfo("vgId:%d, snapshot replication to peer %s:%d", pNode->vgId, host, port);
sInfo("vgId:%d, snapshot replication to peer dnode:%d", pNode->vgId, DID(&destId));
return 0;
}
}
@ -707,10 +704,10 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod
if (term < 0 || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) {
ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
if (syncNodeStartSnapshot(pNode, &destId) < 0) {
sError("vgId:%d, failed to start snapshot for peer %s:%d", pNode->vgId, host, port);
sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId));
return -1;
}
sInfo("vgId:%d, snapshot replication to peer %s:%d", pNode->vgId, host, port);
sInfo("vgId:%d, snapshot replication to peer dnode:%d", pNode->vgId, DID(&destId));
return 0;
}

540
source/libs/sync/src/syncRaftCfg.c
View File

@ -16,382 +16,226 @@
#define _DEFAULT_SOURCE
#include "syncRaftCfg.h"
#include "syncUtil.h"
#include "tjson.h"
// file must already exist!
SRaftCfgIndex *raftCfgIndexOpen(const char *path) {
SRaftCfgIndex *pRaftCfgIndex = taosMemoryMalloc(sizeof(SRaftCfg));
snprintf(pRaftCfgIndex->path, sizeof(pRaftCfgIndex->path), "%s", path);
static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) {
SSyncCfg *pCfg = (SSyncCfg *)pObj;
if (tjsonAddDoubleToObject(pJson, "replicaNum", pCfg->replicaNum) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "myIndex", pCfg->myIndex) < 0) return -1;
pRaftCfgIndex->pFile = taosOpenFile(pRaftCfgIndex->path, TD_FILE_READ | TD_FILE_WRITE);
ASSERT(pRaftCfgIndex->pFile != NULL);
taosLSeekFile(pRaftCfgIndex->pFile, 0, SEEK_SET);
int32_t bufLen = MAX_CONFIG_INDEX_COUNT * 16;
char *pBuf = taosMemoryMalloc(bufLen);
memset(pBuf, 0, bufLen);
int64_t len = taosReadFile(pRaftCfgIndex->pFile, pBuf, bufLen);
ASSERT(len > 0);
int32_t ret = raftCfgIndexFromStr(pBuf, pRaftCfgIndex);
ASSERT(ret == 0);
taosMemoryFree(pBuf);
return pRaftCfgIndex;
}
int32_t raftCfgIndexClose(SRaftCfgIndex *pRaftCfgIndex) {
if (pRaftCfgIndex != NULL) {
int64_t ret = taosCloseFile(&(pRaftCfgIndex->pFile));
ASSERT(ret == 0);
taosMemoryFree(pRaftCfgIndex);
}
return 0;
}
int32_t raftCfgIndexPersist(SRaftCfgIndex *pRaftCfgIndex) {
ASSERT(pRaftCfgIndex != NULL);
char *s = raftCfgIndex2Str(pRaftCfgIndex);
taosLSeekFile(pRaftCfgIndex->pFile, 0, SEEK_SET);
int64_t ret = taosWriteFile(pRaftCfgIndex->pFile, s, strlen(s) + 1);
ASSERT(ret == strlen(s) + 1);
taosMemoryFree(s);
taosFsyncFile(pRaftCfgIndex->pFile);
return 0;
}
int32_t raftCfgIndexAddConfigIndex(SRaftCfgIndex *pRaftCfgIndex, SyncIndex configIndex) {
ASSERT(pRaftCfgIndex->configIndexCount <= MAX_CONFIG_INDEX_COUNT);
(pRaftCfgIndex->configIndexArr)[pRaftCfgIndex->configIndexCount] = configIndex;
++(pRaftCfgIndex->configIndexCount);
return 0;
}
cJSON *raftCfgIndex2Json(SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfgIndex->configIndexCount);
cJSON *pIndexArr = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr);
for (int i = 0; i < pRaftCfgIndex->configIndexCount; ++i) {
char buf64[128];
snprintf(buf64, sizeof(buf64), "%" PRId64, (pRaftCfgIndex->configIndexArr)[i]);
cJSON *pIndexObj = cJSON_CreateObject();
cJSON_AddStringToObject(pIndexObj, "index", buf64);
cJSON_AddItemToArray(pIndexArr, pIndexObj);
}
cJSON *pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SRaftCfgIndex", pRoot);
return pJson;
}
char *raftCfgIndex2Str(SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pJson = raftCfgIndex2Json(pRaftCfgIndex);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
int32_t raftCfgIndexFromJson(const cJSON *pRoot, SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pJson = cJSON_GetObjectItem(pRoot, "SRaftCfgIndex");
cJSON *pJsonConfigIndexCount = cJSON_GetObjectItem(pJson, "configIndexCount");
pRaftCfgIndex->configIndexCount = cJSON_GetNumberValue(pJsonConfigIndexCount);
cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr");
int arraySize = cJSON_GetArraySize(pIndexArr);
ASSERT(arraySize == pRaftCfgIndex->configIndexCount);
memset(pRaftCfgIndex->configIndexArr, 0, sizeof(pRaftCfgIndex->configIndexArr));
for (int i = 0; i < arraySize; ++i) {
cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i);
ASSERT(pIndexObj != NULL);
cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index");
ASSERT(cJSON_IsString(pIndex));
(pRaftCfgIndex->configIndexArr)[i] = atoll(pIndex->valuestring);
SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "nodeInfo", nodeInfo) < 0) return -1;
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SJson *info = tjsonCreateObject();
if (info == NULL) return -1;
if (tjsonAddDoubleToObject(info, "nodePort", pCfg->nodeInfo[i].nodePort) < 0) return -1;
if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn) < 0) return -1;
if (tjsonAddIntegerToObject(info, "nodeId", pCfg->nodeInfo[i].nodeId) < 0) return -1;
if (tjsonAddIntegerToObject(info, "clusterId", pCfg->nodeInfo[i].clusterId) < 0) return -1;
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
}
return 0;
}
int32_t raftCfgIndexFromStr(const char *s, SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pRoot = cJSON_Parse(s);
ASSERT(pRoot != NULL);
static int32_t syncEncodeRaftCfg(const void *pObj, SJson *pJson) {
SRaftCfg *pCfg = (SRaftCfg *)pObj;
if (tjsonAddObject(pJson, "SSyncCfg", syncEncodeSyncCfg, (void *)&pCfg->cfg) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "isStandBy", pCfg->isStandBy) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "snapshotStrategy", pCfg->snapshotStrategy) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "batchSize", pCfg->batchSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "lastConfigIndex", pCfg->lastConfigIndex) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "configIndexCount", pCfg->configIndexCount) < 0) return -1;
int32_t ret = raftCfgIndexFromJson(pRoot, pRaftCfgIndex);
ASSERT(ret == 0);
SJson *configIndexArr = tjsonCreateArray();
if (configIndexArr == NULL) return -1;
if (tjsonAddItemToObject(pJson, "configIndexArr", configIndexArr) < 0) return -1;
for (int32_t i = 0; i < pCfg->configIndexCount; ++i) {
SJson *configIndex = tjsonCreateObject();
if (configIndex == NULL) return -1;
if (tjsonAddIntegerToObject(configIndex, "index", pCfg->configIndexArr[i]) < 0) return -1;
if (tjsonAddItemToArray(configIndexArr, configIndex) < 0) return -1;
}
cJSON_Delete(pRoot);
return 0;
}
int32_t raftCfgIndexCreateFile(const char *path) {
TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
int32_t syncWriteCfgFile(SSyncNode *pNode) {
int32_t code = -1;
char *buffer = NULL;
SJson *pJson = NULL;
TdFilePtr pFile = NULL;
const char *realfile = pNode->configPath;
SRaftCfg *pCfg = &pNode->raftCfg;
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s.bak", realfile);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
int32_t sysErr = errno;
const char *sysErrStr = strerror(errno);
sError("create raft cfg index file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr,
sysErrStr);
ASSERT(0);
return -1;
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to open sync cfg file:%s since %s", pNode->vgId, realfile, terrstr());
goto _OVER;
}
SRaftCfgIndex raftCfgIndex;
memset(raftCfgIndex.configIndexArr, 0, sizeof(raftCfgIndex.configIndexArr));
raftCfgIndex.configIndexCount = 1;
raftCfgIndex.configIndexArr[0] = -1;
terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (tjsonAddObject(pJson, "RaftCfg", syncEncodeRaftCfg, pCfg) < 0) goto _OVER;
char *s = raftCfgIndex2Str(&raftCfgIndex);
int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1);
ASSERT(ret == strlen(s) + 1);
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
taosMemoryFree(s);
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
return 0;
}
// ---------------------------------------
// file must already exist!
SRaftCfg *raftCfgOpen(const char *path) {
SRaftCfg *pCfg = taosMemoryMalloc(sizeof(SRaftCfg));
snprintf(pCfg->path, sizeof(pCfg->path), "%s", path);
pCfg->pFile = taosOpenFile(pCfg->path, TD_FILE_READ | TD_FILE_WRITE);
ASSERT(pCfg->pFile != NULL);
taosLSeekFile(pCfg->pFile, 0, SEEK_SET);
char buf[CONFIG_FILE_LEN] = {0};
int len = taosReadFile(pCfg->pFile, buf, sizeof(buf));
ASSERT(len > 0);
int32_t ret = raftCfgFromStr(buf, pCfg);
ASSERT(ret == 0);
return pCfg;
}
int32_t raftCfgClose(SRaftCfg *pRaftCfg) {
int64_t ret = taosCloseFile(&(pRaftCfg->pFile));
ASSERT(ret == 0);
taosMemoryFree(pRaftCfg);
return 0;
}
int32_t raftCfgPersist(SRaftCfg *pRaftCfg) {
ASSERT(pRaftCfg != NULL);
char *s = raftCfg2Str(pRaftCfg);
taosLSeekFile(pRaftCfg->pFile, 0, SEEK_SET);
char buf[CONFIG_FILE_LEN] = {0};
memset(buf, 0, sizeof(buf));
if (strlen(s) + 1 > CONFIG_FILE_LEN) {
sError("too long config str:%s", s);
ASSERT(0);
if (taosRenameFile(file, realfile) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to rename sync cfg file:%s to %s since %s", pNode->vgId, file, realfile, terrstr());
goto _OVER;
}
snprintf(buf, sizeof(buf), "%s", s);
int64_t ret = taosWriteFile(pRaftCfg->pFile, buf, sizeof(buf));
ASSERT(ret == sizeof(buf));
code = 0;
sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d", pNode->vgId, realfile, len);
// int64_t ret = taosWriteFile(pRaftCfg->pFile, s, strlen(s) + 1);
// ASSERT(ret == strlen(s) + 1);
_OVER:
if (pJson != NULL) tjsonDelete(pJson);
if (buffer != NULL) taosMemoryFree(buffer);
if (pFile != NULL) taosCloseFile(&pFile);
taosMemoryFree(s);
taosFsyncFile(pRaftCfg->pFile);
return 0;
}
int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex) {
ASSERT(pRaftCfg->configIndexCount <= MAX_CONFIG_INDEX_COUNT);
(pRaftCfg->configIndexArr)[pRaftCfg->configIndexCount] = configIndex;
++(pRaftCfg->configIndexCount);
return 0;
}
cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
char u64buf[128] = {0};
cJSON *pRoot = cJSON_CreateObject();
if (pSyncCfg != NULL) {
cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncCfg->replicaNum);
cJSON_AddNumberToObject(pRoot, "myIndex", pSyncCfg->myIndex);
cJSON *pNodeInfoArr = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "nodeInfo", pNodeInfoArr);
for (int i = 0; i < pSyncCfg->replicaNum; ++i) {
cJSON *pNodeInfo = cJSON_CreateObject();
cJSON_AddNumberToObject(pNodeInfo, "nodePort", ((pSyncCfg->nodeInfo)[i]).nodePort);
cJSON_AddStringToObject(pNodeInfo, "nodeFqdn", ((pSyncCfg->nodeInfo)[i]).nodeFqdn);
cJSON_AddItemToArray(pNodeInfoArr, pNodeInfo);
}
if (code != 0) {
sError("vgId:%d, failed to write sync cfg file:%s since %s", pNode->vgId, realfile, terrstr());
}
return pRoot;
}
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) {
memset(pSyncCfg, 0, sizeof(SSyncCfg));
// cJSON *pJson = cJSON_GetObjectItem(pRoot, "SSyncCfg");
const cJSON *pJson = pRoot;
cJSON *pReplicaNum = cJSON_GetObjectItem(pJson, "replicaNum");
ASSERT(cJSON_IsNumber(pReplicaNum));
pSyncCfg->replicaNum = cJSON_GetNumberValue(pReplicaNum);
cJSON *pMyIndex = cJSON_GetObjectItem(pJson, "myIndex");
ASSERT(cJSON_IsNumber(pMyIndex));
pSyncCfg->myIndex = cJSON_GetNumberValue(pMyIndex);
cJSON *pNodeInfoArr = cJSON_GetObjectItem(pJson, "nodeInfo");
int arraySize = cJSON_GetArraySize(pNodeInfoArr);
ASSERT(arraySize == pSyncCfg->replicaNum);
for (int i = 0; i < arraySize; ++i) {
cJSON *pNodeInfo = cJSON_GetArrayItem(pNodeInfoArr, i);
ASSERT(pNodeInfo != NULL);
cJSON *pNodePort = cJSON_GetObjectItem(pNodeInfo, "nodePort");
ASSERT(cJSON_IsNumber(pNodePort));
((pSyncCfg->nodeInfo)[i]).nodePort = cJSON_GetNumberValue(pNodePort);
cJSON *pNodeFqdn = cJSON_GetObjectItem(pNodeInfo, "nodeFqdn");
ASSERT(cJSON_IsString(pNodeFqdn));
snprintf(((pSyncCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pSyncCfg->nodeInfo)[i]).nodeFqdn), "%s",
pNodeFqdn->valuestring);
}
return 0;
}
cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
cJSON *pRoot = cJSON_CreateObject();
cJSON_AddItemToObject(pRoot, "SSyncCfg", syncCfg2Json(&(pRaftCfg->cfg)));
cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy);
cJSON_AddNumberToObject(pRoot, "snapshotStrategy", pRaftCfg->snapshotStrategy);
cJSON_AddNumberToObject(pRoot, "batchSize", pRaftCfg->batchSize);
char buf64[128];
snprintf(buf64, sizeof(buf64), "%" PRId64, pRaftCfg->lastConfigIndex);
cJSON_AddStringToObject(pRoot, "lastConfigIndex", buf64);
cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfg->configIndexCount);
cJSON *pIndexArr = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr);
for (int i = 0; i < pRaftCfg->configIndexCount; ++i) {
snprintf(buf64, sizeof(buf64), "%" PRId64, (pRaftCfg->configIndexArr)[i]);
cJSON *pIndexObj = cJSON_CreateObject();
cJSON_AddStringToObject(pIndexObj, "index", buf64);
cJSON_AddItemToArray(pIndexArr, pIndexObj);
}
cJSON *pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "RaftCfg", pRoot);
return pJson;
}
char *raftCfg2Str(SRaftCfg *pRaftCfg) {
cJSON *pJson = raftCfg2Json(pRaftCfg);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) {
TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
if (pFile == NULL) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
int32_t sysErr = errno;
const char *sysErrStr = strerror(errno);
sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr);
return -1;
}
SRaftCfg raftCfg;
raftCfg.cfg = *pCfg;
raftCfg.isStandBy = meta.isStandBy;
raftCfg.batchSize = meta.batchSize;
raftCfg.snapshotStrategy = meta.snapshotStrategy;
raftCfg.lastConfigIndex = meta.lastConfigIndex;
raftCfg.configIndexCount = 1;
memset(raftCfg.configIndexArr, 0, sizeof(raftCfg.configIndexArr));
raftCfg.configIndexArr[0] = -1;
char *s = raftCfg2Str(&raftCfg);
char buf[CONFIG_FILE_LEN] = {0};
memset(buf, 0, sizeof(buf));
ASSERT(strlen(s) + 1 <= CONFIG_FILE_LEN);
snprintf(buf, sizeof(buf), "%s", s);
int64_t ret = taosWriteFile(pFile, buf, sizeof(buf));
ASSERT(ret == sizeof(buf));
// int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1);
// ASSERT(ret == strlen(s) + 1);
taosMemoryFree(s);
taosCloseFile(&pFile);
return 0;
}
int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
// memset(pRaftCfg, 0, sizeof(SRaftCfg));
cJSON *pJson = cJSON_GetObjectItem(pRoot, "RaftCfg");
cJSON *pJsonIsStandBy = cJSON_GetObjectItem(pJson, "isStandBy");
pRaftCfg->isStandBy = cJSON_GetNumberValue(pJsonIsStandBy);
cJSON *pJsonBatchSize = cJSON_GetObjectItem(pJson, "batchSize");
pRaftCfg->batchSize = cJSON_GetNumberValue(pJsonBatchSize);
cJSON *pJsonSnapshotStrategy = cJSON_GetObjectItem(pJson, "snapshotStrategy");
pRaftCfg->snapshotStrategy = cJSON_GetNumberValue(pJsonSnapshotStrategy);
cJSON *pJsonLastConfigIndex = cJSON_GetObjectItem(pJson, "lastConfigIndex");
pRaftCfg->lastConfigIndex = atoll(cJSON_GetStringValue(pJsonLastConfigIndex));
cJSON *pJsonConfigIndexCount = cJSON_GetObjectItem(pJson, "configIndexCount");
pRaftCfg->configIndexCount = cJSON_GetNumberValue(pJsonConfigIndexCount);
cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr");
int arraySize = cJSON_GetArraySize(pIndexArr);
ASSERT(arraySize == pRaftCfg->configIndexCount);
memset(pRaftCfg->configIndexArr, 0, sizeof(pRaftCfg->configIndexArr));
for (int i = 0; i < arraySize; ++i) {
cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i);
ASSERT(pIndexObj != NULL);
cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index");
ASSERT(cJSON_IsString(pIndex));
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
}
cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
ASSERT(code == 0);
return code;
}
int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg) {
cJSON *pRoot = cJSON_Parse(s);
ASSERT(pRoot != NULL);
static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) {
SSyncCfg *pCfg = (SSyncCfg *)pObj;
int32_t code = 0;
int32_t ret = raftCfgFromJson(pRoot, pRaftCfg);
ASSERT(ret == 0);
tjsonGetInt32ValueFromDouble(pJson, "replicaNum", pCfg->replicaNum, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "myIndex", pCfg->myIndex, code);
if (code < 0) return -1;
SJson *nodeInfo = tjsonGetObjectItem(pJson, "nodeInfo");
if (nodeInfo == NULL) return -1;
pCfg->replicaNum = tjsonGetArraySize(nodeInfo);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SJson *info = tjsonGetArrayItem(nodeInfo, i);
if (info == NULL) return -1;
tjsonGetUInt16ValueFromDouble(info, "nodePort", pCfg->nodeInfo[i].nodePort, code);
if (code < 0) return -1;
code = tjsonGetStringValue(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn);
if (code < 0) return -1;
tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code);
tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code);
}
cJSON_Delete(pRoot);
return 0;
}
static int32_t syncDecodeRaftCfg(const SJson *pJson, void *pObj) {
SRaftCfg *pCfg = (SRaftCfg *)pObj;
int32_t code = 0;
if (tjsonToObject(pJson, "SSyncCfg", syncDecodeSyncCfg, (void *)&pCfg->cfg) < 0) return -1;
tjsonGetInt8ValueFromDouble(pJson, "isStandBy", pCfg->isStandBy, code);
if (code < 0) return -1;
tjsonGetInt8ValueFromDouble(pJson, "snapshotStrategy", pCfg->snapshotStrategy, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "batchSize", pCfg->batchSize, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "lastConfigIndex", pCfg->lastConfigIndex, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "configIndexCount", pCfg->configIndexCount, code);
SJson *configIndexArr = tjsonGetObjectItem(pJson, "configIndexArr");
if (configIndexArr == NULL) return -1;
pCfg->configIndexCount = tjsonGetArraySize(configIndexArr);
for (int32_t i = 0; i < pCfg->configIndexCount; ++i) {
SJson *configIndex = tjsonGetArrayItem(configIndexArr, i);
if (configIndex == NULL) return -1;
tjsonGetNumberValue(configIndex, "index", pCfg->configIndexArr[i], code);
if (code < 0) return -1;
}
return 0;
}
int32_t syncReadCfgFile(SSyncNode *pNode) {
int32_t code = -1;
TdFilePtr pFile = NULL;
char *pData = NULL;
SJson *pJson = NULL;
const char *file = pNode->configPath;
SRaftCfg *pCfg = &pNode->raftCfg;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to open sync cfg file:%s since %s", pNode->vgId, file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to fstat sync cfg file:%s since %s", pNode->vgId, file, terrstr());
goto _OVER;
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, terrstr());
goto _OVER;
}
pData[size] = '\0';
pJson = tjsonParse(pData);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (tjsonToObject(pJson, "RaftCfg", syncDecodeRaftCfg, (void *)pCfg) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
code = 0;
sInfo("vgId:%d, succceed to read sync cfg file %s", pNode->vgId, file);
_OVER:
if (pData != NULL) taosMemoryFree(pData);
if (pJson != NULL) cJSON_Delete(pJson);
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, terrstr());
}
return code;
}
int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex) {
SRaftCfg *pCfg = &pNode->raftCfg;
if (pCfg->configIndexCount <= MAX_CONFIG_INDEX_COUNT) {
return -1;
}
pCfg->configIndexArr[pCfg->configIndexCount] = cfgIndex;
pCfg->configIndexCount++;
return 0;
}

7
source/libs/sync/src/syncRaftStore.c
View File

@ -112,13 +112,6 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) {
cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId);
uint64_t u64 = pRaftStore->voteFor.addr;
char host[128] = {0};
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pRoot, "addr_host", host);
cJSON_AddNumberToObject(pRoot, "addr_port", port);
char *serialized = cJSON_Print(pRoot);
int len2 = strlen(serialized);
ASSERT(len2 < len);

17
source/libs/sync/src/syncReplication.c
View File

@ -107,10 +107,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
pMsg = rpcMsg.pCont;
} else {
char host[64];
uint16_t port;
syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port);
sNError(pSyncNode, "replicate to %s:%d error, next-index:%" PRId64, host, port, nextIndex);
sNError(pSyncNode, "replicate to dnode:%d error, next-index:%" PRId64, DID(pDestId), nextIndex);
return -1;
}
}
@ -171,10 +168,7 @@ int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) {
SRaftId* pDestId = &(pSyncNode->peersId[i]);
ret = syncNodeReplicateOne(pSyncNode, pDestId, true);
if (ret != 0) {
char host[64];
int16_t port;
syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port);
sError("vgId:%d, do append entries error for %s:%d", pSyncNode->vgId, host, port);
sError("vgId:%d, do append entries error for dnode:%d", pSyncNode->vgId, DID(pDestId));
}
}
@ -183,7 +177,6 @@ int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) {
int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
SyncAppendEntries* pMsg = pRpcMsg->pCont;
int32_t ret = 0;
pMsg->destId = *destRaftId;
syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg);
return 0;
@ -229,11 +222,7 @@ int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* dest
if (syncNodeNeedSendAppendEntries(pSyncNode, destRaftId, pMsg)) {
ret = syncNodeSendAppendEntries(pSyncNode, destRaftId, pRpcMsg);
} else {
char logBuf[128];
char host[64];
int16_t port;
syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port);
sNTrace(pSyncNode, "do not repcate to %s:%d for index:%" PRId64, host, port, pMsg->prevLogIndex + 1);
sNTrace(pSyncNode, "do not repcate to dnode:%d for index:%" PRId64, DID(destRaftId), pMsg->prevLogIndex + 1);
rpcFreeCont(pRpcMsg->pCont);
}

18
source/libs/sync/src/syncTimeout.c
View File

@ -24,33 +24,35 @@
#include "syncUtil.h"
static void syncNodeCleanConfigIndex(SSyncNode* ths) {
#if 0
int32_t newArrIndex = 0;
SyncIndex newConfigIndexArr[MAX_CONFIG_INDEX_COUNT] = {0};
SSnapshot snapshot = {0};
ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot);
if (snapshot.lastApplyIndex != SYNC_INDEX_INVALID) {
for (int32_t i = 0; i < ths->pRaftCfg->configIndexCount; ++i) {
if (ths->pRaftCfg->configIndexArr[i] < snapshot.lastConfigIndex) {
for (int32_t i = 0; i < ths->raftCfg.configIndexCount; ++i) {
if (ths->raftCfg.configIndexArr[i] < snapshot.lastConfigIndex) {
// pass
} else {
// save
newConfigIndexArr[newArrIndex] = ths->pRaftCfg->configIndexArr[i];
newConfigIndexArr[newArrIndex] = ths->raftCfg.configIndexArr[i];
++newArrIndex;
}
}
int32_t oldCnt = ths->pRaftCfg->configIndexCount;
ths->pRaftCfg->configIndexCount = newArrIndex;
memcpy(ths->pRaftCfg->configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr));
int32_t oldCnt = ths->raftCfg.configIndexCount;
ths->raftCfg.configIndexCount = newArrIndex;
memcpy(ths->raftCfg.configIndexArr, newConfigIndexArr, sizeof(newConfigIndexArr));
int32_t code = raftCfgPersist(ths->pRaftCfg);
int32_t code = syncWriteCfgFile(ths);
if (code != 0) {
sNFatal(ths, "failed to persist cfg");
} else {
sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->pRaftCfg->configIndexCount);
sNTrace(ths, "clean config index arr, old-cnt:%d, new-cnt:%d", oldCnt, ths->raftCfg.configIndexCount);
}
}
#endif
}
static int32_t syncNodeTimerRoutine(SSyncNode* ths) {

364
source/libs/sync/src/syncUtil.c
View File

@ -21,42 +21,23 @@
#include "syncRaftStore.h"
#include "syncSnapshot.h"
extern void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) {
int32_t len = snprintf(buf, bufLen, "{r-num:%d, my:%d, ", pCfg->replicaNum, pCfg->myIndex);
uint64_t syncUtilAddr2U64(const char* host, uint16_t port) {
uint32_t hostU32 = taosGetIpv4FromFqdn(host);
if (hostU32 == (uint32_t)-1) {
sError("failed to resolve ipv4 addr, host:%s", host);
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
if (i < pCfg->replicaNum - 1) {
len += snprintf(buf + len, bufLen - len, "%s:%d, ", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort);
} else {
len += snprintf(buf + len, bufLen - len, "%s:%d}", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort);
}
}
uint64_t u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16);
return u64;
}
void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) {
uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF);
struct in_addr addr = {.s_addr = hostU32};
taosInetNtoa(addr, host, len);
*port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16);
}
void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet) {
pEpSet->inUse = 0;
pEpSet->numOfEps = 0;
addEpIntoEpSet(pEpSet, pInfo->nodeFqdn, pInfo->nodePort);
}
void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) {
char host[TSDB_FQDN_LEN] = {0};
uint16_t port = 0;
syncUtilU642Addr(raftId->addr, host, sizeof(host), &port);
pEpSet->inUse = 0;
pEpSet->numOfEps = 0;
addEpIntoEpSet(pEpSet, host, port);
pEpSet->numOfEps = 1;
pEpSet->eps[0].port = pInfo->nodePort;
tstrncpy(pEpSet->eps[0].fqdn, pInfo->nodeFqdn, TSDB_FQDN_LEN);
}
bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId) {
@ -69,13 +50,18 @@ bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId*
char ipbuf[128] = {0};
tinet_ntoa(ipbuf, ipv4);
raftId->addr = syncUtilAddr2U64(ipbuf, pInfo->nodePort);
raftId->addr = SYNC_ADDR(pInfo);
raftId->vgId = vgId;
sInfo("vgId:%d, sync addr:%" PRIu64 ", dnode:%d cluster:%" PRId64 " fqdn:%s ip:%s port:%u ipv4:%u", vgId,
raftId->addr, pInfo->nodeId, pInfo->clusterId, pInfo->nodeFqdn, ipbuf, pInfo->nodePort, ipv4);
return true;
}
bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2) {
return pId1->addr == pId2->addr && pId1->vgId == pId2->vgId;
if (pId1->addr == pId2->addr && pId1->vgId == pId2->vgId) return true;
if ((CID(pId1) == 0 || CID(pId2) == 0) && (DID(pId1) == DID(pId2)) && pId1->vgId == pId2->vgId) return true;
return false;
}
bool syncUtilEmptyId(const SRaftId* pId) { return (pId->addr == 0 && pId->vgId == 0); }
@ -91,89 +77,16 @@ int32_t syncUtilElectRandomMS(int32_t min, int32_t max) {
int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; }
cJSON* syncUtilRaftId2Json(const SRaftId* p) {
char u64buf[128] = {0};
cJSON* pRoot = cJSON_CreateObject();
snprintf(u64buf, sizeof(u64buf), "%" PRIu64 "", p->addr);
cJSON_AddStringToObject(pRoot, "addr", u64buf);
char host[128] = {0};
uint16_t port;
syncUtilU642Addr(p->addr, host, sizeof(host), &port);
cJSON_AddStringToObject(pRoot, "host", host);
cJSON_AddNumberToObject(pRoot, "port", port);
cJSON_AddNumberToObject(pRoot, "vgId", p->vgId);
cJSON* pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SRaftId", pRoot);
return pJson;
}
static inline bool syncUtilCanPrint(char c) {
if (c >= 32 && c <= 126) {
return true;
} else {
return false;
}
}
char* syncUtilPrintBin(char* ptr, uint32_t len) {
int64_t memLen = (int64_t)(len + 1);
char* s = taosMemoryMalloc(memLen);
ASSERT(s != NULL);
memset(s, 0, len + 1);
memcpy(s, ptr, len);
for (int32_t i = 0; i < len; ++i) {
if (!syncUtilCanPrint(s[i])) {
s[i] = '.';
}
}
return s;
}
char* syncUtilPrintBin2(char* ptr, uint32_t len) {
uint32_t len2 = len * 4 + 1;
char* s = taosMemoryMalloc(len2);
ASSERT(s != NULL);
memset(s, 0, len2);
char* p = s;
for (int32_t i = 0; i < len; ++i) {
int32_t n = sprintf(p, "%d,", ptr[i]);
p += n;
}
return s;
}
void syncUtilMsgHtoN(void* msg) {
SMsgHead* pHead = msg;
pHead->contLen = htonl(pHead->contLen);
pHead->vgId = htonl(pHead->vgId);
}
void syncUtilMsgNtoH(void* msg) {
SMsgHead* pHead = msg;
pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId);
}
bool syncUtilUserPreCommit(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; }
bool syncUtilUserRollback(tmsg_t msgType) { return msgType != TDMT_SYNC_NOOP && msgType != TDMT_SYNC_LEADER_TRANSFER; }
void syncCfg2SimpleStr(const SSyncCfg* pCfg, char* buf, int32_t bufLen) {
int32_t len = snprintf(buf, bufLen, "{r-num:%d, my:%d, ", pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
if (i < pCfg->replicaNum - 1) {
len += snprintf(buf + len, bufLen - len, "%s:%d, ", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort);
} else {
len += snprintf(buf + len, bufLen - len, "%s:%d}", pCfg->nodeInfo[i].nodeFqdn, pCfg->nodeInfo[i].nodePort);
}
}
}
// for leader
static void syncHearbeatReplyTime2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) {
int32_t len = 5;
@ -222,7 +135,7 @@ static void syncPeerState2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) {
}
void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNode* pNode, const char* format, ...) {
if (pNode == NULL || pNode->pRaftCfg == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return;
if (pNode == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return;
int64_t currentTerm = pNode->pRaftStore->currentTerm;
// save error code, otherwise it will be overwritten
@ -244,11 +157,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
int32_t cacheMiss = pNode->pLogStore->cacheMiss;
char cfgStr[1024];
if (pNode->pRaftCfg != NULL) {
syncCfg2SimpleStr(&(pNode->pRaftCfg->cfg), cfgStr, sizeof(cfgStr));
} else {
return;
}
syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr));
char peerStr[1024] = "{";
syncPeerState2Str(pNode, peerStr, sizeof(peerStr));
@ -259,8 +168,6 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
char hbTimeStr[256] = "hb:{";
syncHearbeatTime2Str(pNode, hbTimeStr, sizeof(hbTimeStr));
int32_t quorum = syncNodeDynamicQuorum(pNode);
char eventLog[512]; // {0};
va_list argpointer;
va_start(argpointer, format);
@ -275,7 +182,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
// restore error code
terrno = errCode;
if (pNode != NULL && pNode->pRaftCfg != NULL) {
if (pNode != NULL) {
taosPrintLog(flags, level, dflag,
"vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", first-ver:%" PRId64
", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64
@ -286,7 +193,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum,
pNode->becomeLeaderNum, pNode->configChangeNum, cacheHit, cacheMiss, pNode->hbSlowNum,
pNode->hbrSlowNum, aqItems, pNode->snapshottingIndex, pNode->replicaNum,
pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum,
pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode),
pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr, hbTimeStr,
hbrTimeStr);
}
@ -295,7 +202,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo
void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotSender* pSender,
const char* format, ...) {
SSyncNode* pNode = pSender->pSyncNode;
if (pNode == NULL || pNode->pRaftCfg == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return;
if (pNode == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return;
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) {
@ -310,17 +217,11 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla
}
char cfgStr[1024];
syncCfg2SimpleStr(&(pNode->pRaftCfg->cfg), cfgStr, sizeof(cfgStr));
syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr));
char peerStr[1024] = "{";
syncPeerState2Str(pNode, peerStr, sizeof(peerStr));
int32_t quorum = syncNodeDynamicQuorum(pNode);
SRaftId destId = pNode->replicasId[pSender->replicaIndex];
char host[64];
uint16_t port;
syncUtilU642Addr(destId.addr, host, sizeof(host), &port);
char eventLog[512]; // {0};
va_list argpointer;
va_start(argpointer, format);
@ -330,24 +231,24 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla
taosPrintLog(flags, level, dflag,
"vgId:%d, %s, sync:%s, {%p s-param:%" PRId64 " e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64
" lcindex:%" PRId64
" seq:%d ack:%d finish:%d replica-index:%d %s:%d}"
" seq:%d ack:%d finish:%d replica-index:%d dnode:%d}"
", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64
", snap-tm:%" PRIu64 ", sby:%d, stgy:%d, bch:%d, r-num:%d, lcfg:%" PRId64
", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s",
pNode->vgId, eventLog, syncStr(pNode->state), pSender, pSender->snapshotParam.start,
pSender->snapshotParam.end, pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm,
pSender->snapshot.lastConfigIndex, pSender->seq, pSender->ack, pSender->finish, pSender->replicaIndex,
host, port, pNode->pRaftStore->currentTerm, pNode->commitIndex, logBeginIndex, logLastIndex,
pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->pRaftCfg->isStandBy,
pNode->pRaftCfg->snapshotStrategy, pNode->pRaftCfg->batchSize, pNode->replicaNum,
pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum,
DID(&pNode->replicasId[pSender->replicaIndex]), pNode->pRaftStore->currentTerm, pNode->commitIndex,
logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm,
pNode->raftCfg.isStandBy, pNode->raftCfg.snapshotStrategy, pNode->raftCfg.batchSize, pNode->replicaNum,
pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode),
pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr);
}
void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t dflag, SSyncSnapshotReceiver* pReceiver,
const char* format, ...) {
SSyncNode* pNode = pReceiver->pSyncNode;
if (pNode == NULL || pNode->pRaftCfg == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return;
if (pNode == NULL || pNode->pRaftStore == NULL || pNode->pLogStore == NULL) return;
SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0};
if (pNode->pFsm != NULL && pNode->pFsm->FpGetSnapshotInfo != NULL) {
@ -362,17 +263,11 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df
}
char cfgStr[1024];
syncCfg2SimpleStr(&(pNode->pRaftCfg->cfg), cfgStr, sizeof(cfgStr));
syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr));
char peerStr[1024] = "{";
syncPeerState2Str(pNode, peerStr, sizeof(peerStr));
int32_t quorum = syncNodeDynamicQuorum(pNode);
SRaftId fromId = pReceiver->fromId;
char host[128];
uint16_t port;
syncUtilU642Addr(fromId.addr, host, sizeof(host), &port);
char eventLog[512]; // {0};
va_list argpointer;
va_start(argpointer, format);
@ -381,19 +276,19 @@ void syncPrintSnapshotReceiverLog(const char* flags, ELogLevel level, int32_t df
taosPrintLog(flags, level, dflag,
"vgId:%d, %s, sync:%s,"
" {%p start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 " from:%s:%d s-param:%" PRId64
" {%p start:%d ack:%d term:%" PRIu64 " start-time:%" PRId64 " from dnode:%d s-param:%" PRId64
" e-param:%" PRId64 " laindex:%" PRId64 " laterm:%" PRIu64 " lcindex:%" PRId64
"}"
", tm:%" PRIu64 ", cmt:%" PRId64 ", fst:%" PRId64 ", lst:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64
", snap-tm:%" PRIu64 ", sby:%d, stgy:%d, bch:%d, r-num:%d, lcfg:%" PRId64
", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s, %s",
pNode->vgId, eventLog, syncStr(pNode->state), pReceiver, pReceiver->start, pReceiver->ack,
pReceiver->term, pReceiver->startTime, host, port, pReceiver->snapshotParam.start,
pReceiver->term, pReceiver->startTime, DID(&pReceiver->fromId), pReceiver->snapshotParam.start,
pReceiver->snapshotParam.end, pReceiver->snapshot.lastApplyIndex, pReceiver->snapshot.lastApplyTerm,
pReceiver->snapshot.lastConfigIndex, pNode->pRaftStore->currentTerm, pNode->commitIndex, logBeginIndex,
logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm,
pNode->pRaftCfg->isStandBy, pNode->pRaftCfg->snapshotStrategy, pNode->pRaftCfg->batchSize,
pNode->replicaNum, pNode->pRaftCfg->lastConfigIndex, pNode->changing, pNode->restoreFinish, quorum,
pNode->raftCfg.isStandBy, pNode->raftCfg.snapshotStrategy, pNode->raftCfg.batchSize, pNode->replicaNum,
pNode->raftCfg.lastConfigIndex, pNode->changing, pNode->restoreFinish, syncNodeDynamicQuorum(pNode),
pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, peerStr, cfgStr);
}
@ -408,56 +303,37 @@ void syncLogRecvTimer(SSyncNode* pSyncNode, const SyncTimeout* pMsg, const char*
}
void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_TRACE)) return;
sNTrace(pSyncNode, "recv sync-local-cmd {cmd:%d-%s, sd-new-term:%" PRId64 ", fc-index:%" PRId64 "}, %s", pMsg->cmd,
syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, pMsg->fcIndex, s);
}
void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"send sync-append-entries-reply to %s:%d, {term:%" PRId64 ", pterm:%" PRId64
"send sync-append-entries-reply to dnode:%d, {term:%" PRId64 ", pterm:%" PRId64
", success:%d, lsend-index:%" PRId64 ", match:%" PRId64 "}, %s",
host, port, pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s);
DID(&pMsg->destId), pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s);
}
void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"recv sync-append-entries-reply from %s:%d {term:%" PRId64 ", pterm:%" PRId64
"recv sync-append-entries-reply from dnode:%d {term:%" PRId64 ", pterm:%" PRId64
", success:%d, lsend-index:%" PRId64 ", match:%" PRId64 "}, %s",
host, port, pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s);
DID(&pMsg->srcId), pMsg->term, pMsg->lastMatchTerm, pMsg->success, pMsg->lastSendIndex, pMsg->matchIndex, s);
}
void syncLogSendHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, bool printX, int64_t timerElapsed,
int64_t execTime) {
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
if (printX) {
sNTrace(pSyncNode,
"send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
"send sync-heartbeat to dnode:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
"}, x",
host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp);
DID(&pMsg->destId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp);
} else {
sNTrace(pSyncNode,
"send sync-heartbeat to %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
"send sync-heartbeat to dnode:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
"}, timer-elapsed:%" PRId64 ", next-exec:%" PRId64,
host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, timerElapsed, execTime);
DID(&pMsg->destId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, timerElapsed,
execTime);
}
}
@ -465,183 +341,109 @@ void syncLogRecvHeartbeat(SSyncNode* pSyncNode, const SyncHeartbeat* pMsg, int64
if (timeDiff > SYNC_HEARTBEAT_SLOW_MS) {
pSyncNode->hbSlowNum++;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNInfo(pSyncNode,
"recv sync-heartbeat from %s:%d slow {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
"}, %s, net elapsed:%" PRId64,
host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff);
"recv sync-heartbeat from dnode:%d slow {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64
", ts:%" PRId64 "}, %s, net elapsed:%" PRId64,
DID(&pMsg->srcId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff);
}
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"recv sync-heartbeat from %s:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
"recv sync-heartbeat from dnode:%d {term:%" PRId64 ", cmt:%" PRId64 ", min-match:%" PRId64 ", ts:%" PRId64
"}, %s, net elapsed:%" PRId64,
host, port, pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff);
DID(&pMsg->srcId), pMsg->term, pMsg->commitIndex, pMsg->minMatchIndex, pMsg->timeStamp, s, timeDiff);
}
void syncLogSendHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode, "send sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s", host, port,
pMsg->term, pMsg->timeStamp, s);
sNTrace(pSyncNode, "send sync-heartbeat-reply from dnode:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s",
DID(&pMsg->destId), pMsg->term, pMsg->timeStamp, s);
}
void syncLogRecvHeartbeatReply(SSyncNode* pSyncNode, const SyncHeartbeatReply* pMsg, int64_t timeDiff, const char* s) {
if (timeDiff > SYNC_HEARTBEAT_REPLY_SLOW_MS) {
pSyncNode->hbrSlowNum++;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"recv sync-heartbeat-reply from %s:%d slow {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64,
host, port, pMsg->term, pMsg->timeStamp, s, timeDiff);
"recv sync-heartbeat-reply from dnode:%d slow {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64,
DID(&pMsg->srcId), pMsg->term, pMsg->timeStamp, s, timeDiff);
}
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"recv sync-heartbeat-reply from %s:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64, host,
port, pMsg->term, pMsg->timeStamp, s, timeDiff);
"recv sync-heartbeat-reply from dnode:%d {term:%" PRId64 ", ts:%" PRId64 "}, %s, net elapsed:%" PRId64,
DID(&pMsg->srcId), pMsg->term, pMsg->timeStamp, s, timeDiff);
}
void syncLogSendSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_DEBUG)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
sNDebug(pSyncNode,
"send sync-snapshot-send to %s:%u, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64
"send sync-snapshot-send to dnode:%d, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", end:%" PRId64
", lterm:%" PRId64 ", stime:%" PRId64,
host, port, s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime);
DID(&pMsg->destId), s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm,
pMsg->startTime);
}
void syncLogRecvSyncSnapshotSend(SSyncNode* pSyncNode, const SyncSnapshotSend* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_DEBUG)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNDebug(pSyncNode,
"recv sync-snapshot-send from %s:%u, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64
"recv sync-snapshot-send from dnode:%d, %s, seq:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64
", lterm:%" PRId64 ", stime:%" PRId64 ", len:%u",
host, port, s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime,
pMsg->dataLen);
DID(&pMsg->srcId), s, pMsg->seq, pMsg->term, pMsg->beginIndex, pMsg->lastIndex, pMsg->lastTerm,
pMsg->startTime, pMsg->dataLen);
}
void syncLogSendSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_DEBUG)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
sNDebug(pSyncNode,
"send sync-snapshot-rsp to %s:%u, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64
"send sync-snapshot-rsp to dnode:%d, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64
", lterm:%" PRId64 ", stime:%" PRId64,
host, port, s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime);
DID(&pMsg->destId), s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm,
pMsg->startTime);
}
void syncLogRecvSyncSnapshotRsp(SSyncNode* pSyncNode, const SyncSnapshotRsp* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_DEBUG)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNDebug(pSyncNode,
"recv sync-snapshot-rsp from %s:%u, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64
"recv sync-snapshot-rsp from dnode:%d, %s, ack:%d, term:%" PRId64 ", begin:%" PRId64 ", lst:%" PRId64
", lterm:%" PRId64 ", stime:%" PRId64,
host, port, s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm, pMsg->startTime);
DID(&pMsg->srcId), s, pMsg->ack, pMsg->term, pMsg->snapBeginIndex, pMsg->lastIndex, pMsg->lastTerm,
pMsg->startTime);
}
void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"recv sync-append-entries from %s:%d {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64
"recv sync-append-entries from dnode:%d {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64
", cmt:%" PRId64 ", pterm:%" PRId64 ", datalen:%d}, %s",
host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm,
DID(&pMsg->srcId), pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm,
pMsg->dataLen, s);
}
void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) {
if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"send sync-append-entries to %s:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64
"send sync-append-entries to dnode:%d, {term:%" PRId64 ", pre-index:%" PRId64 ", pre-term:%" PRId64
", lsend-index:%" PRId64 ", cmt:%" PRId64 ", datalen:%d}, %s",
host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, (pMsg->prevLogIndex + 1), pMsg->commitIndex,
pMsg->dataLen, s);
DID(&pMsg->destId), pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, (pMsg->prevLogIndex + 1),
pMsg->commitIndex, pMsg->dataLen, s);
}
void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, int32_t voteGranted, const char* s) {
// if (!(sDebugFlag & DEBUG_TRACE)) return;
char logBuf[256];
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
if (voteGranted == -1) {
sNInfo(pSyncNode,
"recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host,
port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s);
"recv sync-request-vote from dnode:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s",
DID(&pMsg->srcId), pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s);
} else {
sNInfo(pSyncNode,
"recv sync-request-vote from %s:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, granted:%d",
host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted);
"recv sync-request-vote from dnode:%d, {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64
"}, granted:%d",
DID(&pMsg->srcId), pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, voteGranted);
}
}
void syncLogSendRequestVote(SSyncNode* pNode, const SyncRequestVote* pMsg, const char* s) {
// if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
sNInfo(pNode, "send sync-request-vote to %s:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s", host,
port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s);
sNInfo(pNode, "send sync-request-vote to dnode:%d {term:%" PRId64 ", lindex:%" PRId64 ", lterm:%" PRId64 "}, %s",
DID(&pMsg->destId), pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s);
}
void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) {
// if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port);
sNInfo(pSyncNode, "recv sync-request-vote-reply from %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term,
pMsg->voteGranted, s);
sNInfo(pSyncNode, "recv sync-request-vote-reply from dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->srcId),
pMsg->term, pMsg->voteGranted, s);
}
void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) {
// if (!(sDebugFlag & DEBUG_TRACE)) return;
char host[64];
uint16_t port;
syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port);
sNInfo(pSyncNode, "send sync-request-vote-reply to %s:%d {term:%" PRId64 ", grant:%d}, %s", host, port, pMsg->term,
pMsg->voteGranted, s);
sNInfo(pSyncNode, "send sync-request-vote-reply to dnode:%d {term:%" PRId64 ", grant:%d}, %s", DID(&pMsg->destId),
pMsg->term, pMsg->voteGranted, s);
}

24
source/libs/sync/test/syncRaftCfgIndexTest.cpp
View File

@ -51,26 +51,26 @@ SSyncCfg* createSyncCfg() {
const char* pFile = "./raft_config_index.json";
void test1() {
int32_t code = raftCfgIndexCreateFile(pFile);
ASSERT(code == 0);
// int32_t code = raftCfgIndexCreateFile(pFile);
// ASSERT(code == 0);
SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile);
// SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile);
raftCfgIndexClose(pRaftCfgIndex);
// raftCfgIndexClose(pRaftCfgIndex);
}
void test2() {
SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile);
for (int i = 0; i < 500; ++i) {
raftCfgIndexAddConfigIndex(pRaftCfgIndex, i);
}
raftCfgIndexPersist(pRaftCfgIndex);
raftCfgIndexClose(pRaftCfgIndex);
// SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile);
// for (int i = 0; i < 500; ++i) {
// raftCfgIndexAddConfigIndex(pRaftCfgIndex, i);
// }
// raftCfgIndexPersist(pRaftCfgIndex);
// raftCfgIndexClose(pRaftCfgIndex);
}
void test3() {
SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile);
raftCfgIndexClose(pRaftCfgIndex);
// SRaftCfgIndex* pRaftCfgIndex = raftCfgIndexOpen(pFile);
// raftCfgIndexClose(pRaftCfgIndex);
}
int main() {

56
source/libs/sync/test/syncRaftCfgTest.cpp
View File

@ -67,12 +67,12 @@ void test3() {
if (taosCheckExistFile(s)) {
printf("%s file: %s already exist! \n", (char*)__FUNCTION__, s);
} else {
SRaftCfgMeta meta;
meta.isStandBy = 7;
meta.snapshotStrategy = 9;
meta.batchSize = 10;
meta.lastConfigIndex = 789;
raftCfgCreateFile(pCfg, meta, s);
// SRaftCfgMeta meta;
// meta.isStandBy = 7;
// meta.snapshotStrategy = 9;
// meta.batchSize = 10;
// meta.lastConfigIndex = 789;
// raftCfgCreateFile(pCfg, meta, s);
printf("%s create json file: %s \n", (char*)__FUNCTION__, s);
}
@ -80,37 +80,37 @@ void test3() {
}
void test4() {
SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json");
assert(pCfg != NULL);
// SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json");
// assert(pCfg != NULL);
int32_t ret = raftCfgClose(pCfg);
assert(ret == 0);
// int32_t ret = raftCfgClose(pCfg);
// assert(ret == 0);
}
void test5() {
SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json");
assert(pCfg != NULL);
// SRaftCfg* pCfg = raftCfgOpen("./test3_raft_cfg.json");
// assert(pCfg != NULL);
pCfg->cfg.myIndex = taosGetTimestampSec();
pCfg->isStandBy += 2;
pCfg->snapshotStrategy += 3;
pCfg->batchSize += 4;
pCfg->lastConfigIndex += 1000;
// pCfg->cfg.myIndex = taosGetTimestampSec();
// pCfg->isStandBy += 2;
// pCfg->snapshotStrategy += 3;
// pCfg->batchSize += 4;
// pCfg->lastConfigIndex += 1000;
pCfg->configIndexCount = 5;
for (int i = 0; i < MAX_CONFIG_INDEX_COUNT; ++i) {
(pCfg->configIndexArr)[i] = -1;
}
for (int i = 0; i < pCfg->configIndexCount; ++i) {
(pCfg->configIndexArr)[i] = i * 100;
}
// pCfg->configIndexCount = 5;
// for (int i = 0; i < MAX_CONFIG_INDEX_COUNT; ++i) {
// (pCfg->configIndexArr)[i] = -1;
// }
// for (int i = 0; i < pCfg->configIndexCount; ++i) {
// (pCfg->configIndexArr)[i] = i * 100;
// }
raftCfgPersist(pCfg);
// // raftCfgPersist(pCfg);
printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex);
// printf("%s update json file: %s myIndex->%d \n", (char*)__FUNCTION__, "./test3_raft_cfg.json", pCfg->cfg.myIndex);
int32_t ret = raftCfgClose(pCfg);
assert(ret == 0);
// int32_t ret = raftCfgClose(pCfg);
// assert(ret == 0);
}
int main() {

5
source/libs/sync/test/sync_test_lib/inc/syncTest.h
View File

@ -474,6 +474,11 @@ void syncLocalCmdPrint2(char* s, const SyncLocalCmd* pMsg);
void syncLocalCmdLog(const SyncLocalCmd* pMsg);
void syncLocalCmdLog2(char* s, const SyncLocalCmd* pMsg);
char* syncUtilPrintBin(char* ptr, uint32_t len);
char* syncUtilPrintBin2(char* ptr, uint32_t len);
void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port);
uint64_t syncUtilAddr2U64(const char* host, uint16_t port);
#ifdef __cplusplus
}
#endif

2
source/libs/sync/test/sync_test_lib/src/syncBatch.c
View File

@ -311,7 +311,7 @@ cJSON* syncAppendEntriesBatch2Json(const SyncAppendEntriesBatch* pMsg) {
cJSON* pTmp = pSrcId;
char host[128] = {0};
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
// syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}

78
source/libs/sync/test/sync_test_lib/src/syncIO.c
View File

@ -73,7 +73,7 @@ int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) {
int32_t ret = 0;
{
syncUtilMsgNtoH(pMsg->pCont);
// syncUtilMsgNtoH(pMsg->pCont);
char logBuf[256] = {0};
snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d msgType:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port,
@ -376,7 +376,7 @@ static void *syncIOConsumerFunc(void *param) {
}
static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
syncUtilMsgNtoH(pMsg->pCont);
// syncUtilMsgNtoH(pMsg->pCont);
syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg);
SSyncIO *io = pParent;
@ -432,9 +432,9 @@ static void syncIOTickQ(void *param, void *tmrId) {
SSyncIO *io = (SSyncIO *)param;
SRaftId srcId, destId;
srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
// srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
srcId.vgId = -1;
destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
// destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
destId.vgId = -1;
SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, -1, "syncIOTickQ");
@ -454,9 +454,9 @@ static void syncIOTickPing(void *param, void *tmrId) {
SSyncIO *io = (SSyncIO *)param;
SRaftId srcId, destId;
srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
// srcId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
srcId.vgId = -1;
destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
// destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
destId.vgId = -1;
SyncPing *pMsg = syncPingBuild2(&srcId, &destId, -1, "syncIOTickPing");
// SyncPing *pMsg = syncPingBuild3(&srcId, &destId);
@ -470,4 +470,68 @@ static void syncIOTickPing(void *param, void *tmrId) {
taosTmrReset(syncIOTickPing, io->pingTimerMS, io, io->timerMgr, &io->pingTimer);
}
void syncEntryDestory(SSyncRaftEntry* pEntry) {}
void syncEntryDestory(SSyncRaftEntry* pEntry) {}
void syncUtilMsgNtoH(void* msg) {
SMsgHead* pHead = msg;
pHead->contLen = ntohl(pHead->contLen);
pHead->vgId = ntohl(pHead->vgId);
}
static inline bool syncUtilCanPrint(char c) {
if (c >= 32 && c <= 126) {
return true;
} else {
return false;
}
}
char* syncUtilPrintBin(char* ptr, uint32_t len) {
int64_t memLen = (int64_t)(len + 1);
char* s = taosMemoryMalloc(memLen);
ASSERT(s != NULL);
memset(s, 0, len + 1);
memcpy(s, ptr, len);
for (int32_t i = 0; i < len; ++i) {
if (!syncUtilCanPrint(s[i])) {
s[i] = '.';
}
}
return s;
}
char* syncUtilPrintBin2(char* ptr, uint32_t len) {
uint32_t len2 = len * 4 + 1;
char* s = taosMemoryMalloc(len2);
ASSERT(s != NULL);
memset(s, 0, len2);
char* p = s;
for (int32_t i = 0; i < len; ++i) {
int32_t n = sprintf(p, "%d,", ptr[i]);
p += n;
}
return s;
}
void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port) {
uint32_t hostU32 = (uint32_t)((u64 >> 32) & 0x00000000FFFFFFFF);
struct in_addr addr = {.s_addr = hostU32};
taosInetNtoa(addr, host, len);
*port = (uint16_t)((u64 & 0x00000000FFFF0000) >> 16);
}
uint64_t syncUtilAddr2U64(const char* host, uint16_t port) {
uint32_t hostU32 = taosGetIpv4FromFqdn(host);
if (hostU32 == (uint32_t)-1) {
sError("failed to resolve ipv4 addr, host:%s", host);
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
uint64_t u64 = (((uint64_t)hostU32) << 32) | (((uint32_t)port) << 16);
return u64;
}

4
source/libs/sync/test/sync_test_lib/src/syncIndexMgrDebug.c
View File

@ -53,7 +53,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
cJSON *pReplicas = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i]));
// cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pSyncIndexMgr->replicas))[i]));
}
{
@ -76,7 +76,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) {
cJSON_AddItemToObject(pRoot, "privateTerm", pIndex);
}
snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode);
// snprintf(u64buf, sizeof(u64buf), "%p", pSyncIndexMgr->pSyncNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
}

20
source/libs/sync/test/sync_test_lib/src/syncMainDebug.c
View File

@ -23,7 +23,7 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
if (pSyncNode != NULL) {
// init by SSyncInfo
cJSON_AddNumberToObject(pRoot, "vgId", pSyncNode->vgId);
cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg));
// cJSON_AddItemToObject(pRoot, "SRaftCfg", raftCfg2Json(pSyncNode->pRaftCfg));
cJSON_AddStringToObject(pRoot, "path", pSyncNode->path);
cJSON_AddStringToObject(pRoot, "raftStorePath", pSyncNode->raftStorePath);
cJSON_AddStringToObject(pRoot, "configPath", pSyncNode->configPath);
@ -44,8 +44,8 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
// init internal
cJSON* pMe = syncUtilNodeInfo2Json(&pSyncNode->myNodeInfo);
cJSON_AddItemToObject(pRoot, "myNodeInfo", pMe);
cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId);
cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId);
// cJSON* pRaftId = syncUtilRaftId2Json(&pSyncNode->myRaftId);
// cJSON_AddItemToObject(pRoot, "myRaftId", pRaftId);
cJSON_AddNumberToObject(pRoot, "peersNum", pSyncNode->peersNum);
cJSON* pPeers = cJSON_CreateArray();
@ -56,22 +56,22 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) {
cJSON* pPeersId = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "peersId", pPeersId);
for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {
cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i]));
// cJSON_AddItemToArray(pPeersId, syncUtilRaftId2Json(&pSyncNode->peersId[i]));
}
cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncNode->replicaNum);
cJSON* pReplicasId = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicasId", pReplicasId);
for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i]));
// cJSON_AddItemToArray(pReplicasId, syncUtilRaftId2Json(&pSyncNode->replicasId[i]));
}
// raft algorithm
snprintf(u64buf, sizeof(u64buf), "%p", pSyncNode->pFsm);
cJSON_AddStringToObject(pRoot, "pFsm", u64buf);
cJSON_AddNumberToObject(pRoot, "quorum", pSyncNode->quorum);
cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache);
cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache);
// cJSON* pLaderCache = syncUtilRaftId2Json(&pSyncNode->leaderCache);
// cJSON_AddItemToObject(pRoot, "leaderCache", pLaderCache);
// life cycle
snprintf(u64buf, sizeof(u64buf), "%" PRId64, pSyncNode->rid);
@ -200,8 +200,8 @@ inline char* syncNode2SimpleStr(const SSyncNode* pSyncNode) {
"r-num:%d, "
"lcfg:%" PRId64 ", chging:%d, rsto:%d",
pSyncNode->vgId, syncStr(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex,
logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum,
pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish);
logBeginIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->raftCfg.isStandBy, pSyncNode->replicaNum,
pSyncNode->raftCfg.lastConfigIndex, pSyncNode->changing, pSyncNode->restoreFinish);
return s;
}
@ -243,7 +243,7 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) {
int32_t syncNodePingAll(SSyncNode* pSyncNode) {
int32_t ret = 0;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
SRaftId* destId = &(pSyncNode->replicasId[i]);
SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId);
ret = syncNodePing(pSyncNode, destId, pMsg);

4
source/libs/sync/test/sync_test_lib/src/syncMessageDebug.c
View File

@ -181,7 +181,7 @@ cJSON* syncPing2Json(const SyncPing* pMsg) {
cJSON* pTmp = pSrcId;
char host[128] = {0};
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
// syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pTmp, "addr_host", host);
cJSON_AddNumberToObject(pTmp, "addr_port", port);
}
@ -748,7 +748,7 @@ cJSON* syncSnapshotSend2Json(const SyncSnapshotSend* pMsg) {
snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->lastConfigIndex);
cJSON_AddStringToObject(pRoot, "lastConfigIndex", u64buf);
cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig)));
// cJSON_AddItemToObject(pRoot, "lastConfig", syncCfg2Json((SSyncCfg*)&(pMsg->lastConfig)));
snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->lastTerm);
cJSON_AddStringToObject(pRoot, "lastTerm", u64buf);

13
source/libs/sync/test/sync_test_lib/src/syncRaftCfgDebug.c
View File

@ -17,18 +17,19 @@
#include "syncTest.h"
char *syncCfg2Str(SSyncCfg *pSyncCfg) {
cJSON *pJson = syncCfg2Json(pSyncCfg);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
// cJSON *pJson = syncCfg2Json(pSyncCfg);
// char *serialized = cJSON_Print(pJson);
// cJSON_Delete(pJson);
// return serialized;
return "";
}
int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg) {
cJSON *pRoot = cJSON_Parse(s);
ASSERT(pRoot != NULL);
int32_t ret = syncCfgFromJson(pRoot, pSyncCfg);
ASSERT(ret == 0);
// int32_t ret = syncCfgFromJson(pRoot, pSyncCfg);
// ASSERT(ret == 0);
cJSON_Delete(pRoot);
return 0;

9
source/libs/sync/test/sync_test_lib/src/syncUtilDebug.c
View File

@ -29,8 +29,9 @@ cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p) {
}
char* syncUtilRaftId2Str(const SRaftId* p) {
cJSON* pJson = syncUtilRaftId2Json(p);
char* serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
// cJSON* pJson = syncUtilRaftId2Json(p);
// char* serialized = cJSON_Print(pJson);
// cJSON_Delete(pJson);
// return serialized;
return "";
}

8
source/libs/sync/test/sync_test_lib/src/syncVoteMgrDebug.c
View File

@ -25,7 +25,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
cJSON *pReplicas = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i]));
// cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesGranted->replicas))[i]));
}
int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesGranted->replicaNum);
for (int32_t i = 0; i < pVotesGranted->replicaNum; ++i) {
@ -40,7 +40,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) {
cJSON_AddStringToObject(pRoot, "term", u64buf);
cJSON_AddNumberToObject(pRoot, "quorum", pVotesGranted->quorum);
cJSON_AddNumberToObject(pRoot, "toLeader", pVotesGranted->toLeader);
snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pSyncNode);
snprintf(u64buf, sizeof(u64buf), "%p", pVotesGranted->pNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
bool majority = voteGrantedMajority(pVotesGranted);
@ -68,7 +68,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) {
cJSON *pReplicas = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "replicas", pReplicas);
for (int32_t i = 0; i < pVotesRespond->replicaNum; ++i) {
cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i]));
// cJSON_AddItemToArray(pReplicas, syncUtilRaftId2Json(&(*(pVotesRespond->replicas))[i]));
}
int32_t respondNum = 0;
int32_t *arr = (int32_t *)taosMemoryMalloc(sizeof(int32_t) * pVotesRespond->replicaNum);
@ -85,7 +85,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) {
snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pVotesRespond->term);
cJSON_AddStringToObject(pRoot, "term", u64buf);
snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pSyncNode);
snprintf(u64buf, sizeof(u64buf), "%p", pVotesRespond->pNode);
cJSON_AddStringToObject(pRoot, "pSyncNode", u64buf);
}

5
source/libs/transport/src/tmsgcb.c
View File

@ -24,7 +24,6 @@ static SMsgCb defaultMsgCb;
void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; }
int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) {
ASSERT(msgcb != NULL);
int32_t code = (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg);
if (code != 0) {
rpcFreeCont(pMsg->pCont);
@ -59,3 +58,7 @@ void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg) { (*defaultMsgCb.registerBrokenLin
void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type) { (*defaultMsgCb.releaseHandleFp)(pHandle, type); }
void tmsgReportStartup(const char* name, const char* desc) { (*defaultMsgCb.reportStartupFp)(name, desc); }
int32_t tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port) {
return (*defaultMsgCb.updateDnodeInfoFp)(defaultMsgCb.data, dnodeId, clusterId, fqdn, port);
}

15
source/libs/transport/src/trans.c
View File

@ -160,21 +160,12 @@ int rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) {
int rpcSendResponse(const SRpcMsg* pMsg) { return transSendResponse(pMsg); }
void rpcRefHandle(void* handle, int8_t type) {
assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
(*taosRefHandle[type])(handle);
}
void rpcRefHandle(void* handle, int8_t type) { (*taosRefHandle[type])(handle); }
void rpcUnrefHandle(void* handle, int8_t type) {
assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
(*taosUnRefHandle[type])(handle);
}
void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle); }
int rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); }
int rpcReleaseHandle(void* handle, int8_t type) {
assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT);
return (*transReleaseHandle[type])(handle);
}
int rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); }
int rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) {
// later

13
source/libs/transport/src/transCli.c
View File

@ -651,7 +651,6 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
return;
}
assert(nread <= 0);
if (nread == 0) {
// ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb
// nread might be 0, which does not indicate an error or EOF. This is equivalent to EAGAIN or EWOULDBLOCK under
@ -801,7 +800,11 @@ static void cliSendCb(uv_write_t* req, int status) {
}
void cliSend(SCliConn* pConn) {
assert(!transQueueEmpty(&pConn->cliMsgs));
bool empty = transQueueEmpty(&pConn->cliMsgs);
ASSERTS(empty == false, "trans-cli get invalid msg");
if (empty == true) {
return;
}
SCliMsg* pCliMsg = NULL;
CONN_GET_NEXT_SENDMSG(pConn);
@ -933,7 +936,6 @@ void cliConnCb(uv_connect_t* req, int status) {
transSockInfo2Str(&sockname, pConn->src);
tTrace("%s conn %p connect to server successfully", CONN_GET_INST_LABEL(pConn), pConn);
assert(pConn->stream == req->handle);
cliSend(pConn);
}
@ -1237,7 +1239,7 @@ bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) {
for (int i = 0; ahandle == 0 && i < transQueueSize(&conn->cliMsgs); i++) {
SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, i);
if (cliMsg->type == Release) {
assert(pMsg == NULL);
ASSERTS(pMsg == NULL, "trans-cli recv invaid release-req");
return true;
}
}
@ -1665,7 +1667,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
if (pCtx->retryCode != TSDB_CODE_SUCCESS) {
int32_t code = pResp->code;
// return internal code app
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) {
if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK ||
code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) {
pResp->code = pCtx->retryCode;
}
}

7
source/libs/transport/src/transComm.c
View File

@ -134,7 +134,9 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) {
if (total >= HEADSIZE && !p->invalid) {
*buf = taosMemoryCalloc(1, total);
memcpy(*buf, p->buf, total);
transResetBuffer(connBuf);
if (transResetBuffer(connBuf) < 0) {
return -1;
}
} else {
total = -1;
}
@ -154,7 +156,8 @@ int transResetBuffer(SConnBuffer* connBuf) {
p->total = 0;
p->len = 0;
} else {
assert(0);
ASSERTS(0, "invalid read from sock buf");
return -1;
}
return 0;
}

26
source/libs/transport/src/transSvr.c
View File

@ -267,7 +267,10 @@ static bool uvHandleReq(SSvrConn* pConn) {
tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn,
pConn->refId);
assert(transMsg.info.handle != NULL);
ASSERTS(transMsg.info.handle != NULL, "trans-svr failed to alloc handle to msg");
if (transMsg.info.handle == NULL) {
return false;
}
if (pHead->noResp == 1) {
transMsg.info.refId = -1;
@ -718,8 +721,8 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
return;
}
// free memory allocated by
assert(nread == strlen(notify));
assert(buf->base[0] == notify[0]);
ASSERTS(nread == strlen(notify), "trans-svr mem corrupted");
ASSERTS(buf->base[0] == notify[0], "trans-svr mem corrupted");
taosMemoryFree(buf->base);
SWorkThrd* pThrd = q->data;
@ -731,7 +734,6 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
}
uv_handle_type pending = uv_pipe_pending_type(pipe);
assert(pending == UV_TCP);
SSvrConn* pConn = createConn(pThrd);
@ -971,19 +973,24 @@ static void uvPipeListenCb(uv_stream_t* handle, int status) {
uv_pipe_t* pipe = &(srv->pipe[srv->numOfWorkerReady][0]);
int ret = uv_pipe_init(srv->loop, pipe, 1);
assert(ret == 0);
ASSERTS(ret == 0, "trans-svr failed to init pipe");
if (ret != 0) return;
ret = uv_accept((uv_stream_t*)&srv->pipeListen, (uv_stream_t*)pipe);
assert(ret == 0);
ASSERTS(ret == 0, "trans-svr failed to accept pipe msg");
if (ret != 0) return;
ret = uv_is_readable((uv_stream_t*)pipe);
assert(ret == 1);
ASSERTS(ret == 1, "trans-svr pipe status corrupted");
if (ret != 1) return;
ret = uv_is_writable((uv_stream_t*)pipe);
assert(ret == 1);
ASSERTS(ret == 1, "trans-svr pipe status corrupted");
if (ret != 1) return;
ret = uv_is_closing((uv_handle_t*)pipe);
assert(ret == 0);
ASSERTS(ret == 0, "trans-svr pipe status corrupted");
if (ret != 0) return;
srv->numOfWorkerReady++;
}
@ -1272,7 +1279,6 @@ int transSendResponse(const STransMsg* msg) {
SExHandle* exh = msg->info.handle;
int64_t refId = msg->info.refId;
ASYNC_CHECK_HANDLE(exh, refId);
assert(refId != 0);
STransMsg tmsg = *msg;
tmsg.info.refId = refId;

68
source/os/src/osFile.c
View File

@ -39,14 +39,6 @@
#define _SEND_FILE_STEP_ 1000
#endif
#if defined(WINDOWS)
typedef int32_t FileFd;
typedef int32_t SocketFd;
#else
typedef int32_t FileFd;
typedef int32_t SocketFd;
#endif
typedef int32_t FileFd;
typedef struct TdFile {
@ -54,19 +46,10 @@ typedef struct TdFile {
int refId;
FileFd fd;
FILE *fp;
} * TdFilePtr, TdFile;
} TdFile;
#define FILE_WITH_LOCK 1
typedef struct AutoDelFile *AutoDelFilePtr;
typedef struct AutoDelFile {
char *name;
AutoDelFilePtr lastAutoDelFilePtr;
} AutoDelFile;
static TdThreadMutex autoDelFileLock;
static AutoDelFilePtr nowAutoDelFilePtr = NULL;
static TdThreadOnce autoDelFileInit = PTHREAD_ONCE_INIT;
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath) {
#ifdef WINDOWS
const char *tdengineTmpFileNamePrefix = "tdengine-";
@ -268,34 +251,6 @@ int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
return 0;
}
void autoDelFileList() {
taosThreadMutexLock(&autoDelFileLock);
while (nowAutoDelFilePtr != NULL) {
taosRemoveFile(nowAutoDelFilePtr->name);
AutoDelFilePtr tmp = nowAutoDelFilePtr->lastAutoDelFilePtr;
taosMemoryFree(nowAutoDelFilePtr->name);
taosMemoryFree(nowAutoDelFilePtr);
nowAutoDelFilePtr = tmp;
}
taosThreadMutexUnlock(&autoDelFileLock);
taosThreadMutexDestroy(&autoDelFileLock);
}
void autoDelFileListInit() {
taosThreadMutexInit(&autoDelFileLock, NULL);
atexit(autoDelFileList);
}
void autoDelFileListAdd(const char *path) {
taosThreadOnce(&autoDelFileInit, autoDelFileListInit);
taosThreadMutexLock(&autoDelFileLock);
AutoDelFilePtr tmp = taosMemoryMalloc(sizeof(AutoDelFile));
tmp->lastAutoDelFilePtr = nowAutoDelFilePtr;
tmp->name = taosMemoryStrDup(path);
nowAutoDelFilePtr = tmp;
taosThreadMutexUnlock(&autoDelFileLock);
}
TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
int fd = -1;
FILE *fp = NULL;
@ -313,7 +268,6 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
assert(!(tdFileOptions & TD_FILE_EXCL));
fp = fopen(path, mode);
if (fp == NULL) {
// terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
} else {
@ -331,32 +285,44 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
access |= (tdFileOptions & TD_FILE_TEXT) ? O_TEXT : 0;
access |= (tdFileOptions & TD_FILE_EXCL) ? O_EXCL : 0;
#ifdef WINDOWS
fd = _open(path, access, _S_IREAD | _S_IWRITE);
int32_t pmode = _S_IREAD | _S_IWRITE;
if (tdFileOptions & TD_FILE_AUTO_DEL) {
pmode |= _O_TEMPORARY;
}
fd = _open(path, access, pmode);
#else
fd = open(path, access, S_IRWXU | S_IRWXG | S_IRWXO);
#endif
if (fd == -1) {
// terrno = TAOS_SYSTEM_ERROR(errno);
return NULL;
}
}
TdFilePtr pFile = (TdFilePtr)taosMemoryMalloc(sizeof(TdFile));
if (pFile == NULL) {
// terrno = TSDB_CODE_OUT_OF_MEMORY;
if (fd >= 0) close(fd);
if (fp != NULL) fclose(fp);
return NULL;
}
#if FILE_WITH_LOCK
taosThreadRwlockInit(&(pFile->rwlock), NULL);
#endif
pFile->fd = fd;
pFile->fp = fp;
pFile->refId = 0;
if (tdFileOptions & TD_FILE_AUTO_DEL) {
autoDelFileListAdd(path);
#ifdef WINDOWS
// do nothing, since the property of pmode is set with _O_TEMPORARY; the OS will recycle
// the file handle, as well as the space on disk.
#else
// Remove it instantly, so when the program exits normally/abnormally, the file
// will be automatically remove by OS.
unlink(path);
#endif
}
return pFile;
}

2
source/util/src/talgo.c
View File

@ -39,7 +39,7 @@ static void median(void *src, int64_t size, int64_t s, int64_t e, const void *pa
doswap(elePtrAt(src, size, s), elePtrAt(src, size, e), size, buf);
}
assert(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 &&
ASSERT(comparFn(elePtrAt(src, size, mid), elePtrAt(src, size, s), param) <= 0 &&
comparFn(elePtrAt(src, size, s), elePtrAt(src, size, e), param) <= 0);
#ifdef _DEBUG_VIEW

424
source/util/src/tbuffer.c
View File

@ -1,424 +0,0 @@
/*
* Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "tbuffer.h"
#include "texception.h"
typedef union Un4B {
uint32_t ui;
float f;
} Un4B;
#if __STDC_VERSION__ >= 201112LL
static_assert(sizeof(Un4B) == sizeof(uint32_t), "sizeof(Un4B) must equal to sizeof(uint32_t)");
static_assert(sizeof(Un4B) == sizeof(float), "sizeof(Un4B) must equal to sizeof(float)");
#endif
typedef union Un8B {
uint64_t ull;
double d;
} Un8B;
#if __STDC_VERSION__ >= 201112LL
static_assert(sizeof(Un8B) == sizeof(uint64_t), "sizeof(Un8B) must equal to sizeof(uint64_t)");
static_assert(sizeof(Un8B) == sizeof(double), "sizeof(Un8B) must equal to sizeof(double)");
#endif
////////////////////////////////////////////////////////////////////////////////
// reader functions
size_t tbufSkip(SBufferReader* buf, size_t size) {
if ((buf->pos + size) > buf->size) {
THROW(-1);
}
size_t old = buf->pos;
buf->pos += size;
return old;
}
const char* tbufRead(SBufferReader* buf, size_t size) {
const char* ret = buf->data + buf->pos;
tbufSkip(buf, size);
return ret;
}
void tbufReadToBuffer(SBufferReader* buf, void* dst, size_t size) {
assert(dst != NULL);
// always using memcpy, leave optimization to compiler
memcpy(dst, tbufRead(buf, size), size);
}
static size_t tbufReadLength(SBufferReader* buf) {
// maximum length is 65535, if larger length is required
// this function and the corresponding write function need to be
// revised.
uint16_t l = tbufReadUint16(buf);
return l;
}
const char* tbufReadString(SBufferReader* buf, size_t* len) {
size_t l = tbufReadLength(buf);
const char* ret = buf->data + buf->pos;
tbufSkip(buf, l + 1);
if (ret[l] != 0) {
THROW(-1);
}
if (len != NULL) {
*len = l;
}
return ret;
}
size_t tbufReadToString(SBufferReader* buf, char* dst, size_t size) {
assert(dst != NULL);
size_t len;
const char* str = tbufReadString(buf, &len);
if (len >= size) {
len = size - 1;
}
memcpy(dst, str, len);
dst[len] = 0;
return len;
}
const char* tbufReadBinary(SBufferReader* buf, size_t* len) {
size_t l = tbufReadLength(buf);
const char* ret = buf->data + buf->pos;
tbufSkip(buf, l);
if (len != NULL) {
*len = l;
}
return ret;
}
size_t tbufReadToBinary(SBufferReader* buf, void* dst, size_t size) {
assert(dst != NULL);
size_t len;
const char* data = tbufReadBinary(buf, &len);
if (len >= size) {
len = size;
}
memcpy(dst, data, len);
return len;
}
bool tbufReadBool(SBufferReader* buf) {
bool ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
return ret;
}
char tbufReadChar(SBufferReader* buf) {
char ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
return ret;
}
int8_t tbufReadInt8(SBufferReader* buf) {
int8_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
return ret;
}
uint8_t tbufReadUint8(SBufferReader* buf) {
uint8_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
return ret;
}
int16_t tbufReadInt16(SBufferReader* buf) {
int16_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
if (buf->endian) {
return (int16_t)ntohs(ret);
}
return ret;
}
uint16_t tbufReadUint16(SBufferReader* buf) {
uint16_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
if (buf->endian) {
return ntohs(ret);
}
return ret;
}
int32_t tbufReadInt32(SBufferReader* buf) {
int32_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
if (buf->endian) {
return (int32_t)ntohl(ret);
}
return ret;
}
uint32_t tbufReadUint32(SBufferReader* buf) {
uint32_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
if (buf->endian) {
return ntohl(ret);
}
return ret;
}
int64_t tbufReadInt64(SBufferReader* buf) {
int64_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
if (buf->endian) {
return (int64_t)htobe64(ret); // TODO: ntohll
}
return ret;
}
uint64_t tbufReadUint64(SBufferReader* buf) {
uint64_t ret;
tbufReadToBuffer(buf, &ret, sizeof(ret));
if (buf->endian) {
return htobe64(ret); // TODO: ntohll
}
return ret;
}
float tbufReadFloat(SBufferReader* buf) {
Un4B _un;
tbufReadToBuffer(buf, &_un, sizeof(_un));
if (buf->endian) {
_un.ui = ntohl(_un.ui);
}
return _un.f;
}
double tbufReadDouble(SBufferReader* buf) {
Un8B _un;
tbufReadToBuffer(buf, &_un, sizeof(_un));
if (buf->endian) {
_un.ull = htobe64(_un.ull);
}
return _un.d;
}
////////////////////////////////////////////////////////////////////////////////
// writer functions
void tbufCloseWriter(SBufferWriter* buf) {
taosMemoryFreeClear(buf->data);
// (*buf->allocator)( buf->data, 0 ); // potential memory leak.
buf->data = NULL;
buf->pos = 0;
buf->size = 0;
}
void tbufEnsureCapacity(SBufferWriter* buf, size_t size) {
size += buf->pos;
if (size > buf->size) {
size_t nsize = size + buf->size;
char* data = (*buf->allocator)(buf->data, nsize);
// TODO: the exception should be thrown by the allocator function
if (data == NULL) {
THROW(-1);
}
buf->data = data;
buf->size = nsize;
}
}
size_t tbufReserve(SBufferWriter* buf, size_t size) {
tbufEnsureCapacity(buf, size);
size_t old = buf->pos;
buf->pos += size;
return old;
}
char* tbufGetData(SBufferWriter* buf, bool takeOver) {
char* ret = buf->data;
if (takeOver) {
buf->pos = 0;
buf->size = 0;
buf->data = NULL;
}
return ret;
}
void tbufWrite(SBufferWriter* buf, const void* data, size_t size) {
assert(data != NULL);
tbufEnsureCapacity(buf, size);
memcpy(buf->data + buf->pos, data, size);
buf->pos += size;
}
void tbufWriteAt(SBufferWriter* buf, size_t pos, const void* data, size_t size) {
assert(data != NULL);
// this function can only be called to fill the gap on previous writes,
// so 'pos + size <= buf->pos' must be true
assert(pos + size <= buf->pos);
memcpy(buf->data + pos, data, size);
}
static void tbufWriteLength(SBufferWriter* buf, size_t len) {
// maximum length is 65535, if larger length is required
// this function and the corresponding read function need to be
// revised.
assert(len <= 0xffff);
tbufWriteUint16(buf, (uint16_t)len);
}
void tbufWriteStringLen(SBufferWriter* buf, const char* str, size_t len) {
tbufWriteLength(buf, len);
tbufWrite(buf, str, len);
tbufWriteChar(buf, '\0');
}
void tbufWriteString(SBufferWriter* buf, const char* str) { tbufWriteStringLen(buf, str, strlen(str)); }
void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len) {
tbufWriteLength(buf, len);
tbufWrite(buf, data, len);
}
void tbufWriteBool(SBufferWriter* buf, bool data) { tbufWrite(buf, &data, sizeof(data)); }
void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
void tbufWriteChar(SBufferWriter* buf, char data) { tbufWrite(buf, &data, sizeof(data)); }
void tbufWriteCharAt(SBufferWriter* buf, size_t pos, char data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
void tbufWriteInt8(SBufferWriter* buf, int8_t data) { tbufWrite(buf, &data, sizeof(data)); }
void tbufWriteInt8At(SBufferWriter* buf, size_t pos, int8_t data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
void tbufWriteUint8(SBufferWriter* buf, uint8_t data) { tbufWrite(buf, &data, sizeof(data)); }
void tbufWriteUint8At(SBufferWriter* buf, size_t pos, uint8_t data) { tbufWriteAt(buf, pos, &data, sizeof(data)); }
void tbufWriteInt16(SBufferWriter* buf, int16_t data) {
if (buf->endian) {
data = (int16_t)htons(data);
}
tbufWrite(buf, &data, sizeof(data));
}
void tbufWriteInt16At(SBufferWriter* buf, size_t pos, int16_t data) {
if (buf->endian) {
data = (int16_t)htons(data);
}
tbufWriteAt(buf, pos, &data, sizeof(data));
}
void tbufWriteUint16(SBufferWriter* buf, uint16_t data) {
if (buf->endian) {
data = htons(data);
}
tbufWrite(buf, &data, sizeof(data));
}
void tbufWriteUint16At(SBufferWriter* buf, size_t pos, uint16_t data) {
if (buf->endian) {
data = htons(data);
}
tbufWriteAt(buf, pos, &data, sizeof(data));
}
void tbufWriteInt32(SBufferWriter* buf, int32_t data) {
if (buf->endian) {
data = (int32_t)htonl(data);
}
tbufWrite(buf, &data, sizeof(data));
}
void tbufWriteInt32At(SBufferWriter* buf, size_t pos, int32_t data) {
if (buf->endian) {
data = (int32_t)htonl(data);
}
tbufWriteAt(buf, pos, &data, sizeof(data));
}
void tbufWriteUint32(SBufferWriter* buf, uint32_t data) {
if (buf->endian) {
data = htonl(data);
}
tbufWrite(buf, &data, sizeof(data));
}
void tbufWriteUint32At(SBufferWriter* buf, size_t pos, uint32_t data) {
if (buf->endian) {
data = htonl(data);
}
tbufWriteAt(buf, pos, &data, sizeof(data));
}
void tbufWriteInt64(SBufferWriter* buf, int64_t data) {
if (buf->endian) {
data = (int64_t)htobe64(data);
}
tbufWrite(buf, &data, sizeof(data));
}
void tbufWriteInt64At(SBufferWriter* buf, size_t pos, int64_t data) {
if (buf->endian) {
data = (int64_t)htobe64(data);
}
tbufWriteAt(buf, pos, &data, sizeof(data));
}
void tbufWriteUint64(SBufferWriter* buf, uint64_t data) {
if (buf->endian) {
data = htobe64(data);
}
tbufWrite(buf, &data, sizeof(data));
}
void tbufWriteUint64At(SBufferWriter* buf, size_t pos, uint64_t data) {
if (buf->endian) {
data = htobe64(data);
}
tbufWriteAt(buf, pos, &data, sizeof(data));
}
void tbufWriteFloat(SBufferWriter* buf, float data) {
Un4B _un;
_un.f = data;
if (buf->endian) {
_un.ui = htonl(_un.ui);
}
tbufWrite(buf, &_un, sizeof(_un));
}
void tbufWriteFloatAt(SBufferWriter* buf, size_t pos, float data) {
Un4B _un;
_un.f = data;
if (buf->endian) {
_un.ui = htonl(_un.ui);
}
tbufWriteAt(buf, pos, &_un, sizeof(_un));
}
void tbufWriteDouble(SBufferWriter* buf, double data) {
Un8B _un;
_un.d = data;
if (buf->endian) {
_un.ull = htobe64(_un.ull);
}
tbufWrite(buf, &_un, sizeof(_un));
}
void tbufWriteDoubleAt(SBufferWriter* buf, size_t pos, double data) {
Un8B _un;
_un.d = data;
if (buf->endian) {
_un.ull = htobe64(_un.ull);
}
tbufWriteAt(buf, pos, &_un, sizeof(_un));
}

6
source/util/src/tcompare.c
View File

@ -244,7 +244,7 @@ int32_t compareJsonVal(const void *pLeft, const void *pRight) {
} else if (leftType == TSDB_DATA_TYPE_NULL) {
return 0;
} else {
assert(0);
ASSERTS(0, "data type unexpected");
return 0;
}
}
@ -1285,7 +1285,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_TIMESTAMP:
return setChkInBytes8;
default:
assert(0);
ASSERTS(0, "data type unexpected");
}
}
@ -1308,7 +1308,7 @@ __compar_fn_t getComparFunc(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_TIMESTAMP:
return setChkNotInBytes8;
default:
assert(0);
ASSERTS(0, "data type unexpected");
}
}

Some files were not shown because too many files have changed in this diff Show More