xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch main into FIX/TD-21043-main

This commit is contained in:
Benguang Zhao 2023-01-02 16:41:46 +08:00
parent 2baa71883d d290daa197
commit 4031dc1af4
151 changed files with 5016 additions and 4720 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/cmake.version
View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
SET(TD_VER_NUMBER "3.0.2.1")
SET(TD_VER_NUMBER "3.0.2.2")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)

2
cmake/taosadapter_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
GIT_TAG 5662a6d
GIT_TAG a2e9920
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

2
include/client/taos.h
View File

@ -161,6 +161,8 @@ DLL_EXPORT int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags)
DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name);
DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields);
// let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields`
DLL_EXPORT void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields);
DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert);
DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums);

4
include/common/tmsgcb.h
View File

@ -39,6 +39,7 @@ typedef enum {
QUEUE_MAX,
} EQueueType;
typedef int32_t (*UpdateDnodeInfoFp)(void* pData, int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port);
typedef int32_t (*PutToQueueFp)(void* pMgmt, EQueueType qtype, SRpcMsg* pMsg);
typedef int32_t (*GetQueueSizeFp)(void* pMgmt, int32_t vgId, EQueueType qtype);
typedef int32_t (*SendReqFp)(const SEpSet* pEpSet, SRpcMsg* pMsg);
@ -48,6 +49,7 @@ typedef void (*ReleaseHandleFp)(SRpcHandleInfo* pHandle, int8_t type);
typedef void (*ReportStartup)(const char* name, const char* desc);
typedef struct {
void* data;
void* mgmt;
void* clientRpc;
PutToQueueFp putToQueueFp;
@ -57,6 +59,7 @@ typedef struct {
RegisterBrokenLinkArgFp registerBrokenLinkArgFp;
ReleaseHandleFp releaseHandleFp;
ReportStartup reportStartupFp;
UpdateDnodeInfoFp updateDnodeInfoFp;
} SMsgCb;
void tmsgSetDefault(const SMsgCb* msgcb);
@ -67,6 +70,7 @@ void tmsgSendRsp(SRpcMsg* pMsg);
void tmsgRegisterBrokenLinkArg(SRpcMsg* pMsg);
void tmsgReleaseHandle(SRpcHandleInfo* pHandle, int8_t type);
void tmsgReportStartup(const char* name, const char* desc);
int32_t tmsgUpdateDnodeInfo(int32_t* dnodeId, int64_t* clusterId, char* fqdn, uint16_t* port);
#ifdef __cplusplus
}

1
include/common/trow.h
View File

@ -20,7 +20,6 @@
#include "talgo.h"
#include "taosdef.h"
#include "taoserror.h"
#include "tbuffer.h"
#include "tdataformat.h"
#include "tdef.h"
#include "ttypes.h"

10
include/libs/function/function.h
View File

@ -20,7 +20,6 @@
extern "C" {
#endif
#include "tbuffer.h"
#include "tcommon.h"
#include "tvariant.h"
@ -138,7 +137,7 @@ typedef struct SqlFunctionCtx {
char *pOutput; // final result output buffer, point to sdata->data
int32_t numOfParams;
// input parameter, e.g., top(k, 20), the number of results of top query is kept in param
SFunctParam *param;
SFunctParam *param;
// corresponding output buffer for timestamp of each result, e.g., diff/csum
SColumnInfoData *pTsOutput;
int32_t offset;
@ -152,6 +151,7 @@ typedef struct SqlFunctionCtx {
struct SSDataBlock *pSrcBlock;
struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity
SSerializeDataHandle saveHandle;
int32_t exprIdx;
char udfName[TSDB_FUNC_NAME_LEN];
} SqlFunctionCtx;
@ -182,9 +182,9 @@ struct SScalarParam {
int32_t numOfQualified; // number of qualified elements in the final results
};
void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell);
bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry);
bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry);
void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell);
bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry);
bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry);
typedef struct SPoint {
int64_t key;

2
include/libs/nodes/plannodes.h
View File

@ -121,6 +121,7 @@ typedef struct SAggLogicNode {
bool hasLast;
bool hasTimeLineFunc;
bool onlyHasKeepOrderFunc;
bool hasGroupKeyOptimized;
} SAggLogicNode;
typedef struct SProjectLogicNode {
@ -402,6 +403,7 @@ typedef struct SAggPhysiNode {
SNodeList* pGroupKeys;
SNodeList* pAggFuncs;
bool mergeDataBlock;
bool groupKeyOptimized;
} SAggPhysiNode;
typedef struct SDownstreamSourceNode {

16
include/libs/stream/tstream.h
View File

@ -392,21 +392,6 @@ static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) {
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED);
}
static FORCE_INLINE int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
if (pTask->outputType == TASK_OUTPUT__TABLE) {
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else if (pTask->outputType == TASK_OUTPUT__SMA) {
pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else {
taosWriteQitem(pTask->outputQueue->queue, pBlock);
}
return 0;
}
typedef struct {
SMsgHead head;
int64_t streamId;
@ -584,6 +569,7 @@ int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp);
int32_t streamTryExec(SStreamTask* pTask);
int32_t streamSchedExec(SStreamTask* pTask);
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock);
int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz);

2
include/libs/sync/sync.h
View File

@ -80,6 +80,8 @@ typedef enum {
} ESyncState;
typedef struct SNodeInfo {
int64_t clusterId;
int32_t nodeId;
uint16_t nodePort;
char nodeFqdn[TSDB_FQDN_LEN];
} SNodeInfo;

6
include/os/osMemory.h
View File

@ -22,12 +22,16 @@ extern "C" {
// If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following sectio
#if !defined(WINDOWS)
#ifndef ALLOW_FORBID_FUNC
#define malloc MALLOC_FUNC_TAOS_FORBID
#define calloc CALLOC_FUNC_TAOS_FORBID
#define realloc REALLOC_FUNC_TAOS_FORBID
#define free FREE_FUNC_TAOS_FORBID
#endif
#endif // ifndef ALLOW_FORBID_FUNC
#endif // if !defined(WINDOWS)
void *taosMemoryMalloc(int64_t size);
void *taosMemoryCalloc(int64_t num, int64_t size);

1
include/os/osSysinfo.h
View File

@ -70,6 +70,7 @@ typedef struct {
SysNameInfo taosGetSysNameInfo();
bool taosCheckCurrentInDll();
int taosGetlocalhostname(char *hostname, size_t maxLen);
#ifdef __cplusplus
}

33
include/os/osSystem.h
View File

@ -62,11 +62,38 @@ void taosResetTerminalMode();
taosMemoryFree(strings); \
}
#else
#include <windows.h>
#include <dbghelp.h>
#define STACKSIZE 64
#define taosPrintTrace(flags, level, dflag) \
{ \
taosPrintLog(flags, level, dflag, \
"backtrace not implemented on windows, so detailed stack information cannot be printed"); \
}
unsigned int i; \
void* stack[STACKSIZE]; \
unsigned short frames; \
SYMBOL_INFO* symbol; \
HANDLE process; \
\
process = GetCurrentProcess(); \
\
SymInitialize(process, NULL, TRUE); \
\
frames = CaptureStackBackTrace(0, STACKSIZE, stack, NULL); \
symbol = (SYMBOL_INFO*)calloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char), 1); \
if (symbol != NULL) { \
symbol->MaxNameLen = 255; \
symbol->SizeOfStruct = sizeof(SYMBOL_INFO); \
\
if (frames > 0) { \
taosPrintLog(flags, level, dflag, "obtained %d stack frames", frames); \
for (i = 0; i < frames; i++) { \
SymFromAddr(process, (DWORD64)(stack[i]), 0, symbol); \
taosPrintLog(flags, level, dflag, "frame:%i: %s - 0x%0X", frames - i - 1, symbol->Name, symbol->Address); \
} \
} \
free(symbol); \
} \
}
#endif
#ifdef __cplusplus

168
include/util/tbuffer.h
View File

@ -1,168 +0,0 @@
/*
* Copyright (c) 2020 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_BUFFER_H_
#define _TD_UTIL_BUFFER_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
////////////////////////////////////////////////////////////////////////////////
// usage example
/*
#include <stdio.h>
#include "texception.h"
int32_t main( int32_t argc, char** argv ) {
SBufferWriter bw = tbufInitWriter( NULL, false );
TRY( 1 ) {
//--------------------- write ------------------------
// reserve 1024 bytes for the buffer to improve performance
tbufEnsureCapacity( &bw, 1024 );
// reserve space for the interger count
size_t pos = tbufReserve( &bw, sizeof(int32_t) );
// write 5 integers to the buffer
for( int32_t i = 0; i < 5; i++) {
tbufWriteInt32( &bw, i );
}
// write the integer count to buffer at reserved position
tbufWriteInt32At( &bw, pos, 5 );
// write a string to the buffer
tbufWriteString( &bw, "this is a string.\n" );
// acquire the result and close the write buffer
size_t size = tbufTell( &bw );
char* data = tbufGetData( &bw, false );
//------------------------ read -----------------------
SBufferReader br = tbufInitReader( data, size, false );
// read & print out all integers
int32_t count = tbufReadInt32( &br );
for( int32_t i = 0; i < count; i++ ) {
printf( "%d\n", tbufReadInt32(&br) );
}
// read & print out a string
puts( tbufReadString(&br, NULL) );
// try read another integer, this result in an error as there no this integer
tbufReadInt32( &br );
printf( "you should not see this message.\n" );
} CATCH( code ) {
printf( "exception code is: %d, you will see this message after print out 5 integers and a string.\n", code );
} END_TRY
tbufCloseWriter( &bw );
return 0;
}
*/
typedef struct SBufferReader {
bool endian;
const char* data;
size_t pos;
size_t size;
} SBufferReader;
typedef struct SBufferWriter {
bool endian;
char* data;
size_t pos;
size_t size;
void* (*allocator)(void*, size_t);
} SBufferWriter;
// common functions & macros for both reader & writer
#define tbufTell(buf) ((buf)->pos)
/* ------------------------ BUFFER WRITER FUNCTIONS AND MACROS ------------------------ */
// *Allocator*, function to allocate memory, will use 'realloc' if NULL
// *Endian*, if true, writer functions of primitive types will do 'hton' automatically
#define tbufInitWriter(Allocator, Endian) \
{ .endian = (Endian), .data = NULL, .pos = 0, .size = 0, .allocator = ((Allocator) == NULL ? realloc : (Allocator)) }
void tbufCloseWriter(SBufferWriter* buf);
void tbufEnsureCapacity(SBufferWriter* buf, size_t size);
size_t tbufReserve(SBufferWriter* buf, size_t size);
char* tbufGetData(SBufferWriter* buf, bool takeOver);
void tbufWrite(SBufferWriter* buf, const void* data, size_t size);
void tbufWriteAt(SBufferWriter* buf, size_t pos, const void* data, size_t size);
void tbufWriteStringLen(SBufferWriter* buf, const char* str, size_t len);
void tbufWriteString(SBufferWriter* buf, const char* str);
// the prototype of tbufWriteBinary and tbufWrite are identical
// the difference is: tbufWriteBinary writes the length of the data to the buffer
// first, then the actual data, which means the reader don't need to know data
// size before read. Write only write the data itself, which means the reader
// need to know data size before read.
void tbufWriteBinary(SBufferWriter* buf, const void* data, size_t len);
void tbufWriteBool(SBufferWriter* buf, bool data);
void tbufWriteBoolAt(SBufferWriter* buf, size_t pos, bool data);
void tbufWriteChar(SBufferWriter* buf, char data);
void tbufWriteCharAt(SBufferWriter* buf, size_t pos, char data);
void tbufWriteInt8(SBufferWriter* buf, int8_t data);
void tbufWriteInt8At(SBufferWriter* buf, size_t pos, int8_t data);
void tbufWriteUint8(SBufferWriter* buf, uint8_t data);
void tbufWriteUint8At(SBufferWriter* buf, size_t pos, uint8_t data);
void tbufWriteInt16(SBufferWriter* buf, int16_t data);
void tbufWriteInt16At(SBufferWriter* buf, size_t pos, int16_t data);
void tbufWriteUint16(SBufferWriter* buf, uint16_t data);
void tbufWriteUint16At(SBufferWriter* buf, size_t pos, uint16_t data);
void tbufWriteInt32(SBufferWriter* buf, int32_t data);
void tbufWriteInt32At(SBufferWriter* buf, size_t pos, int32_t data);
void tbufWriteUint32(SBufferWriter* buf, uint32_t data);
void tbufWriteUint32At(SBufferWriter* buf, size_t pos, uint32_t data);
void tbufWriteInt64(SBufferWriter* buf, int64_t data);
void tbufWriteInt64At(SBufferWriter* buf, size_t pos, int64_t data);
void tbufWriteUint64(SBufferWriter* buf, uint64_t data);
void tbufWriteUint64At(SBufferWriter* buf, size_t pos, uint64_t data);
void tbufWriteFloat(SBufferWriter* buf, float data);
void tbufWriteFloatAt(SBufferWriter* buf, size_t pos, float data);
void tbufWriteDouble(SBufferWriter* buf, double data);
void tbufWriteDoubleAt(SBufferWriter* buf, size_t pos, double data);
/* ------------------------ BUFFER READER FUNCTIONS AND MACROS ------------------------ */
// *Endian*, if true, reader functions of primitive types will do 'ntoh' automatically
#define tbufInitReader(Data, Size, Endian) \
{ .endian = (Endian), .data = (Data), .pos = 0, .size = ((Data) == NULL ? 0 : (Size)) }
size_t tbufSkip(SBufferReader* buf, size_t size);
const char* tbufRead(SBufferReader* buf, size_t size);
void tbufReadToBuffer(SBufferReader* buf, void* dst, size_t size);
const char* tbufReadString(SBufferReader* buf, size_t* len);
size_t tbufReadToString(SBufferReader* buf, char* dst, size_t size);
const char* tbufReadBinary(SBufferReader* buf, size_t* len);
size_t tbufReadToBinary(SBufferReader* buf, void* dst, size_t size);
bool tbufReadBool(SBufferReader* buf);
char tbufReadChar(SBufferReader* buf);
int8_t tbufReadInt8(SBufferReader* buf);
uint8_t tbufReadUint8(SBufferReader* buf);
int16_t tbufReadInt16(SBufferReader* buf);
uint16_t tbufReadUint16(SBufferReader* buf);
int32_t tbufReadInt32(SBufferReader* buf);
uint32_t tbufReadUint32(SBufferReader* buf);
int64_t tbufReadInt64(SBufferReader* buf);
uint64_t tbufReadUint64(SBufferReader* buf);
float tbufReadFloat(SBufferReader* buf);
double tbufReadDouble(SBufferReader* buf);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_BUFFER_H_*/

29
include/util/tcompare.h
View File

@ -36,17 +36,18 @@ extern "C" {
#define FLT_GREATEREQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) > (_y)))
#define FLT_LESSEQUAL(_x, _y) (FLT_EQUAL((_x), (_y)) || ((_x) < (_y)))
#define PATTERN_COMPARE_INFO_INITIALIZER \
{ '%', '_' }
#define PATTERN_COMPARE_INFO_INITIALIZER { '%', '_', L'%', L'_' }
typedef struct SPatternCompareInfo {
char matchAll; // symbol for match all wildcard, default: '%'
char matchOne; // symbol for match one wildcard, default: '_'
char matchAll; // symbol for match all wildcard, default: '%'
char matchOne; // symbol for match one wildcard, default: '_'
TdUcs4 umatchAll; // unicode version matchAll
TdUcs4 umatchOne; // unicode version matchOne
} SPatternCompareInfo;
int32_t patternMatch(const char *pattern, const char *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t ssize, const SPatternCompareInfo *pInfo);
int32_t WCSPatternMatch(const TdUcs4 *pattern, const TdUcs4 *str, size_t size, const SPatternCompareInfo *pInfo);
int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, size_t ssize, const SPatternCompareInfo *pInfo);
int32_t taosArrayCompareString(const void *a, const void *b);
@ -79,9 +80,11 @@ int32_t compareDoubleVal(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStr(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight);
int32_t compareStrRegexComp(const void *pLeft, const void *pRight);
int32_t compareStrRegexCompMatch(const void *pLeft, const void *pRight);
int32_t compareStrRegexCompNMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight);
int32_t compareInt8ValDesc(const void *pLeft, const void *pRight);
int32_t compareInt16ValDesc(const void *pLeft, const void *pRight);
@ -99,11 +102,11 @@ int32_t compareUint64ValDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedStrDesc(const void *pLeft, const void *pRight);
int32_t compareLenPrefixedWStrDesc(const void *pLeft, const void *pRight);
int32_t compareStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t comparestrPatternMatch(const void *pLeft, const void *pRight);
int32_t comparestrPatternNMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternMatch(const void *pLeft, const void *pRight);
int32_t compareWStrPatternNotMatch(const void *pLeft, const void *pRight);
int32_t comparewcsPatternMatch(const void *pLeft, const void *pRight);
int32_t comparewcsPatternNMatch(const void *pLeft, const void *pRight);
int32_t compareInt8Int16(const void *pLeft, const void *pRight);
int32_t compareInt8Int32(const void *pLeft, const void *pRight);

19
include/util/tdef.h
View File

@ -189,12 +189,13 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_COLUMNS 4096
#define TSDB_MIN_COLUMNS 2 // PRIMARY COLUMN(timestamp) + other columns
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string
#define TSDB_CGROUP_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string
#define TSDB_CGROUP_LEN 193 // it is a null-terminated string
#define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 65
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_FUNC_NAME_LEN 65
#define TSDB_FUNC_COMMENT_LEN 1024 * 1024
@ -212,8 +213,8 @@ typedef enum ELogicConditionType {
#define TSDB_INDEX_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_INDEX_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_STREAM_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TOPIC_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_STREAM_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_STREAM_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2)
#define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20)
#define TSDB_COL_NAME_LEN 65
@ -310,7 +311,7 @@ typedef enum ELogicConditionType {
#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.
#define TSDB_MAX_KEEP_NS (365 * 292 * 1440) // data in db to be reserved.
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_DEFAULT_KEEP (3650 * 1440) // ten years
#define TSDB_MIN_MINROWS_FBLOCK 10
#define TSDB_MAX_MINROWS_FBLOCK 1000
#define TSDB_DEFAULT_MINROWS_FBLOCK 100

21
include/util/tjson.h
View File

@ -30,6 +30,27 @@ extern "C" {
val = _tmp; \
} while (0)
#define tjsonGetInt32ValueFromDouble(pJson, pName, val, code) \
do { \
double _tmp = 0; \
code = tjsonGetDoubleValue(pJson, pName, &_tmp); \
val = (int32_t)_tmp; \
} while (0)
#define tjsonGetInt8ValueFromDouble(pJson, pName, val, code) \
do { \
double _tmp = 0; \
code = tjsonGetDoubleValue(pJson, pName, &_tmp); \
val = (int8_t)_tmp; \
} while (0)
#define tjsonGetUInt16ValueFromDouble(pJson, pName, val, code) \
do { \
double _tmp = 0; \
code = tjsonGetDoubleValue(pJson, pName, &_tmp); \
val = (uint16_t)_tmp; \
} while (0)
typedef void SJson;
SJson* tjsonCreateObject();

11
include/util/tlog.h
View File

@ -83,9 +83,14 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons
#endif
;
bool taosAssert(bool condition, const char *file, int32_t line, const char *format, ...);
#define ASSERTS(condition, ...) taosAssert(condition, __FILE__, __LINE__, __VA_ARGS__)
#define ASSERT(condition) ASSERTS(condition, "assert info not provided")
bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...);
bool taosAssertRelease(bool condition);
#define ASSERTS(condition, ...) taosAssertDebug(condition, __FILE__, __LINE__, __VA_ARGS__)
#ifdef NDEBUG
#define ASSERT(condition) taosAssertRelease(condition)
#else
#define ASSERT(condition) taosAssertDebug(condition, __FILE__, __LINE__, "assert info not provided")
#endif
// clang-format off
#define uFatal(...) { if (uDebugFlag & DEBUG_FATAL) { taosPrintLog("UTL FATAL", DEBUG_FATAL, tsLogEmbedded ? 255 : uDebugFlag, __VA_ARGS__); }}

6
include/util/tutil.h
View File

@ -29,11 +29,17 @@ extern "C" {
int32_t strdequote(char *src);
size_t strtrim(char *src);
char *strnchr(const char *haystack, char needle, int32_t len, bool skipquote);
TdUcs4* wcsnchr(const TdUcs4* haystack, TdUcs4 needle, size_t len);
char **strsplit(char *src, const char *delim, int32_t *num);
char *strtolower(char *dst, const char *src);
char *strntolower(char *dst, const char *src, int32_t n);
char *strntolower_s(char *dst, const char *src, int32_t n);
int64_t strnatoi(char *num, int32_t len);
size_t tstrncspn(const char *str, size_t ssize, const char *reject, size_t rsize);
size_t twcsncspn(const TdUcs4 *wcs, size_t size, const TdUcs4 *reject, size_t rsize);
char *strbetween(char *string, char *begin, char *end);
char *paGetToken(char *src, char **token, int32_t *tokenLen);

8
source/client/src/clientMain.c
View File

@ -1331,6 +1331,14 @@ int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fiel
return stmtGetColFields(stmt, fieldNum, fields);
}
// let stmt to reclaim TAOS_FIELD_E that was allocated by `taos_stmt_get_tag_fields`/`taos_stmt_get_col_fields`
void taos_stmt_reclaim_fields(TAOS_STMT *stmt, TAOS_FIELD_E *fields)
{
(void)stmt;
if (!fields) return;
taosMemoryFree(fields);
}
int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind) {
if (stmt == NULL || bind == NULL) {
tscError("NULL parameter for %s", __FUNCTION__);

2
source/common/src/tglobal.c
View File

@ -720,6 +720,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
tsSIMDBuiltins = (bool) cfgGetItem(pCfg, "SIMD-builtins")->bval;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32;
tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN);

5
source/common/src/ttypes.c
View File

@ -131,7 +131,10 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
varDataCopy(val, src);
break;
default: {
memcpy(val, src, len);
if (len > 0) {
memcpy(val, src, len);
}
break;
}
}

3
source/dnode/mgmt/exe/dmMain.c
View File

@ -178,7 +178,7 @@ static int32_t dmInitLog() {
}
static void taosCleanupArgs() {
if (global.envCmd != NULL) taosMemoryFree(global.envCmd);
if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd);
}
int main(int argc, char const *argv[]) {
@ -271,7 +271,6 @@ int mainWindows(int argc, char **argv) {
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
taosConvDestroy();
return -1;
}

2
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -180,6 +180,6 @@ int32_t mmWriteFile(const char *path, const SMnodeOpt *pOption) {
return -1;
}
dDebug("successed to write %s, deployed:%d", realfile, pOption->deploy);
dDebug("succeed to write %s, deployed:%d", realfile, pOption->deploy);
return 0;
}

18
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -16,7 +16,7 @@
#define _DEFAULT_SOURCE
#include "vmInt.h"
#define MAX_CONTENT_LEN 1024 * 1024
#define MAX_CONTENT_LEN 2 * 1024 * 1024
SVnodeObj **vmGetVnodeListFromHash(SVnodeMgmt *pMgmt, int32_t *numOfVnodes) {
taosThreadRwlockRdlock(&pMgmt->lock);
@ -60,7 +60,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
dDebug("file %s not exist", file);
dInfo("file %s not exist", file);
code = 0;
goto _OVER;
}
@ -133,7 +133,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
*numOfVnodes = vnodesNum;
code = 0;
dDebug("succcessed to read file %s, numOfVnodes:%d", file, vnodesNum);
dInfo("succcessed to read file %s, numOfVnodes:%d", file, vnodesNum);
_OVER:
if (content != NULL) taosMemoryFree(content);
@ -163,6 +163,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
if (ppVnodes == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
dError("failed to write %s while get vnodelist", file);
goto _OVER;
}
@ -172,6 +173,7 @@ int32_t vmWriteVnodeListToFile(SVnodeMgmt *pMgmt) {
if (content == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
dError("failed to write %s while malloc content", file);
goto _OVER;
}
@ -213,6 +215,12 @@ _OVER:
if (code != 0) return -1;
dDebug("successed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
return taosRenameFile(file, realfile);
dInfo("succeed to write %s, numOfVnodes:%d", realfile, numOfVnodes);
code = taosRenameFile(file, realfile);
if (code != 0) {
dError("failed to rename %s to %s", file, realfile);
}
return code;
}

15
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -134,8 +134,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo));
for (int i = 0; i < pCreate->replica; ++i) {
SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i];
pNode->nodeId = pCreate->replicas[i].id;
pNode->nodePort = pCreate->replicas[i].port;
tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN);
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
}
}
@ -188,8 +190,8 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix,
req.hashSuffix, req.replica, req.selfIndex, req.strict);
for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d id:%d fqdn:%s port:%u", req.vgId, i, req.replicas[i].id, req.replicas[i].fqdn,
req.replicas[i].port);
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
req.replicas[i].id);
}
SReplica *pReplica = &req.replicas[req.selfIndex];
@ -213,7 +215,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
if (pVnode != NULL) {
dDebug("vgId:%d, already exist", req.vgId);
dInfo("vgId:%d, already exist", req.vgId);
tFreeSCreateVnodeReq(&req);
vmReleaseVnode(pMgmt, pVnode);
terrno = TSDB_CODE_VND_ALREADY_EXIST;
@ -358,7 +360,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
int32_t vgId = dropReq.vgId;
dDebug("vgId:%d, start to drop vnode", vgId);
dInfo("vgId:%d, start to drop vnode", vgId);
if (dropReq.dnodeId != pMgmt->pData->dnodeId) {
terrno = TSDB_CODE_INVALID_MSG;
@ -368,7 +370,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, vgId);
if (pVnode == NULL) {
dDebug("vgId:%d, failed to drop since %s", vgId, terrstr());
dInfo("vgId:%d, failed to drop since %s", vgId, terrstr());
terrno = TSDB_CODE_VND_NOT_EXIST;
return -1;
}
@ -383,6 +385,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
vmCloseVnode(pMgmt, pVnode);
vmWriteVnodeListToFile(pMgmt);
dInfo("vgId:%d, is dropped", vgId);
return 0;
}

10
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -140,7 +140,7 @@ static void *vmOpenVnodeInThread(void *param) {
SVnodeMgmt *pMgmt = pThread->pMgmt;
char path[TSDB_FILENAME_LEN];
dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
dInfo("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum);
setThreadName("open-vnodes");
for (int32_t v = 0; v < pThread->vnodeNum; ++v) {
@ -158,14 +158,14 @@ static void *vmOpenVnodeInThread(void *param) {
pThread->failed++;
} else {
vmOpenVnode(pMgmt, pCfg, pImpl);
dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
dInfo("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex);
pThread->opened++;
atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
}
}
dDebug("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
pThread->failed);
dInfo("thread:%d, numOfVnodes:%d, opened:%d failed:%d", pThread->threadIndex, pThread->vnodeNum, pThread->opened,
pThread->failed);
return NULL;
}
@ -498,7 +498,7 @@ static void *vmRestoreVnodeInThread(void *param) {
dError("vgId:%d, failed to restore vnode by thread:%d", pVnode->vgId, pThread->threadIndex);
pThread->failed++;
} else {
dDebug("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex);
dInfo("vgId:%d, is restored by thread:%d", pVnode->vgId, pThread->threadIndex);
pThread->opened++;
atomic_add_fetch_32(&pMgmt->state.openVnodes, 1);
}

2
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -345,6 +345,8 @@ SMsgCb dmGetMsgcb(SDnode *pDnode) {
.registerBrokenLinkArgFp = dmRegisterBrokenLinkArg,
.releaseHandleFp = dmReleaseHandle,
.reportStartupFp = dmReportStartup,
.updateDnodeInfoFp = dmUpdateDnodeInfo,
.data = &pDnode->data,
};
return msgCb;
}

1
source/dnode/mgmt/node_util/inc/dmUtil.h
View File

@ -167,6 +167,7 @@ void dmUpdateEps(SDnodeData *pData, SArray *pDnodeEps);
void dmGetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet);
void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet);
int32_t dmUpdateDnodeInfo(void *pData, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port);
#ifdef __cplusplus
}

82
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -182,22 +182,25 @@ _OVER:
}
int32_t dmWriteEps(SDnodeData *pData) {
int32_t code = -1;
char *content = NULL;
TdFilePtr pFile = NULL;
char file[PATH_MAX] = {0};
char realfile[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%sdnode%sdnode.json.bak", tsDataDir, TD_DIRSEP, TD_DIRSEP);
snprintf(realfile, sizeof(realfile), "%s%sdnode%sdnode.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
TdFilePtr pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
dError("failed to write %s since %s", file, strerror(errno));
dError("failed to open %s since %s", file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
return -1;
goto _OVER;
}
int32_t len = 0;
int32_t maxLen = 256 * 1024;
char *content = taosMemoryCalloc(1, maxLen + 1);
content = taosMemoryCalloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"dnodeId\": %d,\n", pData->dnodeId);
@ -221,20 +224,39 @@ int32_t dmWriteEps(SDnodeData *pData) {
}
len += snprintf(content + len, maxLen - len, "}\n");
taosWriteFile(pFile, content, len);
taosFsyncFile(pFile);
if (taosWriteFile(pFile, content, len) != len) {
dError("failed to write %s since %s", file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
if (taosFsyncFile(pFile) < 0) {
dError("failed to fsync %s since %s", file, strerror(errno));
terrno = TAOS_SYSTEM_ERROR(errno);
goto _OVER;
}
taosCloseFile(&pFile);
taosMemoryFree(content);
taosMemoryFreeClear(content);
if (taosRenameFile(file, realfile) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
dError("failed to rename %s since %s", file, terrstr());
return -1;
goto _OVER;
}
code = 0;
pData->updateTime = taosGetTimestampMs();
dDebug("successed to write %s, dnodeVer:%" PRId64, realfile, pData->dnodeVer);
return 0;
dInfo("succeed to write %s, dnodeVer:%" PRId64, realfile, pData->dnodeVer);
_OVER:
if (content != NULL) taosMemoryFreeClear(content);
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
dError("failed to write file %s since %s", realfile, terrstr());
}
return code;
}
void dmUpdateEps(SDnodeData *pData, SArray *eps) {
@ -332,3 +354,41 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet) {
dInfo("mnode index:%d %s:%u", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port);
}
}
int32_t dmUpdateDnodeInfo(void *data, int32_t *dnodeId, int64_t *clusterId, char *fqdn, uint16_t *port) {
SDnodeData *pData = data;
int32_t ret = -1;
taosThreadRwlockRdlock(&pData->lock);
if (*dnodeId <= 0) {
for (int32_t i = 0; i < (int32_t)taosArrayGetSize(pData->dnodeEps); ++i) {
SDnodeEp *pDnodeEp = taosArrayGet(pData->dnodeEps, i);
if (strcmp(pDnodeEp->ep.fqdn, fqdn) == 0 && pDnodeEp->ep.port == *port) {
dInfo("dnode:%s:%u, update dnodeId from %d to %d", fqdn, *port, *dnodeId, pDnodeEp->id);
*dnodeId = pDnodeEp->id;
*clusterId = pData->clusterId;
ret = 0;
}
}
if (ret != 0) {
dInfo("dnode:%s:%u, failed to update dnodeId:%d", fqdn, *port, *dnodeId);
}
} else {
SDnodeEp *pDnodeEp = taosHashGet(pData->dnodeHash, dnodeId, sizeof(int32_t));
if (pDnodeEp) {
if (strcmp(pDnodeEp->ep.fqdn, fqdn) != 0) {
dInfo("dnode:%d, update port from %s to %s", *dnodeId, fqdn, pDnodeEp->ep.fqdn);
tstrncpy(fqdn, pDnodeEp->ep.fqdn, TSDB_FQDN_LEN);
}
if (pDnodeEp->ep.port != *port) {
dInfo("dnode:%d, update port from %u to %u", *dnodeId, *port, pDnodeEp->ep.port);
*port = pDnodeEp->ep.port;
}
*clusterId = pData->clusterId;
ret = 0;
} else {
dInfo("dnode:%d, failed to update dnode info", *dnodeId);
}
}
taosThreadRwlockUnlock(&pData->lock);
return ret;
}

2
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -105,7 +105,7 @@ int32_t dmWriteFile(const char *path, const char *name, bool deployed) {
return -1;
}
dInfo("successed to write %s, deployed:%d", realfile, deployed);
dInfo("succeed to write %s, deployed:%d", realfile, deployed);
code = 0;
_OVER:

10
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -21,6 +21,7 @@
#include "mndSync.h"
#include "mndTrans.h"
#include "tmisce.h"
#include "mndCluster.h"
#define MNODE_VER_NUMBER 1
#define MNODE_RESERVE_SIZE 64
@ -743,8 +744,12 @@ static void mndReloadSyncConfig(SMnode *pMnode) {
if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) {
SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum];
tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, sizeof(pNode->nodeFqdn));
pNode->nodeId = pObj->pDnode->id;
pNode->clusterId = mndGetClusterId(pMnode);
pNode->nodePort = pObj->pDnode->port;
tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN);
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId);
if (pObj->pDnode->id == pMnode->selfDnodeId) {
cfg.myIndex = cfg.replicaNum;
}
@ -775,7 +780,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) {
mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex);
for (int32_t i = 0; i < cfg.replicaNum; ++i) {
SNodeInfo *pNode = &cfg.nodeInfo[i];
mInfo("vgId:1, index:%d, fqdn:%s port:%d", i, pNode->nodeFqdn, pNode->nodePort);
mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId,
pNode->clusterId);
}
int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg);

8
source/dnode/mnode/impl/src/mndSync.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "mndSync.h"
#include "mndCluster.h"
#include "mndTrans.h"
static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
@ -297,9 +298,12 @@ int32_t mndInitSync(SMnode *pMnode) {
pCfg->myIndex = pMgmt->selfIndex;
for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) {
SNodeInfo *pNode = &pCfg->nodeInfo[i];
tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
pNode->nodeId = pMgmt->replicas[i].id;
pNode->nodePort = pMgmt->replicas[i].port;
mInfo("vgId:1, index:%d ep:%s:%u", i, pNode->nodeFqdn, pNode->nodePort);
tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId,
pNode->clusterId);
}
tsem_init(&pMgmt->syncSem, 0, 0);

1
source/dnode/mnode/impl/src/mndTelem.c
View File

@ -17,7 +17,6 @@
#include "mndTelem.h"
#include "mndCluster.h"
#include "mndSync.h"
#include "tbuffer.h"
#include "thttp.h"
#include "tjson.h"

4
source/dnode/vnode/src/meta/metaEntry.c
View File

@ -99,7 +99,9 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
}
if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1;
} else {
ASSERT(0);
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1;
}
tEndDecode(pCoder);

4
source/dnode/vnode/src/meta/metaSnapshot.c
View File

@ -189,7 +189,8 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
SDecoder* pDecoder = &(SDecoder){0};
tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
metaDecodeEntry(pDecoder, &metaEntry);
code = metaDecodeEntry(pDecoder, &metaEntry);
if (code) goto _err;
code = metaHandleEntry(pMeta, &metaEntry);
if (code) goto _err;
@ -198,6 +199,7 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
return code;
_err:
tDecoderClear(pDecoder);
metaError("vgId:%d, vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code));
return code;
}

14
source/dnode/vnode/src/tq/tq.c
View File

@ -1112,6 +1112,11 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
// do recovery step 1
streamSourceRecoverScanStep1(pTask);
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
return 0;
}
// build msg to launch next step
SStreamRecoverStep2Req req;
code = streamBuildSourceRecover2Req(pTask, &req);
@ -1122,6 +1127,10 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) {
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
return 0;
}
// serialize msg
int32_t len = sizeof(SStreamRecoverStep1Req);
@ -1160,6 +1169,11 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t m
return -1;
}
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
streamMetaReleaseTask(pTq->pStreamMeta, pTask);
return 0;
}
// restore param
code = streamRestoreParam(pTask);
if (code < 0) {

34
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -125,13 +125,17 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "vndStats.timeseries", pCfg->vndStats.numOfTimeSeries) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries) < 0) return -1;
SJson *pNodeInfoArr = tjsonCreateArray();
tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", pNodeInfoArr);
SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1;
for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) {
SJson *pNodeInfo = tjsonCreateObject();
tjsonAddIntegerToObject(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
tjsonAddStringToObject(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
tjsonAddItemToArray(pNodeInfoArr, pNodeInfo);
SJson *info = tjsonCreateObject();
if (info == NULL) return -1;
if (tjsonAddIntegerToObject(info, "nodePort", pCfg->syncCfg.nodeInfo[i].nodePort) < 0) return -1;
if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->syncCfg.nodeInfo[i].nodeFqdn) < 0) return -1;
if (tjsonAddIntegerToObject(info, "nodeId", pCfg->syncCfg.nodeInfo[i].nodeId) < 0) return -1;
if (tjsonAddIntegerToObject(info, "clusterId", pCfg->syncCfg.nodeInfo[i].clusterId) < 0) return -1;
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
}
return 0;
@ -240,15 +244,19 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
tjsonGetNumberValue(pJson, "vndStats.ntimeseries", pCfg->vndStats.numOfNTimeSeries, code);
if (code < 0) return -1;
SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(pNodeInfoArr);
assert(arraySize == pCfg->syncCfg.replicaNum);
SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
int arraySize = tjsonGetArraySize(nodeInfo);
if (arraySize != pCfg->syncCfg.replicaNum) return -1;
for (int i = 0; i < arraySize; ++i) {
SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i);
assert(pNodeInfo != NULL);
tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort, code);
tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
SJson *info = tjsonGetArrayItem(nodeInfo, i);
if (info == NULL) return -1;
tjsonGetNumberValue(info, "nodePort", pCfg->syncCfg.nodeInfo[i].nodePort, code);
if (code < 0) return -1;
tjsonGetStringValue(info, "nodeFqdn", pCfg->syncCfg.nodeInfo[i].nodeFqdn);
if (code < 0) return -1;
tjsonGetNumberValue(info, "nodeId", pCfg->syncCfg.nodeInfo[i].nodeId, code);
tjsonGetNumberValue(info, "clusterId", pCfg->syncCfg.nodeInfo[i].clusterId, code);
}
tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code);

2
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -82,8 +82,10 @@ int32_t vnodeAlter(const char *path, SAlterVnodeReplicaReq *pReq, STfs *pTfs) {
vInfo("vgId:%d, save config, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex);
for (int i = 0; i < pReq->replica; ++i) {
SNodeInfo *pNode = &pCfg->nodeInfo[i];
pNode->nodeId = pReq->replicas[i].id;
pNode->nodePort = pReq->replicas[i].port;
tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn));
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
vInfo("vgId:%d, save config, replica:%d ep:%s:%u", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort);
}

3
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -612,7 +612,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
vInfo("vgId:%d, start to open sync, replica:%d selfIndex:%d", pVnode->config.vgId, pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SNodeInfo *pNode = &pCfg->nodeInfo[i];
vInfo("vgId:%d, index:%d ep:%s:%u", pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort);
vInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId, pNode->clusterId);
}
pVnode->sync = syncOpen(&syncInfo);

1
source/libs/executor/inc/executil.h
View File

@ -18,7 +18,6 @@
#include "function.h"
#include "nodes.h"
#include "plannodes.h"
#include "tbuffer.h"
#include "tcommon.h"
#include "tpagedbuf.h"
#include "tsimplehash.h"

4
source/libs/executor/inc/executorimpl.h
View File

@ -693,7 +693,7 @@ void cleanupExprSupp(SExprSupp* pSup);
void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs);
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey);
const char* pkey, void* pState);
void cleanupAggSup(SAggSupporter* pAggSup);
void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
@ -852,6 +852,8 @@ int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResul
int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize);
void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order);
int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo);
int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos,
int32_t order, int64_t* pData);
#ifdef __cplusplus
}

40
source/libs/executor/src/executorimpl.c
View File

@ -83,6 +83,7 @@ typedef struct SAggOperatorInfo {
uint64_t groupId;
SGroupResInfo groupResInfo;
SExprSupp scalarExprSup;
bool groupKeyOptimized;
} SAggOperatorInfo;
static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
@ -604,9 +605,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB
}
}
bool isTaskKilled(SExecTaskInfo* pTaskInfo) {
return (0 != pTaskInfo->code) ? true : false;
}
bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code) ? true : false; }
void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; }
@ -1349,20 +1348,25 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan
}
}
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock **ppBlock) {
static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBlock** ppBlock) {
if (!tsCountAlwaysReturnValue) {
return TSDB_CODE_SUCCESS;
}
SAggOperatorInfo* pAggInfo = pOperator->info;
if (pAggInfo->groupKeyOptimized) {
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* downstream = pOperator->pDownstream[0];
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION ||
(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN &&
((STableScanInfo *)downstream->info)->hasGroupByTag == true)) {
((STableScanInfo*)downstream->info)->hasGroupByTag == true)) {
return TSDB_CODE_SUCCESS;
}
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
bool hasCountFunc = false;
bool hasCountFunc = false;
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
@ -1411,7 +1415,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
return TSDB_CODE_SUCCESS;
}
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock **ppBlock) {
static void destroyDataBlockForEmptyInput(bool blockAllocated, SSDataBlock** ppBlock) {
if (!blockAllocated) {
return;
}
@ -1437,8 +1441,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
int32_t order = TSDB_ORDER_ASC;
int32_t scanFlag = MAIN_SCAN;
bool hasValidBlock = false;
bool blockAllocated = false;
bool hasValidBlock = false;
bool blockAllocated = false;
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
@ -1481,7 +1485,6 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
}
destroyDataBlockForEmptyInput(blockAllocated, &pBlock);
}
// the downstream operator may return with error code, so let's check the code before generating results.
@ -1636,7 +1639,7 @@ void cleanupAggSup(SAggSupporter* pAggSup) {
}
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey) {
const char* pkey, void* pState) {
int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
if (code != TSDB_CODE_SUCCESS) {
return code;
@ -1648,7 +1651,13 @@ int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo
}
for (int32_t i = 0; i < numOfCols; ++i) {
pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
if (pState) {
pSup->pCtx[i].saveHandle.pBuf = NULL;
pSup->pCtx[i].saveHandle.pState = pState;
pSup->pCtx[i].exprIdx = i;
} else {
pSup->pCtx[i].saveHandle.pBuf = pAggSup->pResultBuf;
}
}
return TSDB_CODE_SUCCESS;
@ -1733,7 +1742,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1755,11 +1765,13 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
}
pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock;
pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized;
pInfo->groupId = UINT64_MAX;
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, destroyAggOperatorInfo,
optrDefaultBufFn, NULL);
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pTableScanInfo = downstream->info;

3
source/libs/executor/src/groupoperator.c
View File

@ -458,7 +458,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

8
source/libs/executor/src/projectoperator.c
View File

@ -102,7 +102,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
}
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -317,7 +318,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
if (pProjectInfo->mergeDataBlocks) {
if (pRes->info.rows > 0) {
pFinalRes->info.id.groupId = 0; //clear groupId
pFinalRes->info.id.groupId = 0; // clear groupId
pFinalRes->info.version = pRes->info.version;
// continue merge data, ignore the group id
@ -402,7 +403,8 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
blockDataEnsureCapacity(pResBlock, numOfRows);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

183
source/libs/executor/src/sysscanoperator.c
View File

@ -65,6 +65,8 @@ typedef struct SSysTableScanInfo {
SSDataBlock* pRes;
int64_t numOfBlocks; // extract basic running information.
SLoadRemoteDataInfo loadInfo;
int32_t tbnameSlotId;
} SSysTableScanInfo;
typedef struct {
@ -346,6 +348,11 @@ static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result);
static int32_t optSysCheckOper(SNode* pOpear);
static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt);
static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name,
SExecTaskInfo* pTaskInfo);
void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode);
static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo,
const char* name, SSDataBlock* pBlock);
__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) {
if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) {
*reverse = true;
@ -1309,83 +1316,111 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) {
getDBNameFromCondition(pInfo->pCondition, dbName);
sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName);
}
SSDataBlock* pBlock = NULL;
if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) {
return sysTableScanUserTables(pOperator);
pBlock = sysTableScanUserTables(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) {
return sysTableScanUserTags(pOperator);
pBlock = sysTableScanUserTags(pOperator);
} else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite &&
IS_SYS_DBNAME(dbName)) {
return sysTableScanUserSTables(pOperator);
pBlock = sysTableScanUserSTables(pOperator);
} else { // load the meta from mnode of the given epset
if (pOperator->status == OP_EXEC_DONE) {
pBlock = sysTableScanFromMNode(pOperator, pInfo, name, pTaskInfo);
}
return sysTableScanFillTbName(pOperator, pInfo, name, pBlock);
}
static SSDataBlock* sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo,
const char* name, SSDataBlock* pBlock) {
if (pBlock != NULL) {
if (pInfo->tbnameSlotId != -1) {
SColumnInfoData* pColumnInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, pInfo->tbnameSlotId);
char varTbName[TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE] = {0};
memcpy(varDataVal(varTbName), name, strlen(name));
varDataSetLen(varTbName, strlen(name));
for (int i = 0; i < pBlock->info.rows; ++i) {
colDataAppend(pColumnInfoData, i, varTbName, NULL);
}
doFilterResult(pBlock, pOperator->exprSupp.pFilterInfo);
}
}
if (pBlock && pBlock->info.rows != 0) {
return pBlock;
} else {
return NULL;
}
}
static SSDataBlock* sysTableScanFromMNode(SOperatorInfo* pOperator, SSysTableScanInfo* pInfo, const char* name,
SExecTaskInfo* pTaskInfo) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
while (1) {
int64_t startTs = taosGetTimestampUs();
tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user));
int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req);
char* buf1 = taosMemoryCalloc(1, contLen);
tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req);
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
while (1) {
int64_t startTs = taosGetTimestampUs();
tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb));
tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user));
int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE
: TDMT_MND_SYSTABLE_RETRIEVE;
int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req);
char* buf1 = taosMemoryCalloc(1, contLen);
tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req);
pMsgSendInfo->param = pOperator;
pMsgSendInfo->msgInfo.pData = buf1;
pMsgSendInfo->msgInfo.len = contLen;
pMsgSendInfo->msgType = msgType;
pMsgSendInfo->fp = loadSysTableCallback;
pMsgSendInfo->requestId = pTaskInfo->id.queryId;
// send the fetch remote task result reques
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo));
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
int64_t transporterId = 0;
int32_t code =
asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo);
tsem_wait(&pInfo->ready);
if (pTaskInfo->code) {
qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo),
pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code));
return NULL;
}
SRetrieveMetaTableRsp* pRsp = pInfo->pRsp;
pInfo->req.showId = pRsp->handle;
if (pRsp->numOfRows == 0 || pRsp->completed) {
pOperator->status = OP_EXEC_DONE;
qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo),
pRsp->numOfRows, pInfo->loadInfo.totalRows);
if (pRsp->numOfRows == 0) {
taosMemoryFree(pRsp);
return NULL;
}
}
int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE
: TDMT_MND_SYSTABLE_RETRIEVE;
char* pStart = pRsp->data;
extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart);
updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator);
pMsgSendInfo->param = pOperator;
pMsgSendInfo->msgInfo.pData = buf1;
pMsgSendInfo->msgInfo.len = contLen;
pMsgSendInfo->msgType = msgType;
pMsgSendInfo->fp = loadSysTableCallback;
pMsgSendInfo->requestId = pTaskInfo->id.queryId;
int64_t transporterId = 0;
int32_t code =
asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo);
tsem_wait(&pInfo->ready);
if (pTaskInfo->code) {
qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo),
pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code));
return NULL;
}
SRetrieveMetaTableRsp* pRsp = pInfo->pRsp;
pInfo->req.showId = pRsp->handle;
if (pRsp->numOfRows == 0 || pRsp->completed) {
pOperator->status = OP_EXEC_DONE;
qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo),
pRsp->numOfRows, pInfo->loadInfo.totalRows);
if (pRsp->numOfRows == 0) {
taosMemoryFree(pRsp);
return NULL;
}
}
char* pStart = pRsp->data;
extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart);
updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator);
// todo log the filter info
doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo);
taosMemoryFree(pRsp);
if (pInfo->pRes->info.rows > 0) {
return pInfo->pRes;
} else if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
// todo log the filter info
doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo);
taosMemoryFree(pRsp);
if (pInfo->pRes->info.rows > 0) {
return pInfo->pRes;
} else if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
}
}
@ -1407,6 +1442,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
goto _error;
}
extractTbnameSlotId(pInfo, pScanNode);
pInfo->accountId = pScanPhyNode->accountId;
pInfo->pUser = taosMemoryStrDup((void*)pUser);
pInfo->sysInfo = pScanPhyNode->sysInfo;
@ -1449,6 +1486,26 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
return NULL;
}
void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhysiNode* pScanNode) {
pInfo->tbnameSlotId = -1;
if (pScanNode->pScanPseudoCols != NULL) {
SNode* pNode = NULL;
FOREACH(pNode, pScanNode->pScanPseudoCols) {
STargetNode* pTargetNode = NULL;
if (nodeType(pNode) == QUERY_NODE_TARGET) {
pTargetNode = (STargetNode*)pNode;
SNode* expr = pTargetNode->pExpr;
if (nodeType(expr) == QUERY_NODE_FUNCTION) {
SFunctionNode* pFuncNode = (SFunctionNode*)expr;
if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
pInfo->tbnameSlotId = pTargetNode->slotId;
}
}
}
}
}
}
void destroySysScanOperator(void* param) {
SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param;
tsem_destroy(&pInfo->ready);

27
source/libs/executor/src/tfill.c
View File

@ -39,14 +39,10 @@ static void doSetVal(SColumnInfoData* pDstColInfoData, int32_t rowIndex, const S
static void setNotFillColumn(SFillInfo* pFillInfo, SColumnInfoData* pDstColInfo, int32_t rowIndex, int32_t colIdx) {
SRowVal* p = NULL;
if (FILL_IS_ASC_FILL(pFillInfo)) {
if (pFillInfo->prev.key != 0) {
p = &pFillInfo->prev; // prev has been set value
} else { // otherwise, use the value in the next row
p = &pFillInfo->next;
}
if (pFillInfo->type == TSDB_FILL_NEXT) {
p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->next : &pFillInfo->prev;
} else {
p = &pFillInfo->next;
p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next;
}
SGroupKeys* pKey = taosArrayGet(p->pRowVal, colIdx);
@ -257,9 +253,16 @@ static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, int32_t rowIndex, SRowVa
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
int32_t type = pFillInfo->pFillCol[i].pExpr->pExpr->nodeType;
if (type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) {
if ( type == QUERY_NODE_COLUMN || type == QUERY_NODE_OPERATOR || type == QUERY_NODE_FUNCTION) {
if (!pFillInfo->pFillCol[i].notFillCol && pFillInfo->type != TSDB_FILL_NEXT) {
continue;
}
int32_t srcSlotId = GET_DEST_SLOT_ID(&pFillInfo->pFillCol[i]);
if (srcSlotId == pFillInfo->srcTsSlotId && pFillInfo->type == TSDB_FILL_LINEAR) {
continue;
}
SColumnInfoData* pSrcCol = taosArrayGet(pFillInfo->pSrcBlock->pDataBlock, srcSlotId);
bool isNull = colDataIsNull_s(pSrcCol, rowIndex);
@ -288,8 +291,12 @@ static int32_t fillResultImpl(SFillInfo* pFillInfo, SSDataBlock* pBlock, int32_t
int64_t ts = ((int64_t*)pTsCol->pData)[pFillInfo->index];
// set the next value for interpolation
if ((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) {
copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, &pFillInfo->next);
if (pFillInfo->currentKey < ts && ascFill) {
SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->next : &pFillInfo->prev;
copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal);
} else if (pFillInfo->currentKey > ts && !ascFill) {
SRowVal* pRVal = pFillInfo->type == TSDB_FILL_NEXT ? &pFillInfo->prev : &pFillInfo->next;
copyCurrentRowIntoBuf(pFillInfo, pFillInfo->index, pRVal);
}
if (((pFillInfo->currentKey < ts && ascFill) || (pFillInfo->currentKey > ts && !ascFill)) &&

122
source/libs/executor/src/timewindowoperator.c
View File

@ -22,7 +22,7 @@
#include "tfill.h"
#include "ttime.h"
#define IS_FINAL_OP(op) ((op)->isFinal)
#define IS_FINAL_OP(op) ((op)->isFinal)
#define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL);
typedef struct SSessionAggOperatorInfo {
@ -119,8 +119,8 @@ static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsL
pRowSup->groupId = groupId;
}
static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
int16_t pos, int16_t order, int64_t* pData) {
FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey,
int32_t pos, int32_t order, int64_t* pData) {
int32_t forwardRows = 0;
if (order == TSDB_ORDER_ASC) {
@ -666,8 +666,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num
setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP);
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &w, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0, pBlock->info.rows,
numOfExprs);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, 0,
pBlock->info.rows, numOfExprs);
if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) {
closeResultRow(pr);
@ -817,13 +817,13 @@ static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) {
} else {
int32_t code = comparePullWinKey(pPullInfo, pPullWins, index);
if (code == 0) {
SPullWindowInfo* pos = taosArrayGet(pPullWins ,index);
SPullWindowInfo* pos = taosArrayGet(pPullWins, index);
pos->window.skey = TMIN(pos->window.skey, pPullInfo->window.skey);
pos->window.ekey = TMAX(pos->window.ekey, pPullInfo->window.ekey);
pos->calWin.skey = TMIN(pos->calWin.skey, pPullInfo->calWin.skey);
pos->calWin.ekey = TMAX(pos->calWin.ekey, pPullInfo->calWin.ekey);
return TSDB_CODE_SUCCESS;
} else if (code > 0 ){
} else if (code > 0) {
index++;
}
}
@ -879,10 +879,10 @@ int32_t compareWinRes(void* pKey, void* data, int32_t index) {
} else if (pRKey->groupId < pDataPos->groupId) {
return -1;
}
if (*(int64_t*)pRKey->key > pDataPos->ts) {
return 1;
} else if (*(int64_t*)pRKey->key < pDataPos->ts){
} else if (*(int64_t*)pRKey->key < pDataPos->ts) {
return -1;
}
return 0;
@ -961,8 +961,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, pInfo, pResult);
@ -996,8 +996,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
}
#endif
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, pInfo, pResult);
}
@ -1164,7 +1164,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
// here we start a new session window
doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
@ -1188,8 +1188,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
pBlock->info.rows, numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
}
static int32_t openStateWindowAggOptr(SOperatorInfo* pOperator) {
@ -1394,8 +1394,8 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) {
size_t keyLen = 0;
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
void* key = tSimpleHashGetKey(pIte, &keyLen);
uint64_t groupId = *(uint64_t*)key;
void* key = tSimpleHashGetKey(pIte, &keyLen);
uint64_t groupId = *(uint64_t*)key;
TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t));
SResultRowPosition* pPos = (SResultRowPosition*)pIte;
int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, resWins);
@ -1656,7 +1656,7 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt
// the primary timestamp column
bool needed = false;
for(int32_t i = 0; i < numOfCols; ++i) {
for (int32_t i = 0; i < numOfCols; ++i) {
SExprInfo* pExpr = pCtx[i].pExpr;
if (fmIsIntervalInterpoFunc(pCtx[i].functionId)) {
needed = true;
@ -1724,7 +1724,7 @@ void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSuppor
void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) {
for (int32_t i = 0; i < numOfExpr; i++) {
// pCtx[i].isStream = true;
// pCtx[i].isStream = true;
}
}
@ -1748,7 +1748,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
int32_t code =
initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -1800,8 +1801,8 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPh
setOperatorInfo(pOperator, "TimeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, destroyIntervalOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -1869,7 +1870,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
// pInfo->numOfRows data belong to the current session window
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
// here we start a new session window
doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
@ -1886,8 +1887,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
pBlock->info.rows, numOfOutput);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
}
static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
@ -2000,7 +2001,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -2018,8 +2020,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi
setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(openStateWindowAggOptr, doStateWindowAgg, NULL, destroyStateWindowOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -2068,7 +2070,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -2091,8 +2094,8 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionW
setOperatorInfo(pOperator, "SessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doSessionWindowAgg, NULL, destroySWindowOperatorInfo,
optrDefaultBufFn, NULL);
pOperator->pTaskInfo = pTaskInfo;
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -2309,7 +2312,8 @@ static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) {
if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) {
void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey));
if (!chIds) {
SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
SPullWindowInfo pull = {
.window = nextWin, .groupId = winKey->groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
// add pull data request
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
int32_t size1 = taosArrayGetSize(pInfo->pChildren);
@ -2395,7 +2399,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
};
void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey));
if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) {
SPullWindowInfo pull = {.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
SPullWindowInfo pull = {
.window = nextWin, .groupId = groupId, .calWin.skey = nextWin.skey, .calWin.ekey = nextWin.skey};
// add pull data request
if (savePullWindow(&pull, pInfo->pPullWins) == TSDB_CODE_SUCCESS) {
int32_t size = taosArrayGetSize(pInfo->pChildren);
@ -2450,7 +2455,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p
}
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows,
pSDataBlock->info.rows, numOfOutput);
pSDataBlock->info.rows, numOfOutput);
SWinKey key = {
.ts = nextWin.skey,
.groupId = groupId,
@ -2545,7 +2550,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
pOperator->status = OP_RES_TO_RETURN;
qDebug("===stream===return data:%s. recv datablock num:%" PRIu64 , IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
qDebug("===stream===return data:%s. recv datablock num:%" PRIu64,
IS_FINAL_OP(pInfo) ? "interval final" : "interval semi", pInfo->numOfDatapack);
pInfo->numOfDatapack = 0;
break;
}
@ -2665,7 +2671,7 @@ int64_t getDeleteMark(SIntervalPhysiNode* pIntervalPhyNode) {
if (pIntervalPhyNode->window.deleteMark <= 0) {
return DEAULT_DELETE_MARK;
}
int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark,pIntervalPhyNode->window.watermark);
int64_t deleteMark = TMAX(pIntervalPhyNode->window.deleteMark, pIntervalPhyNode->window.watermark);
deleteMark = TMAX(deleteMark, pIntervalPhyNode->interval);
return deleteMark;
}
@ -2713,7 +2719,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -2778,8 +2785,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->fpSet =
createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, destroyStreamFinalIntervalOperatorInfo,
optrDefaultBufFn, NULL);
if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) {
initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup);
}
@ -3578,8 +3585,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true,
OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamSessionAgg, NULL, destroyStreamSessionAggOperatorInfo,
optrDefaultBufFn, NULL);
if (downstream) {
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
@ -4083,8 +4090,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED,
pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamStateAgg, NULL, destroyStreamStateOperatorInfo,
optrDefaultBufFn, NULL);
initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -4169,8 +4176,8 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
}
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
pBlock->info.rows, pSup->numOfExprs);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos,
currPos - startPos, pBlock->info.rows, pSup->numOfExprs);
finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo);
resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow));
@ -4190,7 +4197,7 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos,
pBlock->info.rows, pSup->numOfExprs);
pBlock->info.rows, pSup->numOfExprs);
}
static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) {
@ -4341,7 +4348,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num);
code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -4360,8 +4368,8 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream,
setOperatorInfo(pOperator, "TimeMergeAlignedIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL,
false, OP_NOT_OPENED, miaInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, mergeAlignedIntervalAgg, NULL, destroyMAIOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -4496,7 +4504,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo*
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, iaInfo, pResult);
// output previous interval results after this interval (&win) is closed
@ -4528,7 +4536,7 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo*
updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true);
applyAggFunctionOnPartialTuples(pTaskInfo, pExprSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows,
pBlock->info.rows, numOfOutput);
pBlock->info.rows, numOfOutput);
doCloseWindow(pResultRowInfo, iaInfo, pResult);
// output previous interval results after this interval (&nextWin) is closed
@ -4645,7 +4653,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(&pOperator->resultInfo, 4096);
int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(pExprSupp, &pIntervalInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -4665,8 +4674,8 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo);
setOperatorInfo(pOperator, "TimeMergeIntervalAggOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, false,
OP_NOT_OPENED, pMergeIntervalInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo, optrDefaultBufFn, NULL);
pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doMergeIntervalAgg, NULL, destroyMergeIntervalOperatorInfo,
optrDefaultBufFn, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {
@ -4843,7 +4852,8 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys
initResultSizeInfo(&pOperator->resultInfo, 4096);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}

13
source/libs/executor/test/executorTests.cpp
View File

@ -43,6 +43,19 @@ enum {
data_desc = 0x3,
};
TEST(testCase, windowFunctionTest) {
int64_t tsCol[100000];
int32_t rows = 100000;
for (int32_t i = 0; i < rows; i++) {
tsCol[i] = 1648791213000 + i;
}
int32_t ekeyNum = 50000;
int32_t pos = 40000;
int64_t ekey = tsCol[ekeyNum];
int32_t num = getForwardStepsInBlock(rows, binarySearchForKey, ekey, pos, TSDB_ORDER_ASC, tsCol);
ASSERT_EQ(num, ekeyNum - pos + 1);
}
typedef struct SDummyInputInfo {
int32_t totalPages; // numOfPages
int32_t current;

4
source/libs/function/inc/builtinsimpl.h
View File

@ -32,7 +32,7 @@ typedef struct SSumRes {
int16_t type;
int64_t prevTs;
bool isPrevTsSet;
bool overflow; // if overflow is true, dsum to be used for any type;
bool overflow; // if overflow is true, dsum to be used for any type;
} SSumRes;
typedef struct SMinmaxResInfo {
@ -46,7 +46,7 @@ typedef struct SMinmaxResInfo {
} SMinmaxResInfo;
int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc);
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey);
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock);
int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos);

137
source/libs/function/src/builtinsimpl.c
View File

@ -41,15 +41,15 @@
#define HLL_BUCKET_MASK (HLL_BUCKETS - 1)
#define HLL_ALPHA_INF 0.721347520444481703680 // constant for 0.5/ln(2)
//typedef struct SMinmaxResInfo {
// bool assign; // assign the first value or not
// int64_t v;
// STuplePos tuplePos;
// typedef struct SMinmaxResInfo {
// bool assign; // assign the first value or not
// int64_t v;
// STuplePos tuplePos;
//
// STuplePos nullTuplePos;
// bool nullTupleSaved;
// int16_t type;
//} SMinmaxResInfo;
// STuplePos nullTuplePos;
// bool nullTupleSaved;
// int16_t type;
// } SMinmaxResInfo;
typedef struct STopBotResItem {
SVariant v;
@ -540,7 +540,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) {
if (IS_NULL_TYPE(type)) {
// select count(NULL) returns 0
numOfElem = 1;
*((int64_t*)buf) = 0;
*((int64_t*)buf) += 0;
} else {
numOfElem = getNumOfElems(pCtx);
*((int64_t*)buf) += numOfElem;
@ -818,28 +818,31 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple
return;
}
if (pCtx->saveHandle.pBuf != NULL) {
if (pTuplePos->pageId != -1) {
int32_t numOfCols = pCtx->subsidiaries.num;
const char* p = loadTupleData(pCtx, pTuplePos);
if ((pCtx->saveHandle.pBuf != NULL && pTuplePos->pageId != -1) ||
(pCtx->saveHandle.pState && pTuplePos->streamTupleKey.ts > 0)) {
int32_t numOfCols = pCtx->subsidiaries.num;
const char* p = loadTupleData(pCtx, pTuplePos);
bool* nullList = (bool*)p;
char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
bool* nullList = (bool*)p;
char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
// todo set the offset value to optimize the performance.
for (int32_t j = 0; j < numOfCols; ++j) {
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
// todo set the offset value to optimize the performance.
for (int32_t j = 0; j < numOfCols; ++j) {
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes);
if (nullList[j]) {
colDataAppendNULL(pDstCol, rowIndex);
} else {
colDataAppend(pDstCol, rowIndex, pStart, false);
}
pStart += pDstCol->info.bytes;
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes);
if (nullList[j]) {
colDataAppendNULL(pDstCol, rowIndex);
} else {
colDataAppend(pDstCol, rowIndex, pStart, false);
}
pStart += pDstCol->info.bytes;
}
if (pCtx->saveHandle.pState) {
tdbFree((void*)p);
}
}
}
@ -2018,7 +2021,7 @@ static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowInde
}
if (!pInfo->hasResult) {
pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL);
pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock);
} else {
updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos);
}
@ -2071,8 +2074,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
// we will use this opt implementation in an new version that is only available in scan subplan
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first
// function. we will use this opt implementation in an new version that is only available in scan subplan
#if 0
if (blockDataOrder == TSDB_ORDER_ASC) {
// filter according to current result firstly
@ -2179,7 +2182,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function.
// please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first
// function.
#if 0
if (blockDataOrder == TSDB_ORDER_ASC) {
for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
@ -2236,9 +2240,9 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
int32_t round = pInput->numOfRows >> 2;
int32_t reminder = pInput->numOfRows & 0x03;
for (int32_t i = pInput->startRowIndex, tick = 0; tick < round; i += 4, tick += 1) {
int64_t cts = pts[i];
int32_t chosen = i;
for (int32_t i = pInput->startRowIndex, tick = 0; tick < round; i += 4, tick += 1) {
int64_t cts = pts[i];
int32_t chosen = i;
if (cts < pts[i + 1]) {
cts = pts[i + 1];
@ -2262,18 +2266,18 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
}
}
for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) {
char* data = colDataGetData(pInputCol, i);
doSaveCurrentVal(pCtx, i, pts[i], type, data);
pResInfo->numOfRes = 1;
}
for (int32_t i = pInput->startRowIndex + round * 4; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (pResInfo->numOfRes == 0 || pInfo->ts < pts[i]) {
char* data = colDataGetData(pInputCol, i);
doSaveCurrentVal(pCtx, i, pts[i], type, data);
pResInfo->numOfRes = 1;
}
}
} else {
for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
continue;
}
} else {
for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
continue;
}
numOfElems++;
@ -2439,7 +2443,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
int32_t type = pInputCol->info.type;
int32_t type = pInputCol->info.type;
int32_t bytes = pInputCol->info.bytes;
pInfo->bytes = bytes;
@ -2777,7 +2781,7 @@ int32_t topFunction(SqlFunctionCtx* pCtx) {
}
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) {
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pRes->nullTupleSaved = true;
}
return TSDB_CODE_SUCCESS;
@ -2805,7 +2809,7 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) {
}
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) {
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pRes->nullTupleSaved = true;
}
@ -2863,7 +2867,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
// save the data of this tuple
if (pCtx->subsidiaries.num > 0) {
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL);
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock);
}
#ifdef BUF_PAGE_DEBUG
qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId,
@ -2937,8 +2941,7 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid
return buf;
}
static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length,
const STupleKey* pKey) {
static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STupleKey key) {
STuplePos p = {0};
if (pHandle->pBuf != NULL) {
SFilePage* pPage = NULL;
@ -2964,20 +2967,31 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf
releaseBufPage(pHandle->pBuf, pPage);
} else {
// other tuple save policy
if (streamStateFuncPut(pHandle->pState, pKey, pBuf, length) < 0) {
if (streamStateFuncPut(pHandle->pState, &key, pBuf, length) < 0) {
ASSERT(0);
}
p.streamTupleKey = *pKey;
p.streamTupleKey = key;
}
return p;
}
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey) {
STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock) {
prepareBuf(pCtx);
STupleKey key;
if (pCtx->saveHandle.pBuf == NULL) {
SColumnInfoData* pColInfo = taosArrayGet(pSrcBlock->pDataBlock, 0);
ASSERT(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
int64_t skey = *(int64_t*)colDataGetData(pColInfo, rowIndex);
key.groupId = pSrcBlock->info.id.groupId;
key.ts = skey;
key.exprIdx = pCtx->exprIdx;
}
char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf);
return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pKey);
return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, key);
}
static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) {
@ -3623,7 +3637,7 @@ bool histogramFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultIn
pInfo->totalCount = 0;
pInfo->normalized = 0;
char *binTypeStr = strndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz));
char* binTypeStr = strndup(varDataVal(pCtx->param[1].param.pz), varDataLen(pCtx->param[1].param.pz));
int8_t binType = getHistogramBinType(binTypeStr);
taosMemoryFree(binTypeStr);
@ -3947,7 +3961,6 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
char* data = colDataGetData(pCol, i);
SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data);
@ -4423,7 +4436,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da
if (pInfo->numSampled < pInfo->samples) {
sampleAssignResult(pInfo, data, pInfo->numSampled);
if (pCtx->subsidiaries.num > 0) {
pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
pInfo->numSampled++;
} else {
@ -4454,7 +4467,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) {
}
if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) {
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pInfo->nullTupleSaved = true;
}
@ -4758,7 +4771,7 @@ static void doModeAdd(SModeInfo* pInfo, int32_t rowIndex, SqlFunctionCtx* pCtx,
pItem->count += 1;
if (pCtx->subsidiaries.num > 0) {
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock, NULL);
pItem->tuplePos = saveTupleData(pCtx, rowIndex, pCtx->pSrcBlock);
}
taosHashPut(pInfo->pHash, data, hashKeyBytes, &pItem, sizeof(SModeItem*));
@ -4798,7 +4811,7 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) {
}
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) {
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pInfo->nullTupleSaved = true;
}
@ -5299,7 +5312,7 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t len = sprintf(st + VARSTR_HEADER_SIZE,
"Total_Blocks=[%d] Total_Size=[%.2f Kb] Average_size=[%.2f Kb] Compression_Ratio=[%.2f %c]",
pData->numOfBlocks, pData->totalSize / 1024.0, averageSize/1024.0, compRatio, '%');
pData->numOfBlocks, pData->totalSize / 1024.0, averageSize / 1024.0, compRatio, '%');
varDataSetLen(st, len);
colDataAppend(pColInfo, row++, st, false);

37
source/libs/function/src/detail/tminmax.c
View File

@ -64,7 +64,7 @@
static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder, int32_t* rounds, int32_t* width) {
const int32_t bitWidth = 256;
*width = (bitWidth>>3u) / bytes;
*width = (bitWidth >> 3u) / bytes;
*remainder = numOfRows % (*width);
*rounds = numOfRows / (*width);
}
@ -92,8 +92,7 @@ static void calculateRounds(int32_t numOfRows, int32_t bytes, int32_t* remainder
(_v) = (_sec)[j]; \
} \
}
static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFunc, bool signVal) {
int8_t v = 0;
const int8_t* p = pData;
@ -116,7 +115,7 @@ static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFu
const int8_t* q = (const int8_t*)&initVal;
EXTRACT_MAX_VAL(q, p, width, remain, v)
} else { // unsigned value
} else { // unsigned value
for (int32_t i = 0; i < rounds; ++i) {
next = _mm256_lddqu_si256((__m256i*)p);
initVal = _mm256_max_epu8(initVal, next);
@ -126,7 +125,7 @@ static int8_t i8VectorCmpAVX2(const void* pData, int32_t numOfRows, bool isMinFu
const uint8_t* q = (const uint8_t*)&initVal;
EXTRACT_MAX_VAL(q, p, width, remain, v)
}
} else { // min function
if (signVal) {
for (int32_t i = 0; i < rounds; ++i) {
@ -241,7 +240,7 @@ static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool is
// let compare the final results
const int32_t* q = (const int32_t*)&initVal;
EXTRACT_MAX_VAL(q, p, width, remain, v)
} else { // unsigned value
} else { // unsigned value
for (int32_t i = 0; i < rounds; ++i) {
next = _mm256_lddqu_si256((__m256i*)p);
initVal = _mm256_max_epi32(initVal, next);
@ -281,7 +280,7 @@ static int32_t i32VectorCmpAVX2(const int32_t* pData, int32_t numOfRows, bool is
}
static float floatVectorCmpAVX(const float* pData, int32_t numOfRows, bool isMinFunc) {
float v = 0;
float v = 0;
const float* p = pData;
int32_t width, remain, rounds;
@ -358,7 +357,7 @@ static double doubleVectorCmpAVX(const double* pData, int32_t numOfRows, bool is
static int32_t findFirstValPosition(const SColumnInfoData* pCol, int32_t start, int32_t numOfRows) {
int32_t i = start;
while (i < (start + numOfRows) && (colDataIsNull_f(pCol->nullbitmap, i) == true)) {
i += 1;
}
@ -495,7 +494,8 @@ static void handleInt64Col(const void* data, int32_t start, int32_t numOfRows, S
}
}
static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) {
static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf,
bool isMinFunc) {
float* pData = (float*)pCol->pData;
float* val = (float*)&pBuf->v;
@ -525,7 +525,8 @@ static void handleFloatCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRo
pBuf->assign = true;
}
static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf, bool isMinFunc) {
static void handleDoubleCol(SColumnInfoData* pCol, int32_t start, int32_t numOfRows, SMinmaxResInfo* pBuf,
bool isMinFunc) {
double* pData = (double*)pCol->pData;
double* val = (double*)&pBuf->v;
@ -740,7 +741,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
} else {
@ -754,7 +755,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -768,7 +769,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -782,7 +783,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -798,7 +799,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
if (pCtx->subsidiaries.num > 0) {
index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
if (index >= 0) {
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock);
}
}
}
@ -819,7 +820,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
memcpy(&pBuf->v, pCol->pData + (pCol->info.bytes * i), pCol->info.bytes);
if (pCtx->subsidiaries.num > 0) {
pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock);
}
pBuf->assign = true;
numOfElems = 1;
@ -883,9 +884,9 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
_over:
if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) {
pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock);
pBuf->nullTupleSaved = true;
}
return numOfElems;
}
}

1
source/libs/function/src/tfunctionInt.c
View File

@ -20,7 +20,6 @@
#include "ttypes.h"
#include "function.h"
#include "tbuffer.h"
#include "tcompression.h"
#include "tdatablock.h"
#include "tfunctionInt.h"

1
source/libs/function/src/tudf.c
View File

@ -1230,7 +1230,6 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *
if (uvTask->rspBuf.base != NULL) {
SUdfResponse rsp = {0};
void *buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp);
assert(uvTask->rspBuf.len == POINTER_DISTANCE(buf, uvTask->rspBuf.base));
task->errCode = rsp.code;
switch (task->type) {

1
source/libs/function/src/udfd.c
View File

@ -400,7 +400,6 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle;
ASSERT(pMsg->info.ahandle != NULL);
if (pEpSet) {
if (!isEpsetEqual(&global.mgmtEp.epSet, pEpSet)) {

1
source/libs/index/inc/indexUtil.h
View File

@ -36,7 +36,6 @@ extern "C" {
#define SERIALIZE_VAR_TO_BUF(buf, var, type) \
do { \
type c = var; \
assert(sizeof(type) == sizeof(c)); \
memcpy((void *)buf, (void *)&c, sizeof(c)); \
buf += sizeof(c); \
} while (0)

4
source/libs/index/src/index.c
View File

@ -226,7 +226,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
indexDebug("w suid:%" PRIu64 ", colName:%s, colType:%d", key.suid, key.colName, key.colType);
IndexCache** cache = taosHashGet(index->colObj, buf, sz);
assert(*cache != NULL);
ASSERTS(*cache != NULL, "index-cache already release");
if (*cache == NULL) return -1;
int ret = idxCachePut(*cache, p, uid);
if (ret != 0) {
return ret;

5
source/libs/index/src/indexComm.c
View File

@ -170,7 +170,6 @@ TExeCond tCompare(__compar_fn_t func, int8_t cmptype, void* a, void* b, int8_t d
}
return tDoCompare(func, cmptype, &va, &vb);
}
assert(0);
return BREAK;
#endif
}
@ -367,7 +366,7 @@ int32_t idxConvertData(void* src, int8_t type, void** dst) {
tlen = taosEncodeBinary(dst, src, strlen(src));
break;
default:
ASSERT(0);
ASSERTS(0, "index invalid input type");
break;
}
*dst = (char*)*dst - tlen;
@ -459,7 +458,7 @@ int32_t idxConvertDataToStr(void* src, int8_t type, void** dst) {
*dst = (char*)*dst - tlen;
break;
default:
ASSERT(0);
ASSERTS(0, "index invalid input type");
break;
}
return tlen;

4
source/libs/index/src/indexFilter.c
View File

@ -206,7 +206,9 @@ static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) {
static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) {
SOperatorNode *nd = (SOperatorNode *)node;
assert(nodeType(node) == QUERY_NODE_OPERATOR);
if (nodeType(node) != QUERY_NODE_OPERATOR) {
return -1;
}
SColumnNode *l = (SColumnNode *)nd->pLeft;
SValueNode *r = (SValueNode *)nd->pRight;

60
source/libs/index/src/indexFst.c
View File

@ -65,10 +65,7 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes* nodes, bool isFinal) {
taosArrayPush(nodes->stack, &un);
}
FstBuilderNode* fstUnFinishedNodesPopRoot(FstUnFinishedNodes* nodes) {
assert(taosArrayGetSize(nodes->stack) == 1);
FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
assert(un->last == NULL);
return un->node;
}
@ -82,7 +79,6 @@ FstBuilderNode* fstUnFinishedNodesPopFreeze(FstUnFinishedNodes* nodes, CompiledA
FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes) {
FstBuilderNodeUnfinished* un = taosArrayPop(nodes->stack);
assert(un->last == NULL);
return un->node;
}
void fstUnFinishedNodesSetRootOutput(FstUnFinishedNodes* nodes, Output out) {
@ -102,7 +98,8 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output
}
int32_t sz = taosArrayGetSize(nodes->stack) - 1;
FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz);
assert(un->last == NULL);
ASSERTS(un->last == NULL, "index-fst meet unexpected node");
if (un->last != NULL) return;
// FstLastTransition *trn = taosMemoryMalloc(sizeof(FstLastTransition));
// trn->inp = s->data[s->start];
@ -247,7 +244,6 @@ void fstStateCompileForOneTrans(IdxFstFile* w, CompiledAddr addr, FstTransition*
}
void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode* node) {
int32_t sz = taosArrayGetSize(node->trans);
assert(sz <= 256);
uint8_t tSize = 0;
uint8_t oSize = packSize(node->finalOutput);
@ -322,7 +318,7 @@ void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode
// set_comm_input
void fstStateSetCommInput(FstState* s, uint8_t inp) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
uint8_t val;
COMMON_INDEX(inp, 0b111111, val);
@ -331,7 +327,7 @@ void fstStateSetCommInput(FstState* s, uint8_t inp) {
// comm_input
uint8_t fstStateCommInput(FstState* s, bool* null) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
uint8_t v = s->val & 0b00111111;
if (v == 0) {
*null = true;
@ -344,7 +340,7 @@ uint8_t fstStateCommInput(FstState* s, bool* null) {
// input_len
uint64_t fstStateInputLen(FstState* s) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
bool null = false;
fstStateCommInput(s, &null);
return null ? 1 : 0;
@ -352,11 +348,11 @@ uint64_t fstStateInputLen(FstState* s) {
// end_addr
uint64_t fstStateEndAddrForOneTransNext(FstState* s, FstSlice* data) {
assert(s->state == OneTransNext);
ASSERT(s->state == OneTransNext);
return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s);
}
uint64_t fstStateEndAddrForOneTrans(FstState* s, FstSlice* data, PackSizes sizes) {
assert(s->state == OneTrans);
ASSERT(s->state == OneTrans);
return FST_SLICE_LEN(data) - 1 - fstStateInputLen(s) - 1 // pack size
- FST_GET_TRANSITION_PACK_SIZE(sizes) - FST_GET_OUTPUT_PACK_SIZE(sizes);
}
@ -370,7 +366,7 @@ uint64_t fstStateEndAddrForAnyTrans(FstState* state, uint64_t version, FstSlice*
}
// input
uint8_t fstStateInput(FstState* s, FstNode* node) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
FstSlice* slice = &node->data;
bool null = false;
uint8_t inp = fstStateCommInput(s, &null);
@ -378,7 +374,7 @@ uint8_t fstStateInput(FstState* s, FstNode* node) {
return null == false ? inp : data[node->start - 1];
}
uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
FstSlice* slice = &node->data;
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
@ -390,7 +386,7 @@ uint8_t fstStateInputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
// trans_addr
CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) {
assert(s->state == OneTransNext || s->state == OneTrans);
ASSERT(s->state == OneTransNext || s->state == OneTrans);
FstSlice* slice = &node->data;
if (s->state == OneTransNext) {
return (CompiledAddr)(node->end) - 1;
@ -406,7 +402,7 @@ CompiledAddr fstStateTransAddr(FstState* s, FstNode* node) {
}
}
CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
FstSlice* slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
@ -418,7 +414,7 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState* s, FstNode* node, uint64_t i
// sizes
PackSizes fstStateSizes(FstState* s, FstSlice* slice) {
assert(s->state == OneTrans || s->state == AnyTrans);
ASSERT(s->state == OneTrans || s->state == AnyTrans);
uint64_t i;
if (s->state == OneTrans) {
i = FST_SLICE_LEN(slice) - 1 - fstStateInputLen(s) - 1;
@ -431,7 +427,7 @@ PackSizes fstStateSizes(FstState* s, FstSlice* slice) {
}
// Output
Output fstStateOutput(FstState* s, FstNode* node) {
assert(s->state == OneTrans);
ASSERT(s->state == OneTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) {
@ -445,7 +441,7 @@ Output fstStateOutput(FstState* s, FstNode* node) {
return unpackUint64(data + i, oSizes);
}
Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) {
@ -462,19 +458,19 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
// anyTrans specify function
void fstStateSetFinalState(FstState* s, bool yes) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
if (yes) {
s->val |= 0b01000000;
}
return;
}
bool fstStateIsFinalState(FstState* s) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
return (s->val & 0b01000000) == 0b01000000;
}
void fstStateSetStateNtrans(FstState* s, uint8_t n) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
if (n <= 0b00111111) {
s->val = (s->val & 0b11000000) | n;
}
@ -482,7 +478,7 @@ void fstStateSetStateNtrans(FstState* s, uint8_t n) {
}
// state_ntrans
uint8_t fstStateStateNtrans(FstState* s, bool* null) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
*null = false;
uint8_t n = s->val & 0b00111111;
@ -492,16 +488,16 @@ uint8_t fstStateStateNtrans(FstState* s, bool* null) {
return n;
}
uint64_t fstStateTotalTransSize(FstState* s, uint64_t version, PackSizes sizes, uint64_t nTrans) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
uint64_t idxSize = fstStateTransIndexSize(s, version, nTrans);
return nTrans + (nTrans * FST_GET_TRANSITION_PACK_SIZE(sizes)) + idxSize;
}
uint64_t fstStateTransIndexSize(FstState* s, uint64_t version, uint64_t nTrans) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
return (version >= 2 && nTrans > TRANS_INDEX_THRESHOLD) ? 256 : 0;
}
uint64_t fstStateNtransLen(FstState* s) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
bool null = false;
fstStateStateNtrans(s, &null);
return null == true ? 1 : 0;
@ -530,7 +526,7 @@ Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackS
return unpackUint64(data + at, (uint8_t)oSizes);
}
uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
assert(s->state == AnyTrans);
ASSERT(s->state == AnyTrans);
FstSlice* slice = &node->data;
if (node->version >= 2 && node->nTrans > TRANS_INDEX_THRESHOLD) {
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
@ -676,17 +672,17 @@ bool fstNodeGetTransitionAddrAt(FstNode* node, uint64_t i, CompiledAddr* res) {
bool s = true;
FstState* st = &node->state;
if (st->state == OneTransNext) {
assert(i == 0);
ASSERT(i == 0);
fstStateTransAddr(st, node);
} else if (st->state == OneTrans) {
assert(i == 0);
ASSERT(i == 0);
fstStateTransAddr(st, node);
} else if (st->state == AnyTrans) {
fstStateTransAddrForAnyTrans(st, node, i);
} else if (FST_STATE_EMPTY_FINAL(node)) {
s = false;
} else {
assert(0);
ASSERT(0);
}
return s;
}
@ -722,7 +718,7 @@ bool fstNodeFindInput(FstNode* node, uint8_t b, uint64_t* res) {
bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr addr, FstBuilderNode* builderNode) {
int32_t sz = taosArrayGetSize(builderNode->trans);
assert(sz < 256);
ASSERT(sz < 256);
if (sz == 0 && builderNode->isFinal && builderNode->finalOutput == 0) {
return true;
} else if (sz != 1 || builderNode->isFinal) {
@ -804,7 +800,7 @@ void fstBuilderInsertOutput(FstBuilder* b, FstSlice bs, Output in) {
uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out);
if (prefixLen == FST_SLICE_LEN(s)) {
assert(out == 0);
ASSERT(out == 0);
return;
}
@ -848,7 +844,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) {
addr = fstBuilderCompile(b, bn);
fstBuilderNodeDestroy(bn);
assert(addr != NONE_ADDRESS);
ASSERT(addr != NONE_ADDRESS);
}
fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
return;

5
source/libs/index/src/indexFstDfa.c
View File

@ -104,8 +104,9 @@ bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet
DfaState *t = taosArrayGet(builder->dfa->states, state);
for (int i = 0; i < taosArrayGetSize(t->insts); i++) {
int32_t ip = *(int32_t *)taosArrayGet(t->insts, i);
bool succ = sparSetAdd(cur, ip, NULL);
assert(succ == true);
bool succ = sparSetAdd(cur, ip, NULL);
if (succ == false) return false;
}
dfaRun(builder->dfa, cur, next, byte);

14
source/libs/index/src/indexFstFile.c
View File

@ -100,7 +100,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
do {
char key[1024] = {0};
assert(strlen(ctx->file.buf) + 1 + 64 < sizeof(key));
ASSERT(strlen(ctx->file.buf) + 1 + 64 < sizeof(key));
idxGenLRUKey(key, ctx->file.buf, blkId);
LRUHandle* h = taosLRUCacheLookup(ctx->lru, key, strlen(key));
@ -114,7 +114,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
if (left < kBlockSize) {
nread = TMIN(left, len);
int32_t bytes = taosPReadFile(ctx->file.pFile, buf + total, nread, offset);
assert(bytes == nread);
ASSERTS(bytes == nread, "index read incomplete data");
if (bytes != nread) break;
total += bytes;
return total;
@ -124,7 +125,8 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of
SDataBlock* blk = taosMemoryCalloc(1, cacheMemSize);
blk->blockId = blkId;
blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize);
assert(blk->nread <= kBlockSize);
ASSERTS(blk->nread <= kBlockSize, "index read incomplete data");
if (blk->nread > kBlockSize) break;
if (blk->nread < kBlockSize && blk->nread < len) {
taosMemoryFree(blk);
@ -275,7 +277,10 @@ int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) {
// update checksum
IFileCtx* ctx = write->wrt;
int nWrite = ctx->write(ctx, buf, len);
assert(nWrite == len);
ASSERTS(nWrite == len, "index write incomplete data");
if (nWrite != len) {
return -1;
}
write->count += len;
write->summer = taosCalcChecksum(write->summer, buf, len);
@ -302,7 +307,6 @@ int idxFileFlush(IdxFstFile* write) {
}
void idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes) {
assert(1 <= nBytes && nBytes <= 8);
uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t));
for (uint8_t i = 0; i < nBytes; i++) {
buf[i] = (uint8_t)n;

4
source/libs/index/src/indexFstRegister.c
View File

@ -57,8 +57,8 @@ static void fstRegistryCellPromote(SArray* arr, uint32_t start, uint32_t end) {
if (start >= sz && end >= sz) {
return;
}
assert(start >= end);
ASSERTS(start >= end, "index-fst start lower than end");
if (start < end) return;
int32_t s = (int32_t)start;
int32_t e = (int32_t)end;

1
source/libs/index/src/indexFstUtil.c
View File

@ -101,7 +101,6 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) {
int32_t slen;
uint8_t* data = fstSliceData(s, &slen);
assert(tlen <= slen);
uint8_t* buf = taosMemoryMalloc(sizeof(uint8_t) * tlen);
memcpy(buf, data + start, tlen);

16
source/libs/index/src/indexTfile.c
View File

@ -122,7 +122,6 @@ TFileCache* tfileCacheCreate(SIndex* idx, const char* path) {
char buf[128] = {0};
int32_t sz = idxSerialCacheKey(&key, buf);
assert(sz < sizeof(buf));
taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
tfileReaderRef(reader);
}
@ -151,9 +150,8 @@ void tfileCacheDestroy(TFileCache* tcache) {
}
TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
char buf[128] = {0};
int32_t sz = idxSerialCacheKey(key, buf);
assert(sz < sizeof(buf));
char buf[128] = {0};
int32_t sz = idxSerialCacheKey(key, buf);
TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
if (reader == NULL || *reader == NULL) {
return NULL;
@ -877,7 +875,7 @@ static int tfileWriteFooter(TFileWriter* write) {
int nwrite = write->ctx->write(write->ctx, buf, (int32_t)strlen(buf));
indexInfo("tfile write footer size: %d", write->ctx->size(write->ctx));
assert(nwrite == sizeof(FILE_MAGIC_NUMBER));
ASSERTS(nwrite == sizeof(FILE_MAGIC_NUMBER), "index write incomplete data");
return nwrite;
}
static int tfileReaderLoadHeader(TFileReader* reader) {
@ -892,7 +890,6 @@ static int tfileReaderLoadHeader(TFileReader* reader) {
} else {
indexInfo("actual Read: %d, to read: %d, filename: %s", (int)(nread), (int)sizeof(buf), reader->ctx->file.buf);
}
// assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf));
return 0;
@ -914,7 +911,10 @@ static int tfileReaderLoadFst(TFileReader* reader) {
indexInfo("nread = %d, and fst offset=%d, fst size: %d, filename: %s, file size: %d, time cost: %" PRId64 "us", nread,
reader->header.fstOffset, fstSize, ctx->file.buf, size, cost);
// we assuse fst size less than FST_MAX_SIZE
assert(nread > 0 && nread <= fstSize);
ASSERTS(nread > 0 && nread <= fstSize, "index read incomplete fst");
if (nread <= 0 || nread > fstSize) {
return -1;
}
FstSlice st = fstSliceCreate((uint8_t*)buf, nread);
reader->fst = fstCreate(&st);
@ -929,7 +929,7 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
// add block cache
char block[4096] = {0};
int32_t nread = ctx->readFrom(ctx, block, sizeof(block), offset);
assert(nread >= sizeof(uint32_t));
ASSERT(nread >= sizeof(uint32_t));
char* p = block;
int32_t nid = *(int32_t*)p;

1
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -402,6 +402,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
CLONE_NODE_LIST_FIELD(pGroupKeys);
CLONE_NODE_LIST_FIELD(pAggFuncs);
COPY_SCALAR_FIELD(hasGroupKeyOptimized);
return TSDB_CODE_SUCCESS;
}

7
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -1829,6 +1829,7 @@ static const char* jkAggPhysiPlanExprs = "Exprs";
static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock";
static const char* jkAggPhysiPlanGroupKeyOptimized = "GroupKeyOptimized";
static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj;
@ -1846,6 +1847,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanMergeDataBlock, pNode->mergeDataBlock);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanGroupKeyOptimized, pNode->groupKeyOptimized);
}
return code;
}
@ -1866,6 +1870,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanMergeDataBlock, &pNode->mergeDataBlock);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanGroupKeyOptimized, &pNode->groupKeyOptimized);
}
return code;
}

9
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -2370,7 +2370,8 @@ enum {
PHY_AGG_CODE_EXPR,
PHY_AGG_CODE_GROUP_KEYS,
PHY_AGG_CODE_AGG_FUNCS,
PHY_AGG_CODE_MERGE_DATA_BLOCK
PHY_AGG_CODE_MERGE_DATA_BLOCK,
PHY_AGG_CODE_GROUP_KEY_OPTIMIZE
};
static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2389,6 +2390,9 @@ static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_GROUP_KEY_OPTIMIZE, pNode->groupKeyOptimized);
}
return code;
}
@ -2415,6 +2419,9 @@ static int32_t msgToPhysiAggNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_AGG_CODE_MERGE_DATA_BLOCK:
code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock);
break;
case PHY_AGG_CODE_GROUP_KEY_OPTIMIZE:
code = tlvDecodeBool(pTlv, &pNode->groupKeyOptimized);
break;
default:
break;
}

21
source/libs/parser/inc/parAst.h
View File

@ -186,6 +186,7 @@ SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName);
SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort);
SNode* createDropDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, bool force);
SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SToken* pConfig, const SToken* pValue);
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName);
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,
SNode* pRealTable, SNodeList* pCols, SNode* pOptions);
SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding,
@ -193,15 +194,13 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt
SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pIndexName);
SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId);
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pQuery);
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SToken* pSubDbName, bool withMeta);
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pRealTable, bool withMeta);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName);
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId,
const SToken* pTopicName);
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery);
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName,
bool withMeta);
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable,
bool withMeta);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName);
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, SToken* pTopicName);
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
SNode* createDefaultExplainOptions(SAstCreateContext* pCxt);
SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal);
@ -214,9 +213,9 @@ SNode* createCreateFunctionStmt(SAstCreateContext* pCxt, bool ignoreExists, bool
const SToken* pLibPath, SDataType dataType, int32_t bufSize);
SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pFuncName);
SNode* createStreamOptions(SAstCreateContext* pCxt);
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable,
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable,
SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery);
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName);
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName);
SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);

11
source/libs/parser/inc/sql.y
View File

@ -459,8 +459,11 @@ tag_item(A) ::= column_name(B) AS column_alias(C).
/************************************************ create index ********************************************************/
cmd ::= CREATE SMA INDEX not_exists_opt(D)
full_table_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); }
cmd ::= DROP INDEX exists_opt(B) full_table_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); }
full_index_name(A) ON full_table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, A, B, NULL, C); }
cmd ::= DROP INDEX exists_opt(B) full_index_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, A); }
full_index_name(A) ::= index_name(B). { A = createRealTableNodeForIndexName(pCxt, NULL, &B); }
full_index_name(A) ::= db_name(B) NK_DOT index_name(C). { A = createRealTableNodeForIndexName(pCxt, &B, &C); }
index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL
NK_LP duration_literal(C) NK_RP sliding_opt(D) sma_stream_opt(E). { A = createIndexOption(pCxt, B, releaseRawExprNode(pCxt, C), NULL, D, E); }
@ -657,6 +660,10 @@ stream_name(A) ::= NK_ID(B).
%destructor cgroup_name { }
cgroup_name(A) ::= NK_ID(B). { A = B; }
%type index_name { SToken }
%destructor index_name { }
index_name(A) ::= NK_ID(B). { A = B; }
/************************************************ expression **********************************************************/
expr_or_subquery(A) ::= expression(B). { A = B; }
//expr_or_subquery(A) ::= subquery(B). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), NULL); }

64
source/libs/parser/src/parAstCreater.c
View File

@ -201,6 +201,24 @@ static bool checkIndexName(SAstCreateContext* pCxt, SToken* pIndexName) {
return true;
}
static bool checkTopicName(SAstCreateContext* pCxt, SToken* pTopicName) {
trimEscape(pTopicName);
if (pTopicName->n >= TSDB_TOPIC_NAME_LEN) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pTopicName->z);
return false;
}
return true;
}
static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) {
trimEscape(pStreamName);
if (pStreamName->n >= TSDB_STREAM_NAME_LEN) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pStreamName->z);
return false;
}
return true;
}
static bool checkComment(SAstCreateContext* pCxt, const SToken* pCommentToken, bool demand) {
if (NULL == pCommentToken) {
pCxt->errCode = demand ? TSDB_CODE_PAR_SYNTAX_ERROR : TSDB_CODE_SUCCESS;
@ -1509,6 +1527,13 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const
return (SNode*)pStmt;
}
SNode* createRealTableNodeForIndexName(SAstCreateContext* pCxt, SToken* pDbName, SToken* pIndexName) {
if (!checkIndexName(pCxt, pIndexName)) {
return NULL;
}
return createRealTableNode(pCxt, pDbName, pIndexName, NULL);
}
SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SNode* pIndexName,
SNode* pRealTable, SNodeList* pCols, SNode* pOptions) {
CHECK_PARSER_STATUS(pCxt);
@ -1567,9 +1592,11 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pQuery) {
SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
@ -1578,10 +1605,10 @@ SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists,
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SToken* pSubDbName, bool withMeta) {
SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SToken* pSubDbName,
bool withMeta) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pSubDbName, true)) {
if (!checkTopicName(pCxt, pTopicName) || !checkDbName(pCxt, pSubDbName, true)) {
return NULL;
}
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
@ -1593,9 +1620,12 @@ SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, co
return (SNode*)pStmt;
}
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName,
SNode* pRealTable, bool withMeta) {
SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, SToken* pTopicName, SNode* pRealTable,
bool withMeta) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
@ -1607,8 +1637,11 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists,
return (SNode*)pStmt;
}
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName) {
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SDropTopicStmt* pStmt = (SDropTopicStmt*)nodesMakeNode(QUERY_NODE_DROP_TOPIC_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName);
@ -1617,8 +1650,11 @@ SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const
}
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId,
const SToken* pTopicName) {
SToken* pTopicName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkTopicName(pCxt, pTopicName)) {
return NULL;
}
SDropCGroupStmt* pStmt = (SDropCGroupStmt*)nodesMakeNode(QUERY_NODE_DROP_CGROUP_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->ignoreNotExists = ignoreNotExists;
@ -1730,9 +1766,12 @@ SNode* createStreamOptions(SAstCreateContext* pCxt) {
return (SNode*)pOptions;
}
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pStreamName, SNode* pRealTable,
SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* pStreamName, SNode* pRealTable,
SNode* pOptions, SNodeList* pTags, SNode* pSubtable, SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt);
if (!checkStreamName(pCxt, pStreamName)) {
return NULL;
}
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)nodesMakeNode(QUERY_NODE_CREATE_STREAM_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName);
@ -1749,8 +1788,11 @@ SNode* createCreateStreamStmt(SAstCreateContext* pCxt, bool ignoreExists, const
return (SNode*)pStmt;
}
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pStreamName) {
SNode* createDropStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pStreamName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkStreamName(pCxt, pStreamName)) {
return NULL;
}
SDropStreamStmt* pStmt = (SDropStreamStmt*)nodesMakeNode(QUERY_NODE_DROP_STREAM_STMT);
CHECK_OUT_OF_MEM(pStmt);
COPY_STRING_FORM_ID_TOKEN(pStmt->streamName, pStreamName);

6
source/libs/parser/src/parTranslater.c
View File

@ -4948,6 +4948,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
}
if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped");
}
int32_t tagsLen = 0;
for (int32_t i = 0; i < pTableMeta->tableInfo.numOfTags; ++i) {
tagsLen += pTagsSchema[i].bytes;
@ -7482,7 +7486,7 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) {
static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta,
SQuery* pQuery) {
if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped");
}
if (TSDB_SUPER_TABLE == pTableMeta->tableType) {

4467
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

1
source/libs/planner/src/planLogicCreater.c
View File

@ -545,6 +545,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
pAgg->hasLastRow = pSelect->hasLastRowFunc;
pAgg->hasLast = pSelect->hasLastFunc;
pAgg->hasTimeLineFunc = pSelect->hasTimeLineFunc;
pAgg->hasGroupKeyOptimized = false;
pAgg->onlyHasKeepOrderFunc = pSelect->onlyHasKeepOrderFunc;
pAgg->node.groupAction = getGroupAction(pCxt, pSelect);
pAgg->node.requireDataOrder = getRequireDataOrder(pAgg->hasTimeLineFunc, pSelect);

11
source/libs/planner/src/planOptimizer.c
View File

@ -1544,6 +1544,11 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
code = adjustLogicNodeDataRequirement((SLogicNode*)pScan, pNode->resultDataOrder);
}
if (TSDB_CODE_SUCCESS == code) {
if (QUERY_NODE_LOGIC_PLAN_AGG == pNode->pParent->type) {
SAggLogicNode* pParent = (SAggLogicNode*)(pNode->pParent);
pParent->hasGroupKeyOptimized = true;
}
NODES_CLEAR_LIST(pNode->pChildren);
nodesDestroyNode((SNode*)pNode);
}
@ -1569,6 +1574,8 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
break;
}
}
pAgg->hasGroupKeyOptimized = true;
NODES_DESTORY_LIST(pAgg->pGroupKeys);
if (TSDB_CODE_SUCCESS == code && start >= 0) {
code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg);
@ -1577,6 +1584,7 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
if (TSDB_CODE_SUCCESS == code) {
code = partTagsOptRebuildTbanme(pScan->pGroupTags);
}
pCxt->optimized = true;
return code;
}
@ -2380,6 +2388,9 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) {
if (pScan->hasNormalCols) {
return false;
}
if (pScan->tableType == TSDB_SYSTEM_TABLE) {
return false;
}
if (NULL == pNode->pParent || QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) ||
1 != LIST_LENGTH(pNode->pParent->pChildren)) {
return false;

1
source/libs/planner/src/planPhysiCreater.c
View File

@ -872,6 +872,7 @@ static int32_t createAggPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren,
}
pAgg->mergeDataBlock = (GROUP_ACTION_KEEP == pAggLogicNode->node.groupAction ? false : true);
pAgg->groupKeyOptimized = pAggLogicNode->hasGroupKeyOptimized;
SNodeList* pPrecalcExprs = NULL;
SNodeList* pGroupKeys = NULL;

16
source/libs/planner/src/planSpliter.c
View File

@ -333,13 +333,23 @@ static bool stbSplHasPartTbname(SNodeList* pPartKeys) {
return false;
}
static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) {
if (NULL != pAgg->pGroupKeys) {
return stbSplHasPartTbname(pAgg->pGroupKeys);
static bool stbSplNotSystemScan(SLogicNode* pNode) {
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) {
return SCAN_TYPE_SYSTEM_TABLE != ((SScanLogicNode*)pNode)->scanType;
} else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
return stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pNode->pChildren, 0));
} else {
return true;
}
}
static bool stbSplIsPartTableAgg(SAggLogicNode* pAgg) {
if (1 != LIST_LENGTH(pAgg->node.pChildren)) {
return false;
}
if (NULL != pAgg->pGroupKeys) {
return stbSplHasPartTbname(pAgg->pGroupKeys) && stbSplNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0));
}
return stbSplHasPartTbname(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)));
}

17
source/libs/qworker/inc/qwInt.h
View File

@ -114,14 +114,14 @@ typedef struct SQWTaskStatus {
typedef struct SQWTaskCtx {
SRWLatch lock;
int8_t phase;
int8_t inFetch;
int8_t taskType;
int8_t explain;
int8_t needFetch;
int8_t localExec;
int32_t msgType;
int32_t fetchType;
int32_t execId;
int32_t level;
uint64_t sId;
bool queryGotData;
bool queryRsped;
@ -221,8 +221,16 @@ typedef struct SQWorkerMgmt {
#define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase)
#define QW_SET_PHASE(ctx, _value) \
do { \
if ((_value) != QW_PHASE_PRE_FETCH && (_value) != QW_PHASE_POST_FETCH) { \
atomic_store_8(&(ctx)->phase, _value); \
switch (_value) { \
case QW_PHASE_PRE_FETCH: \
ctx->inFetch = 1; \
break; \
case QW_PHASE_POST_FETCH: \
ctx->inFetch = 0; \
break; \
default: \
atomic_store_8(&(ctx)->phase, _value); \
break; \
} \
} while (0)
@ -230,6 +238,7 @@ typedef struct SQWorkerMgmt {
#define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code)
#define QW_QUERY_RUNNING(ctx) (QW_GET_PHASE(ctx) == QW_PHASE_PRE_QUERY || QW_GET_PHASE(ctx) == QW_PHASE_PRE_CQUERY)
#define QW_FETCH_RUNNING(ctx) ((ctx)->inFetch)
#define QW_SET_QTID(id, qId, tId, eId) \
do { \

6
source/libs/qworker/src/qwDbg.c
View File

@ -124,11 +124,11 @@ void qwDbgDumpTasksInfo(SQWorker *mgmt) {
void *key = taosHashGetKey(pIter, NULL);
QW_GET_QTID(key, qId, tId, eId);
QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, fetchType:%d, "
"execId:%x, level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, "
QW_TASK_DLOG("%p lock:%x, phase:%d, type:%d, explain:%d, needFetch:%d, localExec:%d, msgType:%d, "
"sId:%" PRId64 ", level:%d, queryGotData:%d, queryRsped:%d, queryEnd:%d, queryContinue:%d, queryInQueue:%d, "
"rspCode:%x, affectedRows:%" PRId64 ", taskHandle:%p, sinkHandle:%p, tbFName:%s, sver:%d, tver:%d, events:%d,%d,%d,%d,%d",
ctx, ctx->lock, ctx->phase, ctx->taskType, ctx->explain, ctx->needFetch, ctx->localExec, ctx->msgType,
ctx->fetchType, ctx->execId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue,
ctx->sId, ctx->level, ctx->queryGotData, ctx->queryRsped, ctx->queryEnd, ctx->queryContinue,
ctx->queryInQueue, ctx->rspCode, ctx->affectedRows, ctx->taskHandle, ctx->sinkHandle, ctx->tbInfo.tbFName,
ctx->tbInfo.sversion, ctx->tbInfo.tversion, ctx->events[QW_EVENT_CANCEL], ctx->events[QW_EVENT_READY],
ctx->events[QW_EVENT_FETCH], ctx->events[QW_EVENT_DROP], ctx->events[QW_EVENT_CQUERY]);

33
source/libs/qworker/src/qworker.c
View File

@ -508,14 +508,6 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp
}
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) {
if (QW_PHASE_POST_FETCH == phase) {
QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase));
QW_ERR_JRET(TSDB_CODE_APP_ERROR);
}
// qwBuildAndSendDropRsp(&ctx->ctrlConnInfo, code);
// QW_TASK_DLOG("drop rsp send, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code));
QW_ERR_JRET(qwDropTask(QW_FPARAMS()));
QW_ERR_JRET(ctx->rspCode);
}
@ -580,6 +572,7 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
QW_ERR_JRET(qwAcquireTaskCtx(QW_FPARAMS(), &ctx));
ctx->ctrlConnInfo = qwMsg->connInfo;
ctx->sId = sId;
ctx->phase = -1;
QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT));
@ -670,7 +663,7 @@ _return:
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code);
qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
@ -688,6 +681,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
void *rsp = NULL;
int32_t dataLen = 0;
bool queryStop = false;
bool qComplete = false;
do {
ctx = NULL;
@ -712,17 +706,18 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
}
if (rsp) {
bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd);
qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete);
if (qComplete) {
atomic_store_8((int8_t *)&ctx->queryEnd, true);
atomic_store_8((int8_t *)&ctx->queryContinue, 0);
}
qwMsg->connInfo = ctx->dataConnInfo;
QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH);
qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, rsp, dataLen, code);
qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code);
rsp = NULL;
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code,
@ -744,14 +739,13 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) {
rsp = NULL;
qwMsg->connInfo = ctx->dataConnInfo;
qwBuildAndSendFetchRsp(ctx->fetchType, &qwMsg->connInfo, NULL, 0, code);
qwBuildAndSendFetchRsp(ctx->msgType + 1, &qwMsg->connInfo, NULL, 0, code);
QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code),
0);
}
QW_LOCK(QW_WRITE, &ctx->lock);
if ((queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code ||
0 == atomic_load_8((int8_t *)&ctx->queryContinue)) {
if (qComplete || (queryStop && (0 == atomic_load_8((int8_t *)&ctx->queryContinue))) || code) {
// Note: query is not running anymore
QW_SET_PHASE(ctx, 0);
QW_UNLOCK(QW_WRITE, &ctx->lock);
@ -1178,8 +1172,9 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
QW_DLOG("start to stop all tasks, taskNum:%d", taosHashGetSize(mgmt->ctxHash));
uint64_t qId, tId;
uint64_t qId, tId, sId;
int32_t eId;
int64_t rId = 0;
void *pIter = taosHashIterate(mgmt->ctxHash, NULL);
while (pIter) {
SQWTaskCtx *ctx = (SQWTaskCtx *)pIter;
@ -1188,6 +1183,8 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
QW_LOCK(QW_WRITE, &ctx->lock);
sId = ctx->sId;
QW_TASK_DLOG_E("start to force stop task");
if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP) || QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) {
@ -1200,9 +1197,11 @@ void qWorkerStopAllTasks(void *qWorkerMgmt) {
if (QW_QUERY_RUNNING(ctx)) {
qwKillTaskHandle(ctx, TSDB_CODE_VND_STOPPED);
} else if (!QW_EVENT_PROCESSED(ctx, QW_EVENT_DROP)) {
} else if (QW_FETCH_RUNNING(ctx)) {
QW_UPDATE_RSP_CODE(ctx, TSDB_CODE_VND_STOPPED);
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP);
} else {
qwDropTask(QW_FPARAMS());
}
QW_UNLOCK(QW_WRITE, &ctx->lock);

3
source/libs/scalar/inc/sclInt.h
View File

@ -53,9 +53,6 @@ typedef struct SScalarCtx {
#define SCL_IS_COMPARISON_OPERATOR(_opType) ((_opType) >= OP_TYPE_GREATER_THAN && (_opType) < OP_TYPE_IS_NOT_UNKNOWN)
#define SCL_DOWNGRADE_DATETYPE(_type) \
((_type) == TSDB_DATA_TYPE_BIGINT || TSDB_DATA_TYPE_DOUBLE == (_type) || (_type) == TSDB_DATA_TYPE_UBIGINT)
#define SCL_NO_NEED_CONVERT_COMPARISION(_ltype, _rtype, _optr) \
(IS_NUMERIC_TYPE(_ltype) && IS_NUMERIC_TYPE(_rtype) && \
((_optr) >= OP_TYPE_GREATER_THAN && (_optr) <= OP_TYPE_NOT_EQUAL))
#define sclFatal(...) qFatal(__VA_ARGS__)
#define sclError(...) qError(__VA_ARGS__)

22
source/libs/scalar/src/filter.c
View File

@ -130,9 +130,9 @@ __compar_fn_t gDataCompare[] = {compareInt32Val,
compareFloatVal,
compareDoubleVal,
compareLenPrefixedStr,
compareStrPatternMatch,
comparestrPatternMatch,
compareChkInString,
compareWStrPatternMatch,
comparewcsPatternMatch,
compareLenPrefixedWStr,
compareUint8Val,
compareUint16Val,
@ -142,15 +142,17 @@ __compar_fn_t gDataCompare[] = {compareInt32Val,
setChkInBytes2,
setChkInBytes4,
setChkInBytes8,
compareStrRegexCompMatch,
compareStrRegexCompNMatch,
comparestrRegexMatch,
comparestrRegexNMatch,
setChkNotInBytes1,
setChkNotInBytes2,
setChkNotInBytes4,
setChkNotInBytes8,
compareChkNotInString,
compareStrPatternNotMatch,
compareWStrPatternNotMatch};
comparestrPatternNMatch,
comparewcsPatternNMatch,
comparewcsRegexMatch,
comparewcsRegexNMatch,};
__compar_fn_t gInt8SignCompare[] = {compareInt8Val, compareInt8Int16, compareInt8Int32,
compareInt8Int64, compareInt8Float, compareInt8Double};
@ -295,9 +297,9 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
case TSDB_DATA_TYPE_NCHAR: {
if (optr == OP_TYPE_MATCH) {
comparFn = 19;
comparFn = 28;
} else if (optr == OP_TYPE_NMATCH) {
comparFn = 20;
comparFn = 29;
} else if (optr == OP_TYPE_LIKE) {
comparFn = 9;
} else if (optr == OP_TYPE_NOT_LIKE) {
@ -336,7 +338,7 @@ int8_t filterGetCompFuncIdx(int32_t type, int32_t optr) {
__compar_fn_t filterGetCompFunc(int32_t type, int32_t optr) { return gDataCompare[filterGetCompFuncIdx(type, optr)]; }
__compar_fn_t filterGetCompFuncEx(int32_t lType, int32_t rType, int32_t optr) {
if (TSDB_DATA_TYPE_NULL == rType) {
if (TSDB_DATA_TYPE_NULL == rType || TSDB_DATA_TYPE_JSON == rType) {
return NULL;
}
@ -4085,7 +4087,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC
SArray *pList = taosArrayInit(1, POINTER_BYTES);
taosArrayPush(pList, &pSrc);
int32_t code = scalarCalculate(info->sclCtx.node, pList, &output);
code = scalarCalculate(info->sclCtx.node, pList, &output);
taosArrayDestroy(pList);
FLT_ERR_RET(code);

85
source/libs/scalar/src/sclvector.c
View File

@ -37,6 +37,11 @@
#define IS_HELPER_NULL(col, i) colDataIsNull_s(col, i) || IS_JSON_NULL(col->info.type, colDataGetVarData(col, i))
bool noConvertBeforeCompare(int32_t leftType, int32_t rightType, int32_t optr) {
return IS_NUMERIC_TYPE(leftType) && IS_NUMERIC_TYPE(rightType) &&
(optr >= OP_TYPE_GREATER_THAN && optr <= OP_TYPE_NOT_EQUAL);
}
void convertNumberToNumber(const void *inData, void *outData, int8_t inType, int8_t outType) {
switch (outType) {
case TSDB_DATA_TYPE_BOOL: {
@ -338,6 +343,7 @@ static FORCE_INLINE void varToBool(char *buf, SScalarParam *pOut, int32_t rowInd
colDataAppendInt8(pOut->columnData, rowIndex, (int8_t *)&v);
}
// todo remove this malloc
static FORCE_INLINE void varToNchar(char *buf, SScalarParam *pOut, int32_t rowIndex, int32_t *overflow) {
int32_t len = 0;
int32_t inputLen = varDataLen(buf);
@ -399,6 +405,8 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
}
pCtx->pOut->numOfRows = pCtx->pIn->numOfRows;
char* tmp = NULL;
for (int32_t i = pCtx->startIndex; i <= pCtx->endIndex; ++i) {
if (IS_HELPER_NULL(pCtx->pIn->columnData, i)) {
colDataAppendNULL(pCtx->pOut->columnData, i);
@ -421,12 +429,16 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
continue;
}
}
int32_t bufSize = pCtx->pIn->columnData->info.bytes;
char *tmp = taosMemoryMalloc(varDataTLen(data));
if (!tmp) {
sclError("out of memory in vectorConvertFromVarData");
return TSDB_CODE_OUT_OF_MEMORY;
if (tmp == NULL) {
tmp = taosMemoryMalloc(bufSize);
if (tmp == NULL) {
sclError("out of memory in vectorConvertFromVarData");
return TSDB_CODE_OUT_OF_MEMORY;
}
}
if (vton) {
memcpy(tmp, data, varDataTLen(data));
} else {
@ -434,6 +446,7 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
memcpy(tmp, varDataVal(data), varDataLen(data));
tmp[varDataLen(data)] = 0;
} else if (TSDB_DATA_TYPE_NCHAR == convertType) {
// we need to convert it to native char string, and then perform the string to numeric data
ASSERT(varDataLen(data) <= bufSize);
int len = taosUcs4ToMbs((TdUcs4 *)varDataVal(data), varDataLen(data), tmp);
@ -448,9 +461,11 @@ int32_t vectorConvertFromVarData(SSclVectorConvCtx *pCtx, int32_t *overflow) {
}
(*func)(tmp, pCtx->pOut, i, overflow);
taosMemoryFreeClear(tmp);
}
if (tmp != NULL) {
taosMemoryFreeClear(tmp);
}
return TSDB_CODE_SUCCESS;
}
@ -901,9 +916,11 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) {
int32_t vectorConvertSingleCol(SScalarParam *input, SScalarParam *output, int32_t type, int32_t startIndex,
int32_t numOfRows) {
SDataType t = {.type = type, .bytes = tDataTypes[type].bytes};
output->numOfRows = input->numOfRows;
SDataType t = {.type = type};
t.bytes = IS_VAR_DATA_TYPE(t.type)? input->columnData->info.bytes:tDataTypes[type].bytes;
int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -925,25 +942,43 @@ int32_t vectorConvertCols(SScalarParam *pLeft, SScalarParam *pRight, SScalarPara
return TSDB_CODE_SUCCESS;
}
int8_t type = 0;
int32_t code = 0;
SScalarParam *param1 = NULL, *paramOut1 = NULL;
SScalarParam *param2 = NULL, *paramOut2 = NULL;
int32_t code = 0;
if (leftType < rightType) {
// always convert least data
if (IS_VAR_DATA_TYPE(leftType) && IS_VAR_DATA_TYPE(rightType) && (pLeft->numOfRows != pRight->numOfRows) &&
leftType != TSDB_DATA_TYPE_JSON && rightType != TSDB_DATA_TYPE_JSON) {
param1 = pLeft;
param2 = pRight;
paramOut1 = pLeftOut;
paramOut2 = pRightOut;
} else {
param1 = pRight;
param2 = pLeft;
paramOut1 = pRightOut;
paramOut2 = pLeftOut;
}
int8_t type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
if (0 == type) {
return TSDB_CODE_SUCCESS;
if (pLeft->numOfRows > pRight->numOfRows) {
type = leftType;
} else {
type = rightType;
}
} else {
// we only define half value in the convert-matrix, so make sure param1 always less equal than param2
if (leftType < rightType) {
param1 = pLeft;
param2 = pRight;
paramOut1 = pLeftOut;
paramOut2 = pRightOut;
} else {
param1 = pRight;
param2 = pLeft;
paramOut1 = pRightOut;
paramOut2 = pLeftOut;
}
type = vectorGetConvertType(GET_PARAM_TYPE(param1), GET_PARAM_TYPE(param2));
if (0 == type) {
return TSDB_CODE_SUCCESS;
}
}
if (type != GET_PARAM_TYPE(param1)) {
@ -1683,23 +1718,13 @@ void vectorCompareImpl(SScalarParam *pLeft, SScalarParam *pRight, SScalarParam *
SScalarParam *param1 = NULL;
SScalarParam *param2 = NULL;
if (SCL_NO_NEED_CONVERT_COMPARISION(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
if (noConvertBeforeCompare(GET_PARAM_TYPE(pLeft), GET_PARAM_TYPE(pRight), optr)) {
param1 = pLeft;
param2 = pRight;
} else {
vectorConvertCols(pLeft, pRight, &pLeftOut, &pRightOut, startIndex, numOfRows);
if (pLeftOut.columnData != NULL) {
param1 = &pLeftOut;
} else {
param1 = pLeft;
}
if (pRightOut.columnData != NULL) {
param2 = &pRightOut;
} else {
param2 = pRight;
}
param1 = (pLeftOut.columnData != NULL) ? &pLeftOut : pLeft;
param2 = (pRightOut.columnData != NULL) ? &pRightOut : pRight;
}
doVectorCompare(param1, param2, pOut, startIndex, numOfRows, _ord, optr);

10
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -344,7 +344,7 @@ TEST(constantTest, int_or_binary) {
ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE);
SValueNode *v = (SValueNode *)res;
ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BIGINT);
ASSERT_EQ(v->datum.b, scltLeftV | scltRightV);
ASSERT_EQ(v->datum.i, scltLeftV | scltRightV);
nodesDestroyNode(res);
}
@ -1101,7 +1101,8 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do
opType == OP_TYPE_LIKE || opType == OP_TYPE_NOT_LIKE || opType == OP_TYPE_MATCH ||
opType == OP_TYPE_NMATCH) {
printf("op:%s,3result:%d,except:%f\n", operatorTypeStr(opType), *((bool *)colDataGetData(column, 0)), exceptValue);
ASSERT_EQ(*((bool *)colDataGetData(column, 0)), exceptValue);
assert(*(bool *)colDataGetData(column, 0) == exceptValue);
// ASSERT_EQ((int) *((bool *)colDataGetData(column, 0)), (int)exceptValue);
}
taosArrayDestroyEx(blockList, scltFreeDataBlock);
@ -1426,7 +1427,7 @@ TEST(columnTest, json_column_logic_op) {
printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n");
key = "k9";
bool eRes8[len + len1] = {false, false, false, false, false, false, false, true, true, false, true, false, true};
bool eRes8[len + len1] = {false, false, false, false, false, false, false, true, true, false, true, true, true};
for (int i = 0; i < len; i++) {
makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false);
}
@ -1437,6 +1438,9 @@ TEST(columnTest, json_column_logic_op) {
for (int i = len; i < len + len1; i++) {
void *rightData = prepareNchar(inputNchar[i - len]);
if (i == 11) {
printf("abc\n");
}
makeCalculate(row, key, TSDB_DATA_TYPE_NCHAR, rightData, eRes8[i], op[i], false);
taosMemoryFree(rightData);
}

29
source/libs/stream/src/stream.c
View File

@ -187,6 +187,23 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq,
return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1;
}
int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) {
if (pTask->outputType == TASK_OUTPUT__TABLE) {
pTask->tbSink.tbSinkFunc(pTask, pTask->tbSink.vnode, 0, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else if (pTask->outputType == TASK_OUTPUT__SMA) {
pTask->smaSink.smaSink(pTask->smaSink.vnode, pTask->smaSink.smaId, pBlock->blocks);
taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes);
taosFreeQitem(pBlock);
} else {
ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH);
taosWriteQitem(pTask->outputQueue->queue, pBlock);
streamDispatch(pTask);
}
return 0;
}
int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) {
qDebug("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId,
pReq->upstreamTaskId);
@ -199,9 +216,9 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S
return -1;
}
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamDispatch(pTask);
}
/*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/
/*streamDispatch(pTask);*/
/*}*/
} else {
streamSchedExec(pTask);
}
@ -237,9 +254,9 @@ int32_t streamProcessRunReq(SStreamTask* pTask) {
return -1;
}
if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
streamDispatch(pTask);
}
/*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/
/*streamDispatch(pTask);*/
/*}*/
return 0;
}

6
source/libs/stream/src/streamExec.c
View File

@ -110,10 +110,14 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
int32_t batchCnt = 0;
while (1) {
if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) {
return 0;
}
SSDataBlock* output = NULL;
uint64_t ts = 0;
if (qExecTask(exec, &output, &ts) < 0) {
ASSERT(0);
return -1;
}
if (output == NULL) {
if (qStreamRecoverScanFinished(exec)) {

1
source/libs/stream/src/streamMeta.c
View File

@ -294,6 +294,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta) {
tdbTbcClose(pCur);
return -1;
}
pTask->taskStatus = TASK_STATUS__NORMAL;
}
tdbFree(pKey);

1
source/libs/stream/src/streamRecover.c
View File

@ -200,7 +200,6 @@ int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req*
int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) {
void* exec = pTask->exec.executor;
if (qStreamSourceRecoverStep2(exec, ver) < 0) {
ASSERT(0);
}
return streamScanExec(pTask, 100);
}

36
source/libs/sync/inc/syncIndexMgr.h
View File

@ -25,29 +25,27 @@ extern "C" {
// SIndexMgr -----------------------------
typedef struct SSyncIndexMgr {
SRaftId (*replicas)[TSDB_MAX_REPLICA];
SyncIndex index[TSDB_MAX_REPLICA];
SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function
int64_t startTimeArr[TSDB_MAX_REPLICA];
int64_t recvTimeArr[TSDB_MAX_REPLICA];
SyncIndex index[TSDB_MAX_REPLICA];
SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function
int64_t startTimeArr[TSDB_MAX_REPLICA];
int64_t recvTimeArr[TSDB_MAX_REPLICA];
int32_t replicaNum;
SSyncNode *pSyncNode;
SSyncNode *pNode;
} SSyncIndexMgr;
SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode);
void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode);
void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr);
void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr);
void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index);
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId);
SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode);
void syncIndexMgrUpdate(SSyncIndexMgr *pIndexMgr, SSyncNode *pNode);
void syncIndexMgrDestroy(SSyncIndexMgr *pIndexMgr);
void syncIndexMgrClear(SSyncIndexMgr *pIndexMgr);
void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncIndex index);
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);
void syncIndexMgrSetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t startTime);
int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId);
void syncIndexMgrSetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t recvTime);
int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId);
void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term);
SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId);
void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime);
int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);
void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime);
int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);
void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term);
SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId);
#ifdef __cplusplus
}

16
source/libs/sync/inc/syncInt.h
View File

@ -53,6 +53,18 @@ typedef struct SyncPreSnapshot SyncPreSnapshot;
typedef struct SSyncLogBuffer SSyncLogBuffer;
typedef struct SSyncLogReplMgr SSyncLogReplMgr;
#define MAX_CONFIG_INDEX_COUNT 256
typedef struct SRaftCfg {
SSyncCfg cfg;
int32_t batchSize;
int8_t isStandBy;
int8_t snapshotStrategy;
SyncIndex lastConfigIndex;
int32_t configIndexCount;
SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT];
} SRaftCfg;
typedef struct SRaftId {
SyncNodeId addr;
SyncGroupId vgId;
@ -93,7 +105,7 @@ typedef struct SPeerState {
typedef struct SSyncNode {
// init by SSyncInfo
SyncGroupId vgId;
SRaftCfg* pRaftCfg;
SRaftCfg raftCfg;
char path[TSDB_FILENAME_LEN];
char raftStorePath[TSDB_FILENAME_LEN * 2];
char configPath[TSDB_FILENAME_LEN * 2];
@ -112,6 +124,7 @@ typedef struct SSyncNode {
int32_t peersNum;
SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA];
SEpSet peersEpset[TSDB_MAX_REPLICA];
SRaftId peersId[TSDB_MAX_REPLICA];
int32_t replicaNum;
@ -245,7 +258,6 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode);
// utils --------------
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode);
int32_t syncCacheEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry, LRUHandle** h);
bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode);

61
source/libs/sync/inc/syncRaftCfg.h
View File

@ -22,64 +22,9 @@ extern "C" {
#include "syncInt.h"
#define CONFIG_FILE_LEN 2048
#define MAX_CONFIG_INDEX_COUNT 256
typedef struct SRaftCfgIndex {
TdFilePtr pFile;
char path[TSDB_FILENAME_LEN * 2];
SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT];
int32_t configIndexCount;
} SRaftCfgIndex;
SRaftCfgIndex *raftCfgIndexOpen(const char *path);
int32_t raftCfgIndexClose(SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexPersist(SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexAddConfigIndex(SRaftCfgIndex *pRaftCfgIndex, SyncIndex configIndex);
cJSON *raftCfgIndex2Json(SRaftCfgIndex *pRaftCfgIndex);
char *raftCfgIndex2Str(SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexFromJson(const cJSON *pRoot, SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexFromStr(const char *s, SRaftCfgIndex *pRaftCfgIndex);
int32_t raftCfgIndexCreateFile(const char *path);
typedef struct SRaftCfg {
SSyncCfg cfg;
TdFilePtr pFile;
char path[TSDB_FILENAME_LEN * 2];
int8_t isStandBy;
int32_t batchSize;
int8_t snapshotStrategy;
SyncIndex lastConfigIndex;
SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT];
int32_t configIndexCount;
} SRaftCfg;
SRaftCfg *raftCfgOpen(const char *path);
int32_t raftCfgClose(SRaftCfg *pRaftCfg);
int32_t raftCfgPersist(SRaftCfg *pRaftCfg);
int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex);
void syncCfg2SimpleStr(const SSyncCfg *pCfg, char *str, int32_t bufLen);
cJSON *syncCfg2Json(SSyncCfg *pSyncCfg);
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg);
cJSON *raftCfg2Json(SRaftCfg *pRaftCfg);
char *raftCfg2Str(SRaftCfg *pRaftCfg);
int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg);
int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg);
typedef struct SRaftCfgMeta {
int8_t isStandBy;
int32_t batchSize;
int8_t snapshotStrategy;
SyncIndex lastConfigIndex;
} SRaftCfgMeta;
int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path);
int32_t syncWriteCfgFile(SSyncNode *pNode);
int32_t syncReadCfgFile(SSyncNode *pNode);
int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex);
#ifdef __cplusplus
}

19
source/libs/sync/inc/syncUtil.h
View File

@ -62,22 +62,19 @@ extern "C" {
// clang-format on
uint64_t syncUtilAddr2U64(const char* host, uint16_t port);
void syncUtilU642Addr(uint64_t u64, char* host, int64_t len, uint16_t* port);
void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet);
void syncUtilRaftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet);
bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId);
bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2);
bool syncUtilEmptyId(const SRaftId* pId);
#define CID(pRaftId) (int32_t)(((pRaftId)->addr) >> 32)
#define DID(pRaftId) (int32_t)((pRaftId)->addr)
#define SYNC_ADDR(pInfo) (int64_t)(((pInfo)->clusterId << 32) | (pInfo)->nodeId)
void syncUtilNodeInfo2EpSet(const SNodeInfo* pInfo, SEpSet* pEpSet);
bool syncUtilNodeInfo2RaftId(const SNodeInfo* pInfo, SyncGroupId vgId, SRaftId* raftId);
bool syncUtilSameId(const SRaftId* pId1, const SRaftId* pId2);
bool syncUtilEmptyId(const SRaftId* pId);
int32_t syncUtilElectRandomMS(int32_t min, int32_t max);
int32_t syncUtilQuorum(int32_t replicaNum);
cJSON* syncUtilRaftId2Json(const SRaftId* p);
const char* syncStr(ESyncState state);
char* syncUtilPrintBin(char* ptr, uint32_t len);
char* syncUtilPrintBin2(char* ptr, uint32_t len);
void syncUtilMsgHtoN(void* msg);
void syncUtilMsgNtoH(void* msg);
bool syncUtilUserPreCommit(tmsg_t msgType);
bool syncUtilUserRollback(tmsg_t msgType);

12
source/libs/sync/inc/syncVoteMgr.h
View File

@ -30,12 +30,12 @@ typedef struct SVotesGranted {
SyncTerm term;
int32_t quorum;
bool toLeader;
SSyncNode *pSyncNode;
SSyncNode *pNode;
} SVotesGranted;
SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode);
SVotesGranted *voteGrantedCreate(SSyncNode *pNode);
void voteGrantedDestroy(SVotesGranted *pVotesGranted);
void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pSyncNode);
void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pNode);
bool voteGrantedMajority(SVotesGranted *pVotesGranted);
void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg);
void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term);
@ -45,12 +45,12 @@ typedef struct SVotesRespond {
bool isRespond[TSDB_MAX_REPLICA];
int32_t replicaNum;
SyncTerm term;
SSyncNode *pSyncNode;
SSyncNode *pNode;
} SVotesRespond;
SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode);
SVotesRespond *votesRespondCreate(SSyncNode *pNode);
void votesRespondDestory(SVotesRespond *pVotesRespond);
void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pSyncNode);
void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pNode);
bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId);
void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg);
void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term);

2
source/libs/sync/src/syncElection.c
View File

@ -94,7 +94,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) {
voteGrantedUpdate(pSyncNode->pVotesGranted, pSyncNode);
votesRespondUpdate(pSyncNode->pVotesRespond, pSyncNode);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum);
syncNodeCandidate2Leader(pSyncNode);
pSyncNode->pVotesGranted->toLeader = true;

2
source/libs/sync/src/syncEnv.c
View File

@ -115,7 +115,7 @@ void syncHbTimerDataRemove(int64_t rid) { taosRemoveRef(gHbDataRefId, rid); }
SSyncHbTimerData *syncHbTimerDataAcquire(int64_t rid) {
SSyncHbTimerData *pData = taosAcquireRef(gHbDataRefId, rid);
if (pData == NULL) {
sError("failed to acquire hb-timer-data from refId:%" PRId64, rid);
sInfo("failed to acquire hb-timer-data from refId:%" PRId64, rid);
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
}

173
source/libs/sync/src/syncIndexMgr.c
View File

@ -17,173 +17,152 @@
#include "syncIndexMgr.h"
#include "syncUtil.h"
SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) {
SSyncIndexMgr *pSyncIndexMgr = taosMemoryCalloc(1, sizeof(SSyncIndexMgr));
if (pSyncIndexMgr == NULL) {
SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) {
SSyncIndexMgr *pIndexMgr = taosMemoryCalloc(1, sizeof(SSyncIndexMgr));
if (pIndexMgr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pSyncIndexMgr->replicas = &(pSyncNode->replicasId);
pSyncIndexMgr->replicaNum = pSyncNode->replicaNum;
pSyncIndexMgr->pSyncNode = pSyncNode;
syncIndexMgrClear(pSyncIndexMgr);
pIndexMgr->replicas = &pNode->replicasId;
pIndexMgr->replicaNum = pNode->replicaNum;
pIndexMgr->pNode = pNode;
syncIndexMgrClear(pIndexMgr);
return pSyncIndexMgr;
return pIndexMgr;
}
void syncIndexMgrUpdate(SSyncIndexMgr *pSyncIndexMgr, SSyncNode *pSyncNode) {
pSyncIndexMgr->replicas = &(pSyncNode->replicasId);
pSyncIndexMgr->replicaNum = pSyncNode->replicaNum;
pSyncIndexMgr->pSyncNode = pSyncNode;
syncIndexMgrClear(pSyncIndexMgr);
void syncIndexMgrUpdate(SSyncIndexMgr *pIndexMgr, SSyncNode *pNode) {
pIndexMgr->replicas = &pNode->replicasId;
pIndexMgr->replicaNum = pNode->replicaNum;
pIndexMgr->pNode = pNode;
syncIndexMgrClear(pIndexMgr);
}
void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) {
if (pSyncIndexMgr != NULL) {
taosMemoryFree(pSyncIndexMgr);
void syncIndexMgrDestroy(SSyncIndexMgr *pIndexMgr) {
if (pIndexMgr != NULL) {
taosMemoryFree(pIndexMgr);
}
}
void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) {
memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index));
memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm));
void syncIndexMgrClear(SSyncIndexMgr *pIndexMgr) {
memset(pIndexMgr->index, 0, sizeof(pIndexMgr->index));
memset(pIndexMgr->privateTerm, 0, sizeof(pIndexMgr->privateTerm));
// int64_t timeNow = taosGetMonotonicMs();
int64_t timeNow = taosGetTimestampMs();
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
pSyncIndexMgr->startTimeArr[i] = 0;
pSyncIndexMgr->recvTimeArr[i] = timeNow;
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
pIndexMgr->startTimeArr[i] = 0;
pIndexMgr->recvTimeArr[i] = timeNow;
}
/*
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
pSyncIndexMgr->index[i] = 0;
}
*/
}
void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
(pSyncIndexMgr->index)[i] = index;
void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncIndex index) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
(pIndexMgr->index)[i] = index;
return;
}
}
// maybe config change
// ASSERT(0);
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, index mgr set for %s:%d, index:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port,
index);
sError("vgId:%d, indexmgr set index:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, index,
DID(pRaftId), CID(pRaftId));
}
SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pDestId) {
SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) {
for (int i = 0; i < pNode->replicaNum; i++) {
if (syncUtilSameId(&(pNode->replicasId[i]), pDestId)) {
if (syncUtilSameId(&pNode->replicasId[i], pRaftId)) {
return pNode->logReplMgrs[i];
}
}
sError("vgId:%d, indexmgr get replmgr from dnode:%d cluster:%d failed", pNode->vgId, DID(pRaftId), CID(pRaftId));
return NULL;
}
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
if (pSyncIndexMgr == NULL) {
return SYNC_INDEX_INVALID;
}
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
SyncIndex idx = (pSyncIndexMgr->index)[i];
SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
SyncIndex idx = (pIndexMgr->index)[i];
return idx;
}
}
sError("vgId:%d, indexmgr get index from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return SYNC_INDEX_INVALID;
}
void syncIndexMgrSetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t startTime) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
(pSyncIndexMgr->startTimeArr)[i] = startTime;
void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
(pIndexMgr->startTimeArr)[i] = startTime;
return;
}
}
// maybe config change
// ASSERT(0);
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, index mgr set for %s:%d, start-time:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port,
startTime);
sError("vgId:%d, indexmgr set start-time:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId,
startTime, DID(pRaftId), CID(pRaftId));
}
int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
int64_t startTime = (pSyncIndexMgr->startTimeArr)[i];
int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
int64_t startTime = (pIndexMgr->startTimeArr)[i];
return startTime;
}
}
ASSERT(0);
sError("vgId:%d, indexmgr get start-time from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return -1;
}
void syncIndexMgrSetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t recvTime) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
(pSyncIndexMgr->recvTimeArr)[i] = recvTime;
void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
(pIndexMgr->recvTimeArr)[i] = recvTime;
return;
}
}
// maybe config change
// ASSERT(0);
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, index mgr set for %s:%d, recv-time:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port,
recvTime);
sError("vgId:%d, indexmgr set recv-time:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, recvTime,
DID(pRaftId), CID(pRaftId));
}
int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
int64_t recvTime = (pSyncIndexMgr->recvTimeArr)[i];
int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
int64_t recvTime = (pIndexMgr->recvTimeArr)[i];
return recvTime;
}
}
sError("vgId:%d, indexmgr get recv-time from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return -1;
}
void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
(pSyncIndexMgr->privateTerm)[i] = term;
void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
(pIndexMgr->privateTerm)[i] = term;
return;
}
}
// maybe config change
// ASSERT(0);
char host[128];
uint16_t port;
syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port);
sError("vgId:%d, index mgr set for %s:%d, term:%" PRIu64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, term);
sError("vgId:%d, indexmgr set term:%" PRId64 " for dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, term,
DID(pRaftId), CID(pRaftId));
}
SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) {
SyncTerm term = (pSyncIndexMgr->privateTerm)[i];
SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) {
for (int i = 0; i < pIndexMgr->replicaNum; ++i) {
if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) {
SyncTerm term = (pIndexMgr->privateTerm)[i];
return term;
}
}
ASSERT(0);
sError("vgId:%d, indexmgr get term from dnode:%d cluster:%d failed", pIndexMgr->pNode->vgId, DID(pRaftId),
CID(pRaftId));
return -1;
}

302
source/libs/sync/src/syncMain.c
View File

@ -306,15 +306,10 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) {
for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {
int64_t matchIndex = syncIndexMgrGetIndex(pSyncNode->pMatchIndex, &(pSyncNode->peersId[i]));
if (lastApplyIndex > matchIndex) {
do {
char host[64];
uint16_t port;
syncUtilU642Addr(pSyncNode->peersId[i].addr, host, sizeof(host), &port);
sNTrace(pSyncNode,
"new-snapshot-index:%" PRId64 " is greater than match-index:%" PRId64
" of %s:%d, do not delete wal",
lastApplyIndex, matchIndex, host, port);
} while (0);
sNTrace(pSyncNode,
"new-snapshot-index:%" PRId64 " is greater than match-index:%" PRId64
" of dnode:%d, do not delete wal",
lastApplyIndex, matchIndex, DID(&pSyncNode->peersId[i]));
syncNodeRelease(pSyncNode);
return 0;
@ -556,7 +551,7 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) {
(void)syncBuildLeaderTransfer(&rpcMsg, pSyncNode->vgId);
SyncLeaderTransfer* pMsg = rpcMsg.pCont;
pMsg->newLeaderId.addr = syncUtilAddr2U64(newLeader.nodeFqdn, newLeader.nodePort);
pMsg->newLeaderId.addr = SYNC_ADDR(&newLeader);
pMsg->newLeaderId.vgId = pSyncNode->vgId;
pMsg->newNodeInfo = newLeader;
@ -622,9 +617,9 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) {
return -1;
}
ASSERT(rid == pSyncNode->rid);
sMeta->lastConfigIndex = pSyncNode->pRaftCfg->lastConfigIndex;
sMeta->lastConfigIndex = pSyncNode->raftCfg.lastConfigIndex;
sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->pRaftCfg->lastConfigIndex);
sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->raftCfg.lastConfigIndex);
syncNodeRelease(pSyncNode);
return 0;
@ -637,13 +632,13 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct
}
ASSERT(rid == pSyncNode->rid);
ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0];
ASSERT(pSyncNode->raftCfg.configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0];
for (int32_t i = 0; i < pSyncNode->pRaftCfg->configIndexCount; ++i) {
if ((pSyncNode->pRaftCfg->configIndexArr)[i] > lastIndex &&
(pSyncNode->pRaftCfg->configIndexArr)[i] <= snapshotIndex) {
lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.configIndexCount; ++i) {
if ((pSyncNode->raftCfg.configIndexArr)[i] > lastIndex &&
(pSyncNode->raftCfg.configIndexArr)[i] <= snapshotIndex) {
lastIndex = (pSyncNode->raftCfg.configIndexArr)[i];
}
}
sMeta->lastConfigIndex = lastIndex;
@ -656,13 +651,13 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct
#endif
SyncIndex syncNodeGetSnapshotConfigIndex(SSyncNode* pSyncNode, SyncIndex snapshotLastApplyIndex) {
ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0];
ASSERT(pSyncNode->raftCfg.configIndexCount >= 1);
SyncIndex lastIndex = (pSyncNode->raftCfg.configIndexArr)[0];
for (int32_t i = 0; i < pSyncNode->pRaftCfg->configIndexCount; ++i) {
if ((pSyncNode->pRaftCfg->configIndexArr)[i] > lastIndex &&
(pSyncNode->pRaftCfg->configIndexArr)[i] <= snapshotLastApplyIndex) {
lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.configIndexCount; ++i) {
if ((pSyncNode->raftCfg.configIndexArr)[i] > lastIndex &&
(pSyncNode->raftCfg.configIndexArr)[i] <= snapshotLastApplyIndex) {
lastIndex = (pSyncNode->raftCfg.configIndexArr)[i];
}
}
sTrace("vgId:%d, sync get last config index, index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId,
@ -677,15 +672,15 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) {
SSyncNode* pSyncNode = syncNodeAcquire(rid);
if (pSyncNode == NULL) return;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
SEp* pEp = &pEpSet->eps[i];
tstrncpy(pEp->fqdn, pSyncNode->pRaftCfg->cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN);
pEp->port = (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodePort;
tstrncpy(pEp->fqdn, pSyncNode->raftCfg.cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN);
pEp->port = (pSyncNode->raftCfg.cfg.nodeInfo)[i].nodePort;
pEpSet->numOfEps++;
sDebug("vgId:%d, sync get retry epset, index:%d %s:%d", pSyncNode->vgId, i, pEp->fqdn, pEp->port);
}
if (pEpSet->numOfEps > 0) {
pEpSet->inUse = (pSyncNode->pRaftCfg->cfg.myIndex + 1) % pEpSet->numOfEps;
pEpSet->inUse = (pSyncNode->raftCfg.cfg.myIndex + 1) % pEpSet->numOfEps;
}
sInfo("vgId:%d, sync get retry epset numOfEps:%d inUse:%d", pSyncNode->vgId, pEpSet->numOfEps, pEpSet->inUse);
@ -849,60 +844,57 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
}
}
snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP);
if (!taosCheckExistFile(pSyncNode->configPath)) {
// create a new raft config file
SRaftCfgMeta meta = {0};
meta.isStandBy = pSyncInfo->isStandBy;
meta.snapshotStrategy = pSyncInfo->snapshotStrategy;
meta.lastConfigIndex = SYNC_INDEX_INVALID;
meta.batchSize = pSyncInfo->batchSize;
if (raftCfgCreateFile(&pSyncInfo->syncCfg, meta, pSyncNode->configPath) != 0) {
sError("vgId:%d, failed to create raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
if (pSyncInfo->syncCfg.replicaNum == 0) {
sInfo("vgId:%d, sync config not input", pSyncNode->vgId);
pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
}
} else {
// update syncCfg by raft_config.json
pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
if (pSyncNode->pRaftCfg == NULL) {
sError("vgId:%d, failed to open raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->pRaftCfg->cfg, &pSyncInfo->syncCfg)) {
sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId);
pSyncNode->pRaftCfg->cfg = pSyncInfo->syncCfg;
if (raftCfgPersist(pSyncNode->pRaftCfg) != 0) {
sError("vgId:%d, failed to persist raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
} else {
sInfo("vgId:%d, use sync config from raft cfg file", pSyncNode->vgId);
pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg;
}
raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
}
// init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId;
SSyncCfg* pCfg = &pSyncInfo->syncCfg;
sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SNodeInfo* pNode = &pCfg->nodeInfo[i];
sInfo("vgId:%d, index:%d ep:%s:%u", pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort);
}
memcpy(pSyncNode->path, pSyncInfo->path, sizeof(pSyncNode->path));
snprintf(pSyncNode->raftStorePath, sizeof(pSyncNode->raftStorePath), "%s%sraft_store.json", pSyncInfo->path,
TD_DIRSEP);
snprintf(pSyncNode->configPath, sizeof(pSyncNode->configPath), "%s%sraft_config.json", pSyncInfo->path, TD_DIRSEP);
if (!taosCheckExistFile(pSyncNode->configPath)) {
// create a new raft config file
pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy;
pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy;
pSyncNode->raftCfg.lastConfigIndex = SYNC_INDEX_INVALID;
pSyncNode->raftCfg.batchSize = pSyncInfo->batchSize;
pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg;
pSyncNode->raftCfg.configIndexCount = 1;
pSyncNode->raftCfg.configIndexArr[0] = -1;
if (syncWriteCfgFile(pSyncNode) != 0) {
sError("vgId:%d, failed to create sync cfg file", pSyncNode->vgId);
goto _error;
}
} else {
// update syncCfg by raft_config.json
if (syncReadCfgFile(pSyncNode) != 0) {
sError("vgId:%d, failed to read sync cfg file", pSyncNode->vgId);
goto _error;
}
if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) {
sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId);
pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg;
if (syncWriteCfgFile(pSyncNode) != 0) {
sError("vgId:%d, failed to write sync cfg file", pSyncNode->vgId);
goto _error;
}
} else {
sInfo("vgId:%d, use sync config from sync cfg file", pSyncNode->vgId);
pSyncInfo->syncCfg = pSyncNode->raftCfg.cfg;
}
}
// init by SSyncInfo
pSyncNode->vgId = pSyncInfo->vgId;
SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg;
sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SNodeInfo* pNode = &pCfg->nodeInfo[i];
(void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort);
sInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pSyncNode->vgId, i, pNode->nodeFqdn, pNode->nodePort,
pNode->nodeId, pNode->clusterId);
}
pSyncNode->pWal = pSyncInfo->pWal;
pSyncNode->msgcb = pSyncInfo->msgcb;
pSyncNode->syncSendMSg = pSyncInfo->syncSendMSg;
@ -916,26 +908,20 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
goto _error;
}
// init raft config
pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath);
if (pSyncNode->pRaftCfg == NULL) {
sError("vgId:%d, failed to open raft cfg file at %s", pSyncNode->vgId, pSyncNode->configPath);
goto _error;
}
// init internal
pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex];
pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex];
if (!syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId)) {
sError("vgId:%d, failed to determine my raft member id", pSyncNode->vgId);
goto _error;
}
// init peersNum, peers, peersId
pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1;
pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1;
int32_t j = 0;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
if (i != pSyncNode->pRaftCfg->cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
if (i != pSyncNode->raftCfg.cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i];
syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]);
j++;
}
}
@ -947,9 +933,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
}
// init replicaNum, replicasId
pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
if (!syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) {
pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
if (!syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) {
sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i);
goto _error;
}
@ -958,7 +944,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) {
// init raft algorithm
pSyncNode->pFsm = pSyncInfo->pFsm;
pSyncInfo->pFsm = NULL;
pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum);
pSyncNode->leaderCache = EMPTY_RAFT_ID;
// init life cycle outside
@ -1295,8 +1281,6 @@ void syncNodeClose(SSyncNode* pSyncNode) {
pSyncNode->pLogStore = NULL;
syncLogBufferDestroy(pSyncNode->pLogBuf);
pSyncNode->pLogBuf = NULL;
raftCfgClose(pSyncNode->pRaftCfg);
pSyncNode->pRaftCfg = NULL;
syncNodeStopPingTimer(pSyncNode);
syncNodeStopElectTimer(pSyncNode);
@ -1332,7 +1316,7 @@ void syncNodeClose(SSyncNode* pSyncNode) {
taosMemoryFree(pSyncNode);
}
ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->pRaftCfg->snapshotStrategy; }
ESyncStrategy syncNodeStrategy(SSyncNode* pSyncNode) { return pSyncNode->raftCfg.snapshotStrategy; }
// timer control --------------
int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) {
@ -1394,7 +1378,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) {
void syncNodeResetElectTimer(SSyncNode* pSyncNode) {
int32_t electMS;
if (pSyncNode->pRaftCfg->isStandBy) {
if (pSyncNode->raftCfg.isStandBy) {
electMS = TIMER_MAX_MS;
} else {
electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine);
@ -1463,55 +1447,46 @@ int32_t syncNodeRestartHeartbeatTimer(SSyncNode* pSyncNode) {
return 0;
}
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
SEpSet epSet;
syncUtilRaftId2EpSet(destRaftId, &epSet);
int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pNode, SRpcMsg* pMsg) {
SEpSet* epSet = NULL;
for (int32_t i = 0; i < pNode->peersNum; ++i) {
if (destRaftId->addr == pNode->peersId[i].addr) {
epSet = &pNode->peersEpset[i];
break;
}
}
if (pSyncNode->syncSendMSg != NULL) {
if (pNode->syncSendMSg != NULL && epSet != NULL) {
syncUtilMsgHtoN(pMsg->pCont);
pMsg->info.noResp = 1;
return pSyncNode->syncSendMSg(&epSet, pMsg);
return pNode->syncSendMSg(epSet, pMsg);
} else {
sError("vgId:%d, sync send msg by id error, fp-send-msg is null", pSyncNode->vgId);
sError("vgId:%d, sync send msg by id error, fp:%p epset:%p", pNode->vgId, pNode->syncSendMSg, epSet);
rpcFreeCont(pMsg->pCont);
terrno = TSDB_CODE_SYN_INTERNAL_ERROR;
return -1;
}
}
int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg) {
SEpSet epSet;
syncUtilNodeInfo2EpSet(nodeInfo, &epSet);
if (pSyncNode->syncSendMSg != NULL) {
// htonl
syncUtilMsgHtoN(pMsg->pCont);
pMsg->info.noResp = 1;
pSyncNode->syncSendMSg(&epSet, pMsg);
} else {
sError("vgId:%d, sync send msg by info error, fp-send-msg is null", pSyncNode->vgId);
}
return 0;
}
inline bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config) {
inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) {
bool b1 = false;
bool b2 = false;
for (int32_t i = 0; i < config->replicaNum; ++i) {
if (strcmp((config->nodeInfo)[i].nodeFqdn, pSyncNode->myNodeInfo.nodeFqdn) == 0 &&
(config->nodeInfo)[i].nodePort == pSyncNode->myNodeInfo.nodePort) {
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
if (strcmp(pCfg->nodeInfo[i].nodeFqdn, pNode->myNodeInfo.nodeFqdn) == 0 &&
pCfg->nodeInfo[i].nodePort == pNode->myNodeInfo.nodePort) {
b1 = true;
break;
}
}
for (int32_t i = 0; i < config->replicaNum; ++i) {
SRaftId raftId;
raftId.addr = syncUtilAddr2U64((config->nodeInfo)[i].nodeFqdn, (config->nodeInfo)[i].nodePort);
raftId.vgId = pSyncNode->vgId;
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SRaftId raftId = {
.addr = SYNC_ADDR(&pCfg->nodeInfo[i]),
.vgId = pNode->vgId,
};
if (syncUtilSameId(&raftId, &(pSyncNode->myRaftId))) {
if (syncUtilSameId(&raftId, &pNode->myRaftId)) {
b2 = true;
break;
}
@ -1535,14 +1510,14 @@ static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg
}
void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncIndex lastConfigChangeIndex) {
SSyncCfg oldConfig = pSyncNode->pRaftCfg->cfg;
SSyncCfg oldConfig = pSyncNode->raftCfg.cfg;
if (!syncIsConfigChanged(&oldConfig, pNewConfig)) {
sInfo("vgId:1, sync not reconfig since not changed");
return;
}
pSyncNode->pRaftCfg->cfg = *pNewConfig;
pSyncNode->pRaftCfg->lastConfigIndex = lastConfigChangeIndex;
pSyncNode->raftCfg.cfg = *pNewConfig;
pSyncNode->raftCfg.lastConfigIndex = lastConfigChangeIndex;
pSyncNode->configChangeNum++;
@ -1565,21 +1540,18 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// log begin config change
char oldCfgStr[1024] = {0};
char newCfgStr[1024] = {0};
syncCfg2SimpleStr(&oldConfig, oldCfgStr, sizeof(oldCfgStr));
syncCfg2SimpleStr(pNewConfig, oldCfgStr, sizeof(oldCfgStr));
sNInfo(pSyncNode, "begin do config change, from %s to %s", oldCfgStr, oldCfgStr);
sNInfo(pSyncNode, "begin do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum,
pNewConfig->replicaNum);
if (IamInNew) {
pSyncNode->pRaftCfg->isStandBy = 0; // change isStandBy to normal
pSyncNode->raftCfg.isStandBy = 0; // change isStandBy to normal
}
if (isDrop) {
pSyncNode->pRaftCfg->isStandBy = 1; // set standby
pSyncNode->raftCfg.isStandBy = 1; // set standby
}
// add last config index
raftCfgAddConfigIndex(pSyncNode->pRaftCfg, lastConfigChangeIndex);
syncAddCfgIndex(pSyncNode, lastConfigChangeIndex);
if (IamInNew) {
//-----------------------------------------
@ -1596,15 +1568,16 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// init internal
pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex];
pSyncNode->myNodeInfo = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex];
syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId);
// init peersNum, peers, peersId
pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1;
pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1;
int32_t j = 0;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
if (i != pSyncNode->pRaftCfg->cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i];
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
if (i != pSyncNode->raftCfg.cfg.myIndex) {
pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i];
syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]);
j++;
}
}
@ -1613,13 +1586,13 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// init replicaNum, replicasId
pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
syncUtilNodeInfo2RaftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum;
for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) {
syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
}
// update quorum first
pSyncNode->quorum = syncUtilQuorum(pSyncNode->pRaftCfg->cfg.replicaNum);
pSyncNode->quorum = syncUtilQuorum(pSyncNode->raftCfg.cfg.replicaNum);
syncIndexMgrUpdate(pSyncNode->pNextIndex, pSyncNode);
syncIndexMgrUpdate(pSyncNode->pMatchIndex, pSyncNode);
@ -1639,11 +1612,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
bool reset = false;
for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) {
if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) {
char host[128];
uint16_t port;
syncUtilU642Addr((pSyncNode->replicasId)[i].addr, host, sizeof(host), &port);
sNTrace(pSyncNode, "snapshot sender reset for: %" PRId64 ", newIndex:%d, %s:%d, %p",
(pSyncNode->replicasId)[i].addr, i, host, port, oldSenders[j]);
sNTrace(pSyncNode, "snapshot sender reset for:%" PRId64 ", newIndex:%d, dnode:%d, %p",
(pSyncNode->replicasId)[i].addr, i, DID(&pSyncNode->replicasId[i]), oldSenders[j]);
pSyncNode->senders[i] = oldSenders[j];
oldSenders[j] = NULL;
@ -1653,8 +1623,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
int32_t oldreplicaIndex = pSyncNode->senders[i]->replicaIndex;
pSyncNode->senders[i]->replicaIndex = i;
sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, %s:%d, %p, reset:%d", oldreplicaIndex,
i, host, port, pSyncNode->senders[i], reset);
sNTrace(pSyncNode, "snapshot sender udpate replicaIndex from %d to %d, dnode:%d, %p, reset:%d",
oldreplicaIndex, i, DID(&pSyncNode->replicasId[i]), pSyncNode->senders[i], reset);
break;
}
@ -1686,33 +1656,30 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde
}
// persist cfg
raftCfgPersist(pSyncNode->pRaftCfg);
syncWriteCfgFile(pSyncNode);
char tmpbuf[1024] = {0};
snprintf(tmpbuf, sizeof(tmpbuf), "config change from %d to %d, index:%" PRId64 ", %s --> %s",
oldConfig.replicaNum, pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr);
// change isStandBy to normal (election timeout)
if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) {
syncNodeBecomeLeader(pSyncNode, tmpbuf);
syncNodeBecomeLeader(pSyncNode, "");
// Raft 3.6.2 Committing entries from previous terms
syncNodeAppendNoop(pSyncNode);
// syncMaybeAdvanceCommitIndex(pSyncNode);
} else {
syncNodeBecomeFollower(pSyncNode, tmpbuf);
syncNodeBecomeFollower(pSyncNode, "");
}
} else {
// persist cfg
raftCfgPersist(pSyncNode->pRaftCfg);
sNInfo(pSyncNode, "do not config change from %d to %d, index:%" PRId64 ", %s --> %s", oldConfig.replicaNum,
pNewConfig->replicaNum, lastConfigChangeIndex, oldCfgStr, newCfgStr);
syncWriteCfgFile(pSyncNode);
sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum);
}
_END:
// log end config change
sNInfo(pSyncNode, "end do config change, from %s to %s", oldCfgStr, newCfgStr);
sNInfo(pSyncNode, "end do config change, from %d to %d", pSyncNode->vgId, oldConfig.replicaNum,
pNewConfig->replicaNum);
}
// raft state change --------------
@ -2897,9 +2864,10 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p
int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg) {
for (int32_t i = 0; i < pNewCfg->replicaNum; ++i) {
SRaftId raftId;
raftId.addr = syncUtilAddr2U64((pNewCfg->nodeInfo)[i].nodeFqdn, (pNewCfg->nodeInfo)[i].nodePort);
raftId.vgId = ths->vgId;
SRaftId raftId = {
.addr = SYNC_ADDR(&pNewCfg->nodeInfo[i]),
.vgId = ths->vgId,
};
if (syncUtilSameId(&(ths->myRaftId), &raftId)) {
pNewCfg->myIndex = i;

23
source/libs/sync/src/syncPipeline.c
View File

@ -679,18 +679,15 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod
SSyncLogBuffer* pBuf = pNode->pLogBuf;
SRaftId destId = pMsg->srcId;
ASSERT(pMgr->restored == false);
char host[64];
uint16_t port;
syncUtilU642Addr(destId.addr, host, sizeof(host), &port);
if (pMgr->endIndex == 0) {
ASSERT(pMgr->startIndex == 0);
ASSERT(pMgr->matchIndex == 0);
if (pMsg->matchIndex < 0) {
pMgr->restored = true;
sInfo("vgId:%d, sync log repl mgr restored. peer: %s:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
pNode->vgId, host, port, destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
return 0;
}
@ -705,21 +702,21 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod
if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) {
pMgr->matchIndex = pMsg->matchIndex;
pMgr->restored = true;
sInfo("vgId:%d, sync log repl mgr restored. peer: %s:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64
", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")",
pNode->vgId, host, port, destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex,
pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex);
return 0;
}
if (pMsg->success == false && pMsg->matchIndex >= pMsg->lastSendIndex) {
sWarn("vgId:%d, failed to rollback match index. peer: %s:%d, match index: %" PRId64 ", last sent: %" PRId64,
pNode->vgId, host, port, pMsg->matchIndex, pMsg->lastSendIndex);
sWarn("vgId:%d, failed to rollback match index. peer: dnode:%d, match index: %" PRId64 ", last sent: %" PRId64,
pNode->vgId, DID(&destId), pMsg->matchIndex, pMsg->lastSendIndex);
if (syncNodeStartSnapshot(pNode, &destId) < 0) {
sError("vgId:%d, failed to start snapshot for peer %s:%d", pNode->vgId, host, port);
sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId));
return -1;
}
sInfo("vgId:%d, snapshot replication to peer %s:%d", pNode->vgId, host, port);
sInfo("vgId:%d, snapshot replication to peer dnode:%d", pNode->vgId, DID(&destId));
return 0;
}
}
@ -734,10 +731,10 @@ int32_t syncLogReplMgrProcessReplyInRecoveryMode(SSyncLogReplMgr* pMgr, SSyncNod
if (term < 0 || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) {
ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST);
if (syncNodeStartSnapshot(pNode, &destId) < 0) {
sError("vgId:%d, failed to start snapshot for peer %s:%d", pNode->vgId, host, port);
sError("vgId:%d, failed to start snapshot for peer dnode:%d", pNode->vgId, DID(&destId));
return -1;
}
sInfo("vgId:%d, snapshot replication to peer %s:%d", pNode->vgId, host, port);
sInfo("vgId:%d, snapshot replication to peer dnode:%d", pNode->vgId, DID(&destId));
return 0;
}

540
source/libs/sync/src/syncRaftCfg.c
View File

@ -16,382 +16,226 @@
#define _DEFAULT_SOURCE
#include "syncRaftCfg.h"
#include "syncUtil.h"
#include "tjson.h"
// file must already exist!
SRaftCfgIndex *raftCfgIndexOpen(const char *path) {
SRaftCfgIndex *pRaftCfgIndex = taosMemoryMalloc(sizeof(SRaftCfg));
snprintf(pRaftCfgIndex->path, sizeof(pRaftCfgIndex->path), "%s", path);
static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) {
SSyncCfg *pCfg = (SSyncCfg *)pObj;
if (tjsonAddDoubleToObject(pJson, "replicaNum", pCfg->replicaNum) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "myIndex", pCfg->myIndex) < 0) return -1;
pRaftCfgIndex->pFile = taosOpenFile(pRaftCfgIndex->path, TD_FILE_READ | TD_FILE_WRITE);
ASSERT(pRaftCfgIndex->pFile != NULL);
taosLSeekFile(pRaftCfgIndex->pFile, 0, SEEK_SET);
int32_t bufLen = MAX_CONFIG_INDEX_COUNT * 16;
char *pBuf = taosMemoryMalloc(bufLen);
memset(pBuf, 0, bufLen);
int64_t len = taosReadFile(pRaftCfgIndex->pFile, pBuf, bufLen);
ASSERT(len > 0);
int32_t ret = raftCfgIndexFromStr(pBuf, pRaftCfgIndex);
ASSERT(ret == 0);
taosMemoryFree(pBuf);
return pRaftCfgIndex;
}
int32_t raftCfgIndexClose(SRaftCfgIndex *pRaftCfgIndex) {
if (pRaftCfgIndex != NULL) {
int64_t ret = taosCloseFile(&(pRaftCfgIndex->pFile));
ASSERT(ret == 0);
taosMemoryFree(pRaftCfgIndex);
}
return 0;
}
int32_t raftCfgIndexPersist(SRaftCfgIndex *pRaftCfgIndex) {
ASSERT(pRaftCfgIndex != NULL);
char *s = raftCfgIndex2Str(pRaftCfgIndex);
taosLSeekFile(pRaftCfgIndex->pFile, 0, SEEK_SET);
int64_t ret = taosWriteFile(pRaftCfgIndex->pFile, s, strlen(s) + 1);
ASSERT(ret == strlen(s) + 1);
taosMemoryFree(s);
taosFsyncFile(pRaftCfgIndex->pFile);
return 0;
}
int32_t raftCfgIndexAddConfigIndex(SRaftCfgIndex *pRaftCfgIndex, SyncIndex configIndex) {
ASSERT(pRaftCfgIndex->configIndexCount <= MAX_CONFIG_INDEX_COUNT);
(pRaftCfgIndex->configIndexArr)[pRaftCfgIndex->configIndexCount] = configIndex;
++(pRaftCfgIndex->configIndexCount);
return 0;
}
cJSON *raftCfgIndex2Json(SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pRoot = cJSON_CreateObject();
cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfgIndex->configIndexCount);
cJSON *pIndexArr = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr);
for (int i = 0; i < pRaftCfgIndex->configIndexCount; ++i) {
char buf64[128];
snprintf(buf64, sizeof(buf64), "%" PRId64, (pRaftCfgIndex->configIndexArr)[i]);
cJSON *pIndexObj = cJSON_CreateObject();
cJSON_AddStringToObject(pIndexObj, "index", buf64);
cJSON_AddItemToArray(pIndexArr, pIndexObj);
}
cJSON *pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "SRaftCfgIndex", pRoot);
return pJson;
}
char *raftCfgIndex2Str(SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pJson = raftCfgIndex2Json(pRaftCfgIndex);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
int32_t raftCfgIndexFromJson(const cJSON *pRoot, SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pJson = cJSON_GetObjectItem(pRoot, "SRaftCfgIndex");
cJSON *pJsonConfigIndexCount = cJSON_GetObjectItem(pJson, "configIndexCount");
pRaftCfgIndex->configIndexCount = cJSON_GetNumberValue(pJsonConfigIndexCount);
cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr");
int arraySize = cJSON_GetArraySize(pIndexArr);
ASSERT(arraySize == pRaftCfgIndex->configIndexCount);
memset(pRaftCfgIndex->configIndexArr, 0, sizeof(pRaftCfgIndex->configIndexArr));
for (int i = 0; i < arraySize; ++i) {
cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i);
ASSERT(pIndexObj != NULL);
cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index");
ASSERT(cJSON_IsString(pIndex));
(pRaftCfgIndex->configIndexArr)[i] = atoll(pIndex->valuestring);
SJson *nodeInfo = tjsonCreateArray();
if (nodeInfo == NULL) return -1;
if (tjsonAddItemToObject(pJson, "nodeInfo", nodeInfo) < 0) return -1;
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SJson *info = tjsonCreateObject();
if (info == NULL) return -1;
if (tjsonAddDoubleToObject(info, "nodePort", pCfg->nodeInfo[i].nodePort) < 0) return -1;
if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn) < 0) return -1;
if (tjsonAddIntegerToObject(info, "nodeId", pCfg->nodeInfo[i].nodeId) < 0) return -1;
if (tjsonAddIntegerToObject(info, "clusterId", pCfg->nodeInfo[i].clusterId) < 0) return -1;
if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1;
}
return 0;
}
int32_t raftCfgIndexFromStr(const char *s, SRaftCfgIndex *pRaftCfgIndex) {
cJSON *pRoot = cJSON_Parse(s);
ASSERT(pRoot != NULL);
static int32_t syncEncodeRaftCfg(const void *pObj, SJson *pJson) {
SRaftCfg *pCfg = (SRaftCfg *)pObj;
if (tjsonAddObject(pJson, "SSyncCfg", syncEncodeSyncCfg, (void *)&pCfg->cfg) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "isStandBy", pCfg->isStandBy) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "snapshotStrategy", pCfg->snapshotStrategy) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "batchSize", pCfg->batchSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "lastConfigIndex", pCfg->lastConfigIndex) < 0) return -1;
if (tjsonAddDoubleToObject(pJson, "configIndexCount", pCfg->configIndexCount) < 0) return -1;
int32_t ret = raftCfgIndexFromJson(pRoot, pRaftCfgIndex);
ASSERT(ret == 0);
SJson *configIndexArr = tjsonCreateArray();
if (configIndexArr == NULL) return -1;
if (tjsonAddItemToObject(pJson, "configIndexArr", configIndexArr) < 0) return -1;
for (int32_t i = 0; i < pCfg->configIndexCount; ++i) {
SJson *configIndex = tjsonCreateObject();
if (configIndex == NULL) return -1;
if (tjsonAddIntegerToObject(configIndex, "index", pCfg->configIndexArr[i]) < 0) return -1;
if (tjsonAddItemToArray(configIndexArr, configIndex) < 0) return -1;
}
cJSON_Delete(pRoot);
return 0;
}
int32_t raftCfgIndexCreateFile(const char *path) {
TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
int32_t syncWriteCfgFile(SSyncNode *pNode) {
int32_t code = -1;
char *buffer = NULL;
SJson *pJson = NULL;
TdFilePtr pFile = NULL;
const char *realfile = pNode->configPath;
SRaftCfg *pCfg = &pNode->raftCfg;
char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s.bak", realfile);
pFile = taosOpenFile(file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
if (pFile == NULL) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
int32_t sysErr = errno;
const char *sysErrStr = strerror(errno);
sError("create raft cfg index file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr,
sysErrStr);
ASSERT(0);
return -1;
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to open sync cfg file:%s since %s", pNode->vgId, realfile, terrstr());
goto _OVER;
}
SRaftCfgIndex raftCfgIndex;
memset(raftCfgIndex.configIndexArr, 0, sizeof(raftCfgIndex.configIndexArr));
raftCfgIndex.configIndexCount = 1;
raftCfgIndex.configIndexArr[0] = -1;
terrno = TSDB_CODE_OUT_OF_MEMORY;
pJson = tjsonCreateObject();
if (pJson == NULL) goto _OVER;
if (tjsonAddObject(pJson, "RaftCfg", syncEncodeRaftCfg, pCfg) < 0) goto _OVER;
char *s = raftCfgIndex2Str(&raftCfgIndex);
int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1);
ASSERT(ret == strlen(s) + 1);
buffer = tjsonToString(pJson);
if (buffer == NULL) goto _OVER;
taosMemoryFree(s);
int32_t len = strlen(buffer);
if (taosWriteFile(pFile, buffer, len) <= 0) goto _OVER;
if (taosFsyncFile(pFile) < 0) goto _OVER;
taosCloseFile(&pFile);
return 0;
}
// ---------------------------------------
// file must already exist!
SRaftCfg *raftCfgOpen(const char *path) {
SRaftCfg *pCfg = taosMemoryMalloc(sizeof(SRaftCfg));
snprintf(pCfg->path, sizeof(pCfg->path), "%s", path);
pCfg->pFile = taosOpenFile(pCfg->path, TD_FILE_READ | TD_FILE_WRITE);
ASSERT(pCfg->pFile != NULL);
taosLSeekFile(pCfg->pFile, 0, SEEK_SET);
char buf[CONFIG_FILE_LEN] = {0};
int len = taosReadFile(pCfg->pFile, buf, sizeof(buf));
ASSERT(len > 0);
int32_t ret = raftCfgFromStr(buf, pCfg);
ASSERT(ret == 0);
return pCfg;
}
int32_t raftCfgClose(SRaftCfg *pRaftCfg) {
int64_t ret = taosCloseFile(&(pRaftCfg->pFile));
ASSERT(ret == 0);
taosMemoryFree(pRaftCfg);
return 0;
}
int32_t raftCfgPersist(SRaftCfg *pRaftCfg) {
ASSERT(pRaftCfg != NULL);
char *s = raftCfg2Str(pRaftCfg);
taosLSeekFile(pRaftCfg->pFile, 0, SEEK_SET);
char buf[CONFIG_FILE_LEN] = {0};
memset(buf, 0, sizeof(buf));
if (strlen(s) + 1 > CONFIG_FILE_LEN) {
sError("too long config str:%s", s);
ASSERT(0);
if (taosRenameFile(file, realfile) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to rename sync cfg file:%s to %s since %s", pNode->vgId, file, realfile, terrstr());
goto _OVER;
}
snprintf(buf, sizeof(buf), "%s", s);
int64_t ret = taosWriteFile(pRaftCfg->pFile, buf, sizeof(buf));
ASSERT(ret == sizeof(buf));
code = 0;
sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d", pNode->vgId, realfile, len);
// int64_t ret = taosWriteFile(pRaftCfg->pFile, s, strlen(s) + 1);
// ASSERT(ret == strlen(s) + 1);
_OVER:
if (pJson != NULL) tjsonDelete(pJson);
if (buffer != NULL) taosMemoryFree(buffer);
if (pFile != NULL) taosCloseFile(&pFile);
taosMemoryFree(s);
taosFsyncFile(pRaftCfg->pFile);
return 0;
}
int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex) {
ASSERT(pRaftCfg->configIndexCount <= MAX_CONFIG_INDEX_COUNT);
(pRaftCfg->configIndexArr)[pRaftCfg->configIndexCount] = configIndex;
++(pRaftCfg->configIndexCount);
return 0;
}
cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
char u64buf[128] = {0};
cJSON *pRoot = cJSON_CreateObject();
if (pSyncCfg != NULL) {
cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncCfg->replicaNum);
cJSON_AddNumberToObject(pRoot, "myIndex", pSyncCfg->myIndex);
cJSON *pNodeInfoArr = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "nodeInfo", pNodeInfoArr);
for (int i = 0; i < pSyncCfg->replicaNum; ++i) {
cJSON *pNodeInfo = cJSON_CreateObject();
cJSON_AddNumberToObject(pNodeInfo, "nodePort", ((pSyncCfg->nodeInfo)[i]).nodePort);
cJSON_AddStringToObject(pNodeInfo, "nodeFqdn", ((pSyncCfg->nodeInfo)[i]).nodeFqdn);
cJSON_AddItemToArray(pNodeInfoArr, pNodeInfo);
}
if (code != 0) {
sError("vgId:%d, failed to write sync cfg file:%s since %s", pNode->vgId, realfile, terrstr());
}
return pRoot;
}
int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) {
memset(pSyncCfg, 0, sizeof(SSyncCfg));
// cJSON *pJson = cJSON_GetObjectItem(pRoot, "SSyncCfg");
const cJSON *pJson = pRoot;
cJSON *pReplicaNum = cJSON_GetObjectItem(pJson, "replicaNum");
ASSERT(cJSON_IsNumber(pReplicaNum));
pSyncCfg->replicaNum = cJSON_GetNumberValue(pReplicaNum);
cJSON *pMyIndex = cJSON_GetObjectItem(pJson, "myIndex");
ASSERT(cJSON_IsNumber(pMyIndex));
pSyncCfg->myIndex = cJSON_GetNumberValue(pMyIndex);
cJSON *pNodeInfoArr = cJSON_GetObjectItem(pJson, "nodeInfo");
int arraySize = cJSON_GetArraySize(pNodeInfoArr);
ASSERT(arraySize == pSyncCfg->replicaNum);
for (int i = 0; i < arraySize; ++i) {
cJSON *pNodeInfo = cJSON_GetArrayItem(pNodeInfoArr, i);
ASSERT(pNodeInfo != NULL);
cJSON *pNodePort = cJSON_GetObjectItem(pNodeInfo, "nodePort");
ASSERT(cJSON_IsNumber(pNodePort));
((pSyncCfg->nodeInfo)[i]).nodePort = cJSON_GetNumberValue(pNodePort);
cJSON *pNodeFqdn = cJSON_GetObjectItem(pNodeInfo, "nodeFqdn");
ASSERT(cJSON_IsString(pNodeFqdn));
snprintf(((pSyncCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pSyncCfg->nodeInfo)[i]).nodeFqdn), "%s",
pNodeFqdn->valuestring);
}
return 0;
}
cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
cJSON *pRoot = cJSON_CreateObject();
cJSON_AddItemToObject(pRoot, "SSyncCfg", syncCfg2Json(&(pRaftCfg->cfg)));
cJSON_AddNumberToObject(pRoot, "isStandBy", pRaftCfg->isStandBy);
cJSON_AddNumberToObject(pRoot, "snapshotStrategy", pRaftCfg->snapshotStrategy);
cJSON_AddNumberToObject(pRoot, "batchSize", pRaftCfg->batchSize);
char buf64[128];
snprintf(buf64, sizeof(buf64), "%" PRId64, pRaftCfg->lastConfigIndex);
cJSON_AddStringToObject(pRoot, "lastConfigIndex", buf64);
cJSON_AddNumberToObject(pRoot, "configIndexCount", pRaftCfg->configIndexCount);
cJSON *pIndexArr = cJSON_CreateArray();
cJSON_AddItemToObject(pRoot, "configIndexArr", pIndexArr);
for (int i = 0; i < pRaftCfg->configIndexCount; ++i) {
snprintf(buf64, sizeof(buf64), "%" PRId64, (pRaftCfg->configIndexArr)[i]);
cJSON *pIndexObj = cJSON_CreateObject();
cJSON_AddStringToObject(pIndexObj, "index", buf64);
cJSON_AddItemToArray(pIndexArr, pIndexObj);
}
cJSON *pJson = cJSON_CreateObject();
cJSON_AddItemToObject(pJson, "RaftCfg", pRoot);
return pJson;
}
char *raftCfg2Str(SRaftCfg *pRaftCfg) {
cJSON *pJson = raftCfg2Json(pRaftCfg);
char *serialized = cJSON_Print(pJson);
cJSON_Delete(pJson);
return serialized;
}
int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) {
TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
if (pFile == NULL) {
int32_t err = terrno;
const char *errStr = tstrerror(err);
int32_t sysErr = errno;
const char *sysErrStr = strerror(errno);
sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr);
return -1;
}
SRaftCfg raftCfg;
raftCfg.cfg = *pCfg;
raftCfg.isStandBy = meta.isStandBy;
raftCfg.batchSize = meta.batchSize;
raftCfg.snapshotStrategy = meta.snapshotStrategy;
raftCfg.lastConfigIndex = meta.lastConfigIndex;
raftCfg.configIndexCount = 1;
memset(raftCfg.configIndexArr, 0, sizeof(raftCfg.configIndexArr));
raftCfg.configIndexArr[0] = -1;
char *s = raftCfg2Str(&raftCfg);
char buf[CONFIG_FILE_LEN] = {0};
memset(buf, 0, sizeof(buf));
ASSERT(strlen(s) + 1 <= CONFIG_FILE_LEN);
snprintf(buf, sizeof(buf), "%s", s);
int64_t ret = taosWriteFile(pFile, buf, sizeof(buf));
ASSERT(ret == sizeof(buf));
// int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1);
// ASSERT(ret == strlen(s) + 1);
taosMemoryFree(s);
taosCloseFile(&pFile);
return 0;
}
int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) {
// memset(pRaftCfg, 0, sizeof(SRaftCfg));
cJSON *pJson = cJSON_GetObjectItem(pRoot, "RaftCfg");
cJSON *pJsonIsStandBy = cJSON_GetObjectItem(pJson, "isStandBy");
pRaftCfg->isStandBy = cJSON_GetNumberValue(pJsonIsStandBy);
cJSON *pJsonBatchSize = cJSON_GetObjectItem(pJson, "batchSize");
pRaftCfg->batchSize = cJSON_GetNumberValue(pJsonBatchSize);
cJSON *pJsonSnapshotStrategy = cJSON_GetObjectItem(pJson, "snapshotStrategy");
pRaftCfg->snapshotStrategy = cJSON_GetNumberValue(pJsonSnapshotStrategy);
cJSON *pJsonLastConfigIndex = cJSON_GetObjectItem(pJson, "lastConfigIndex");
pRaftCfg->lastConfigIndex = atoll(cJSON_GetStringValue(pJsonLastConfigIndex));
cJSON *pJsonConfigIndexCount = cJSON_GetObjectItem(pJson, "configIndexCount");
pRaftCfg->configIndexCount = cJSON_GetNumberValue(pJsonConfigIndexCount);
cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr");
int arraySize = cJSON_GetArraySize(pIndexArr);
ASSERT(arraySize == pRaftCfg->configIndexCount);
memset(pRaftCfg->configIndexArr, 0, sizeof(pRaftCfg->configIndexArr));
for (int i = 0; i < arraySize; ++i) {
cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i);
ASSERT(pIndexObj != NULL);
cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index");
ASSERT(cJSON_IsString(pIndex));
(pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring);
}
cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg");
int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg));
ASSERT(code == 0);
return code;
}
int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg) {
cJSON *pRoot = cJSON_Parse(s);
ASSERT(pRoot != NULL);
static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) {
SSyncCfg *pCfg = (SSyncCfg *)pObj;
int32_t code = 0;
int32_t ret = raftCfgFromJson(pRoot, pRaftCfg);
ASSERT(ret == 0);
tjsonGetInt32ValueFromDouble(pJson, "replicaNum", pCfg->replicaNum, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "myIndex", pCfg->myIndex, code);
if (code < 0) return -1;
SJson *nodeInfo = tjsonGetObjectItem(pJson, "nodeInfo");
if (nodeInfo == NULL) return -1;
pCfg->replicaNum = tjsonGetArraySize(nodeInfo);
for (int32_t i = 0; i < pCfg->replicaNum; ++i) {
SJson *info = tjsonGetArrayItem(nodeInfo, i);
if (info == NULL) return -1;
tjsonGetUInt16ValueFromDouble(info, "nodePort", pCfg->nodeInfo[i].nodePort, code);
if (code < 0) return -1;
code = tjsonGetStringValue(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn);
if (code < 0) return -1;
tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code);
tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code);
}
cJSON_Delete(pRoot);
return 0;
}
static int32_t syncDecodeRaftCfg(const SJson *pJson, void *pObj) {
SRaftCfg *pCfg = (SRaftCfg *)pObj;
int32_t code = 0;
if (tjsonToObject(pJson, "SSyncCfg", syncDecodeSyncCfg, (void *)&pCfg->cfg) < 0) return -1;
tjsonGetInt8ValueFromDouble(pJson, "isStandBy", pCfg->isStandBy, code);
if (code < 0) return -1;
tjsonGetInt8ValueFromDouble(pJson, "snapshotStrategy", pCfg->snapshotStrategy, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "batchSize", pCfg->batchSize, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "lastConfigIndex", pCfg->lastConfigIndex, code);
if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(pJson, "configIndexCount", pCfg->configIndexCount, code);
SJson *configIndexArr = tjsonGetObjectItem(pJson, "configIndexArr");
if (configIndexArr == NULL) return -1;
pCfg->configIndexCount = tjsonGetArraySize(configIndexArr);
for (int32_t i = 0; i < pCfg->configIndexCount; ++i) {
SJson *configIndex = tjsonGetArrayItem(configIndexArr, i);
if (configIndex == NULL) return -1;
tjsonGetNumberValue(configIndex, "index", pCfg->configIndexArr[i], code);
if (code < 0) return -1;
}
return 0;
}
int32_t syncReadCfgFile(SSyncNode *pNode) {
int32_t code = -1;
TdFilePtr pFile = NULL;
char *pData = NULL;
SJson *pJson = NULL;
const char *file = pNode->configPath;
SRaftCfg *pCfg = &pNode->raftCfg;
pFile = taosOpenFile(file, TD_FILE_READ);
if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to open sync cfg file:%s since %s", pNode->vgId, file, terrstr());
goto _OVER;
}
int64_t size = 0;
if (taosFStatFile(pFile, &size, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to fstat sync cfg file:%s since %s", pNode->vgId, file, terrstr());
goto _OVER;
}
pData = taosMemoryMalloc(size + 1);
if (pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
if (taosReadFile(pFile, pData, size) != size) {
terrno = TAOS_SYSTEM_ERROR(errno);
sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, terrstr());
goto _OVER;
}
pData[size] = '\0';
pJson = tjsonParse(pData);
if (pJson == NULL) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
if (tjsonToObject(pJson, "RaftCfg", syncDecodeRaftCfg, (void *)pCfg) < 0) {
terrno = TSDB_CODE_INVALID_JSON_FORMAT;
goto _OVER;
}
code = 0;
sInfo("vgId:%d, succceed to read sync cfg file %s", pNode->vgId, file);
_OVER:
if (pData != NULL) taosMemoryFree(pData);
if (pJson != NULL) cJSON_Delete(pJson);
if (pFile != NULL) taosCloseFile(&pFile);
if (code != 0) {
sError("vgId:%d, failed to read sync cfg file:%s since %s", pNode->vgId, file, terrstr());
}
return code;
}
int32_t syncAddCfgIndex(SSyncNode *pNode, SyncIndex cfgIndex) {
SRaftCfg *pCfg = &pNode->raftCfg;
if (pCfg->configIndexCount <= MAX_CONFIG_INDEX_COUNT) {
return -1;
}
pCfg->configIndexArr[pCfg->configIndexCount] = cfgIndex;
pCfg->configIndexCount++;
return 0;
}

7
source/libs/sync/src/syncRaftStore.c
View File

@ -112,13 +112,6 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) {
cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId);
uint64_t u64 = pRaftStore->voteFor.addr;
char host[128] = {0};
uint16_t port;
syncUtilU642Addr(u64, host, sizeof(host), &port);
cJSON_AddStringToObject(pRoot, "addr_host", host);
cJSON_AddNumberToObject(pRoot, "addr_port", port);
char *serialized = cJSON_Print(pRoot);
int len2 = strlen(serialized);
ASSERT(len2 < len);

17
source/libs/sync/src/syncReplication.c
View File

@ -107,10 +107,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh
pMsg = rpcMsg.pCont;
} else {
char host[64];
uint16_t port;
syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port);
sNError(pSyncNode, "replicate to %s:%d error, next-index:%" PRId64, host, port, nextIndex);
sNError(pSyncNode, "replicate to dnode:%d error, next-index:%" PRId64, DID(pDestId), nextIndex);
return -1;
}
}
@ -171,10 +168,7 @@ int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) {
SRaftId* pDestId = &(pSyncNode->peersId[i]);
ret = syncNodeReplicateOne(pSyncNode, pDestId, true);
if (ret != 0) {
char host[64];
int16_t port;
syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port);
sError("vgId:%d, do append entries error for %s:%d", pSyncNode->vgId, host, port);
sError("vgId:%d, do append entries error for dnode:%d", pSyncNode->vgId, DID(pDestId));
}
}
@ -183,7 +177,6 @@ int32_t syncNodeReplicateOld(SSyncNode* pSyncNode) {
int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
SyncAppendEntries* pMsg = pRpcMsg->pCont;
int32_t ret = 0;
pMsg->destId = *destRaftId;
syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg);
return 0;
@ -229,11 +222,7 @@ int32_t syncNodeMaybeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* dest
if (syncNodeNeedSendAppendEntries(pSyncNode, destRaftId, pMsg)) {
ret = syncNodeSendAppendEntries(pSyncNode, destRaftId, pRpcMsg);
} else {
char logBuf[128];
char host[64];
int16_t port;
syncUtilU642Addr(destRaftId->addr, host, sizeof(host), &port);
sNTrace(pSyncNode, "do not repcate to %s:%d for index:%" PRId64, host, port, pMsg->prevLogIndex + 1);
sNTrace(pSyncNode, "do not repcate to dnode:%d for index:%" PRId64, DID(destRaftId), pMsg->prevLogIndex + 1);
rpcFreeCont(pRpcMsg->pCont);
}

Some files were not shown because too many files have changed in this diff Show More