Merge branch '3.0' of https://github.com/taosdata/TDengine into feature/3.0_mhli
This commit is contained in:
commit
d891f31be2
|
@ -130,6 +130,9 @@ def pre_test(){
|
|||
def pre_test_win(){
|
||||
bat '''
|
||||
hostname
|
||||
taskkill /f /t /im python.exe
|
||||
taskkill /f /t /im bash.exe
|
||||
taskkill /f /t /im taosd.exe
|
||||
ipconfig
|
||||
set
|
||||
date /t
|
||||
|
|
|
@ -98,12 +98,12 @@ ELSE ()
|
|||
ENDIF ()
|
||||
|
||||
IF (${SANITIZER} MATCHES "true")
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3")
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -fsanitize=address -fsanitize=undefined -fsanitize-recover=all -fsanitize=float-divide-by-zero -fsanitize=float-cast-overflow -fno-sanitize=shift-base -fno-sanitize=alignment -g3 -Wformat=0")
|
||||
MESSAGE(STATUS "Will compile with Address Sanitizer!")
|
||||
ELSE ()
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3")
|
||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wno-literal-suffix -Werror=return-type -fPIC -gdwarf-2 -g3 -Wformat=0")
|
||||
ENDIF ()
|
||||
|
||||
MESSAGE("System processor ID: ${CMAKE_SYSTEM_PROCESSOR}")
|
||||
|
|
|
@ -56,7 +56,8 @@ typedef enum EStreamType {
|
|||
STREAM_CLEAR,
|
||||
STREAM_INVALID,
|
||||
STREAM_GET_ALL,
|
||||
STREAM_DELETE,
|
||||
STREAM_DELETE_RESULT,
|
||||
STREAM_DELETE_DATA,
|
||||
STREAM_RETRIEVE,
|
||||
STREAM_PULL_DATA,
|
||||
STREAM_PULL_OVER,
|
||||
|
|
|
@ -222,6 +222,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock);
|
|||
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize);
|
||||
|
||||
int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n);
|
||||
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
|
||||
|
||||
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
|
||||
int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src);
|
||||
|
@ -235,6 +236,7 @@ SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index);
|
|||
void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress);
|
||||
const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData);
|
||||
|
||||
void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag);
|
||||
void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag);
|
||||
// for debug
|
||||
char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf);
|
||||
|
|
|
@ -78,6 +78,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag);
|
|||
int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag);
|
||||
int32_t tTagToValArray(const STag *pTag, SArray **ppArray);
|
||||
void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove
|
||||
int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag** ppTag, void* pMsgBuf);
|
||||
|
||||
// STRUCT =================
|
||||
struct STColumn {
|
||||
|
|
|
@ -436,15 +436,16 @@ typedef struct {
|
|||
int32_t ttl;
|
||||
int32_t numOfColumns;
|
||||
int32_t numOfTags;
|
||||
int32_t numOfFuncs;
|
||||
int32_t commentLen;
|
||||
int32_t ast1Len;
|
||||
int32_t ast2Len;
|
||||
SArray* pColumns; // array of SField
|
||||
SArray* pTags; // array of SField
|
||||
char* comment;
|
||||
SArray* pFuncs;
|
||||
char* pComment;
|
||||
char* pAst1;
|
||||
char* pAst2;
|
||||
SArray* pFuncs;
|
||||
} SMCreateStbReq;
|
||||
|
||||
int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq);
|
||||
|
|
|
@ -34,6 +34,7 @@ typedef enum EFunctionType {
|
|||
FUNCTION_TYPE_ELAPSED,
|
||||
FUNCTION_TYPE_IRATE,
|
||||
FUNCTION_TYPE_LAST_ROW,
|
||||
FUNCTION_TYPE_LAST_ROWT, //TODO: removed
|
||||
FUNCTION_TYPE_MAX,
|
||||
FUNCTION_TYPE_MIN,
|
||||
FUNCTION_TYPE_MODE,
|
||||
|
@ -195,6 +196,7 @@ bool fmIsIntervalInterpoFunc(int32_t funcId);
|
|||
bool fmIsInterpFunc(int32_t funcId);
|
||||
bool fmIsLastRowFunc(int32_t funcId);
|
||||
bool fmIsSystemInfoFunc(int32_t funcId);
|
||||
bool fmIsImplicitTsFunc(int32_t funcId);
|
||||
|
||||
int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc);
|
||||
|
||||
|
|
|
@ -104,6 +104,7 @@ typedef struct SIndefRowsFuncLogicNode {
|
|||
SNodeList* pFuncs;
|
||||
bool isTailFunc;
|
||||
bool isUniqueFunc;
|
||||
bool isTimeLineFunc;
|
||||
} SIndefRowsFuncLogicNode;
|
||||
|
||||
typedef struct SInterpFuncLogicNode {
|
||||
|
|
|
@ -251,7 +251,7 @@ typedef struct SSelectStmt {
|
|||
char stmtName[TSDB_TABLE_NAME_LEN];
|
||||
uint8_t precision;
|
||||
bool isEmptyResult;
|
||||
bool isTimeOrderQuery;
|
||||
bool isTimeLineResult;
|
||||
bool hasAggFuncs;
|
||||
bool hasRepeatScanFuncs;
|
||||
bool hasIndefiniteRowsFunc;
|
||||
|
@ -261,6 +261,7 @@ typedef struct SSelectStmt {
|
|||
bool hasTailFunc;
|
||||
bool hasInterpFunc;
|
||||
bool hasLastRowFunc;
|
||||
bool hasTimeLineFunc;
|
||||
bool groupSort;
|
||||
} SSelectStmt;
|
||||
|
||||
|
|
|
@ -65,7 +65,7 @@ typedef struct SQueryExecRes {
|
|||
} SQueryExecRes;
|
||||
|
||||
typedef struct SIndexMeta {
|
||||
#ifdef WINDOWS
|
||||
#if defined(WINDOWS) || defined(_TD_DARWIN_64)
|
||||
size_t avoidCompilationErrors;
|
||||
#endif
|
||||
|
||||
|
@ -249,13 +249,6 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
|
|||
((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_NODE_NOT_DEPLOYED || \
|
||||
(_code) == TSDB_CODE_SYN_NOT_LEADER || (_code) == TSDB_CODE_APP_NOT_READY)
|
||||
|
||||
#define NEED_SCHEDULER_RETRY_ERROR(_code) \
|
||||
(NEED_SCHEDULER_REDIRECT_ERROR(_code) || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \
|
||||
(_code) == TSDB_CODE_SCH_TIMEOUT_ERROR || (_code) == TSDB_CODE_RPC_BROKEN_LINK)
|
||||
|
||||
|
||||
|
||||
|
||||
#define REQUEST_TOTAL_EXEC_TIMES 2
|
||||
|
||||
#define qFatal(...) \
|
||||
|
|
|
@ -61,45 +61,23 @@ extern "C" {
|
|||
} \
|
||||
}
|
||||
|
||||
#define WAL_HEAD_VER 0
|
||||
#define WAL_PROTO_VER 0
|
||||
#define WAL_NOSUFFIX_LEN 20
|
||||
#define WAL_SUFFIX_AT (WAL_NOSUFFIX_LEN + 1)
|
||||
#define WAL_LOG_SUFFIX "log"
|
||||
#define WAL_INDEX_SUFFIX "idx"
|
||||
#define WAL_REFRESH_MS 1000
|
||||
#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead))
|
||||
#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalCkHead))
|
||||
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
|
||||
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
|
||||
#define WAL_MAGIC 0xFAFBFCFDULL
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef enum {
|
||||
TAOS_WAL_NOLOG = 0,
|
||||
TAOS_WAL_WRITE = 1,
|
||||
TAOS_WAL_FSYNC = 2,
|
||||
} EWalType;
|
||||
|
||||
// used by sync module
|
||||
typedef struct {
|
||||
int8_t isWeek;
|
||||
uint64_t seqNum;
|
||||
uint64_t term;
|
||||
} SSyncLogMeta;
|
||||
|
||||
typedef struct SWalReadHead {
|
||||
int8_t headVer;
|
||||
int8_t reserved;
|
||||
int16_t msgType;
|
||||
int32_t bodyLen;
|
||||
int64_t ingestTs; // not implemented
|
||||
int64_t version;
|
||||
|
||||
// sync meta
|
||||
SSyncLogMeta syncMeta;
|
||||
|
||||
char body[];
|
||||
} SWalReadHead;
|
||||
|
||||
typedef struct {
|
||||
int32_t vgId;
|
||||
int32_t fsyncPeriod; // millisecond
|
||||
|
@ -110,13 +88,6 @@ typedef struct {
|
|||
EWalType level; // wal level
|
||||
} SWalCfg;
|
||||
|
||||
typedef struct {
|
||||
uint64_t magic;
|
||||
uint32_t cksumHead;
|
||||
uint32_t cksumBody;
|
||||
SWalReadHead head;
|
||||
} SWalHead;
|
||||
|
||||
typedef struct SWalVer {
|
||||
int64_t firstVer;
|
||||
int64_t verInSnapshotting;
|
||||
|
@ -125,6 +96,35 @@ typedef struct SWalVer {
|
|||
int64_t lastVer;
|
||||
} SWalVer;
|
||||
|
||||
#pragma pack(push, 1)
|
||||
// used by sync module
|
||||
typedef struct {
|
||||
int8_t isWeek;
|
||||
uint64_t seqNum;
|
||||
uint64_t term;
|
||||
} SSyncLogMeta;
|
||||
|
||||
typedef struct {
|
||||
int8_t protoVer;
|
||||
int64_t version;
|
||||
int16_t msgType;
|
||||
int32_t bodyLen;
|
||||
int64_t ingestTs; // not implemented
|
||||
|
||||
// sync meta
|
||||
SSyncLogMeta syncMeta;
|
||||
|
||||
char body[];
|
||||
} SWalCont;
|
||||
|
||||
typedef struct {
|
||||
uint64_t magic;
|
||||
uint32_t cksumHead;
|
||||
uint32_t cksumBody;
|
||||
SWalCont head;
|
||||
} SWalCkHead;
|
||||
#pragma pack(pop)
|
||||
|
||||
typedef struct SWal {
|
||||
// cfg
|
||||
SWalCfg cfg;
|
||||
|
@ -134,7 +134,7 @@ typedef struct SWal {
|
|||
TdFilePtr pWriteLogTFile;
|
||||
TdFilePtr pWriteIdxTFile;
|
||||
int32_t writeCur;
|
||||
SArray *fileInfoSet;
|
||||
SArray *fileInfoSet; // SArray<SWalFileInfo>
|
||||
// status
|
||||
int64_t totSize;
|
||||
int64_t lastRollSeq;
|
||||
|
@ -146,7 +146,7 @@ typedef struct SWal {
|
|||
// path
|
||||
char path[WAL_PATH_LEN];
|
||||
// reusable write head
|
||||
SWalHead writeHead;
|
||||
SWalCkHead writeHead;
|
||||
} SWal; // WAL HANDLE
|
||||
|
||||
typedef struct SWalReadHandle {
|
||||
|
@ -158,11 +158,8 @@ typedef struct SWalReadHandle {
|
|||
int64_t capacity;
|
||||
int64_t status; // if cursor valid
|
||||
TdThreadMutex mutex;
|
||||
SWalHead *pHead;
|
||||
SWalCkHead *pHead;
|
||||
} SWalReadHandle;
|
||||
#pragma pack(pop)
|
||||
|
||||
// typedef int32_t (*FWalWrite)(void *ahandle, void *pHead);
|
||||
|
||||
// module initialization
|
||||
int32_t walInit();
|
||||
|
@ -174,9 +171,9 @@ int32_t walAlter(SWal *, SWalCfg *pCfg);
|
|||
void walClose(SWal *);
|
||||
|
||||
// write
|
||||
int64_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body,
|
||||
int32_t walWriteWithSyncInfo(SWal *, int64_t index, tmsg_t msgType, SSyncLogMeta syncMeta, const void *body,
|
||||
int32_t bodyLen);
|
||||
int64_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
|
||||
int32_t walWrite(SWal *, int64_t index, tmsg_t msgType, const void *body, int32_t bodyLen);
|
||||
void walFsync(SWal *, bool force);
|
||||
|
||||
// apis for lifecycle management
|
||||
|
@ -196,9 +193,9 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
|
|||
|
||||
// only for tq usage
|
||||
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
|
||||
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
|
||||
int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
|
||||
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead);
|
||||
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalCkHead *pHead);
|
||||
int32_t walFetchBody(SWalReadHandle *pRead, SWalCkHead **ppHead);
|
||||
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalCkHead *pHead);
|
||||
|
||||
typedef struct {
|
||||
int64_t refId;
|
||||
|
@ -210,6 +207,8 @@ void walCloseRef(SWalRef *);
|
|||
int32_t walRefVer(SWalRef *, int64_t ver);
|
||||
int32_t walUnrefVer(SWal *);
|
||||
|
||||
bool walLogExist(SWal *, int64_t ver);
|
||||
|
||||
// lifecycle check
|
||||
bool walIsEmpty(SWal *);
|
||||
int64_t walGetFirstVer(SWal *);
|
||||
|
|
|
@ -41,7 +41,6 @@ extern "C" {
|
|||
#include <sys/types.h>
|
||||
#include <termios.h>
|
||||
#include <sys/statvfs.h>
|
||||
#include <sys/prctl.h>
|
||||
#include <sys/shm.h>
|
||||
#include <sys/wait.h>
|
||||
|
||||
|
|
|
@ -63,7 +63,7 @@ int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val);
|
|||
int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val);
|
||||
int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val);
|
||||
int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val);
|
||||
void *atomic_add_fetch_ptr(void *ptr, void *val);
|
||||
void *atomic_add_fetch_ptr(void *ptr, int64_t val);
|
||||
int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val);
|
||||
int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val);
|
||||
int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val);
|
||||
|
@ -73,7 +73,7 @@ int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val);
|
|||
int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val);
|
||||
int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val);
|
||||
int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val);
|
||||
void *atomic_sub_fetch_ptr(void *ptr, void *val);
|
||||
void *atomic_sub_fetch_ptr(void *ptr, int64_t val);
|
||||
int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val);
|
||||
int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val);
|
||||
int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val);
|
||||
|
|
|
@ -25,17 +25,17 @@ extern "C" {
|
|||
// If the error is in a third-party library, place this header file under the third-party library header file.
|
||||
// When you want to use this feature, you should find or add the same function in the following sectio
|
||||
#ifndef ALLOW_FORBID_FUNC
|
||||
#define open OPEN_FUNC_TAOS_FORBID
|
||||
#define fopen FOPEN_FUNC_TAOS_FORBID
|
||||
#define access ACCESS_FUNC_TAOS_FORBID
|
||||
#define stat STAT_FUNC_TAOS_FORBID
|
||||
#define lstat LSTAT_FUNC_TAOS_FORBID
|
||||
#define fstat FSTAT_FUNC_TAOS_FORBID
|
||||
#define close CLOSE_FUNC_TAOS_FORBID
|
||||
#define fclose FCLOSE_FUNC_TAOS_FORBID
|
||||
#define fsync FSYNC_FUNC_TAOS_FORBID
|
||||
#define getline GETLINE_FUNC_TAOS_FORBID
|
||||
// #define fflush FFLUSH_FUNC_TAOS_FORBID
|
||||
#define open OPEN_FUNC_TAOS_FORBID
|
||||
#define fopen FOPEN_FUNC_TAOS_FORBID
|
||||
#define access ACCESS_FUNC_TAOS_FORBID
|
||||
#define stat STAT_FUNC_TAOS_FORBID
|
||||
#define lstat LSTAT_FUNC_TAOS_FORBID
|
||||
#define fstat FSTAT_FUNC_TAOS_FORBID
|
||||
#define close CLOSE_FUNC_TAOS_FORBID
|
||||
#define fclose FCLOSE_FUNC_TAOS_FORBID
|
||||
#define fsync FSYNC_FUNC_TAOS_FORBID
|
||||
#define getline GETLINE_FUNC_TAOS_FORBID
|
||||
// #define fflush FFLUSH_FUNC_TAOS_FORBID
|
||||
#endif
|
||||
|
||||
#ifndef PATH_MAX
|
||||
|
@ -43,54 +43,54 @@ extern "C" {
|
|||
#endif
|
||||
|
||||
typedef struct TdFile *TdFilePtr;
|
||||
|
||||
#define TD_FILE_CREATE 0x0001
|
||||
#define TD_FILE_WRITE 0x0002
|
||||
#define TD_FILE_READ 0x0004
|
||||
#define TD_FILE_TRUNC 0x0008
|
||||
#define TD_FILE_APPEND 0x0010
|
||||
#define TD_FILE_TEXT 0x0020
|
||||
#define TD_FILE_AUTO_DEL 0x0040
|
||||
#define TD_FILE_EXCL 0x0080
|
||||
#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
|
||||
TdFilePtr taosOpenFile(const char *path,int32_t tdFileOptions);
|
||||
|
||||
#define TD_FILE_CREATE 0x0001
|
||||
#define TD_FILE_WRITE 0x0002
|
||||
#define TD_FILE_READ 0x0004
|
||||
#define TD_FILE_TRUNC 0x0008
|
||||
#define TD_FILE_APPEND 0x0010
|
||||
#define TD_FILE_TEXT 0x0020
|
||||
#define TD_FILE_AUTO_DEL 0x0040
|
||||
#define TD_FILE_EXCL 0x0080
|
||||
#define TD_FILE_STREAM 0x0100 // Only support taosFprintfFile, taosGetLineFile, taosEOFFile
|
||||
TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions);
|
||||
|
||||
#define TD_FILE_ACCESS_EXIST_OK 0x1
|
||||
#define TD_FILE_ACCESS_READ_OK 0x2
|
||||
#define TD_FILE_ACCESS_WRITE_OK 0x4
|
||||
bool taosCheckAccessFile(const char *pathname, int mode);
|
||||
|
||||
bool taosCheckAccessFile(const char *pathname, int mode);
|
||||
|
||||
int32_t taosLockFile(TdFilePtr pFile);
|
||||
int32_t taosUnLockFile(TdFilePtr pFile);
|
||||
|
||||
|
||||
int32_t taosUmaskFile(int32_t maskVal);
|
||||
|
||||
|
||||
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime);
|
||||
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
|
||||
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
|
||||
bool taosCheckExistFile(const char *pathname);
|
||||
|
||||
|
||||
int64_t taosLSeekFile(TdFilePtr pFile, int64_t offset, int32_t whence);
|
||||
int32_t taosFtruncateFile(TdFilePtr pFile, int64_t length);
|
||||
int32_t taosFsyncFile(TdFilePtr pFile);
|
||||
|
||||
|
||||
int64_t taosReadFile(TdFilePtr pFile, void *buf, int64_t count);
|
||||
int64_t taosPReadFile(TdFilePtr pFile, void *buf, int64_t count, int64_t offset);
|
||||
int64_t taosWriteFile(TdFilePtr pFile, const void *buf, int64_t count);
|
||||
void taosFprintfFile(TdFilePtr pFile, const char *format, ...);
|
||||
|
||||
int64_t taosGetLineFile(TdFilePtr pFile, char ** __restrict ptrBuf);
|
||||
int64_t taosGetLineFile(TdFilePtr pFile, char **__restrict ptrBuf);
|
||||
int64_t taosGetsFile(TdFilePtr pFile, int32_t maxSize, char *__restrict buf);
|
||||
|
||||
int32_t taosEOFFile(TdFilePtr pFile);
|
||||
|
||||
int64_t taosCloseFile(TdFilePtr *ppFile);
|
||||
|
||||
|
||||
int32_t taosCloseFile(TdFilePtr *ppFile);
|
||||
|
||||
int32_t taosRenameFile(const char *oldName, const char *newName);
|
||||
int64_t taosCopyFile(const char *from, const char *to);
|
||||
int32_t taosRemoveFile(const char *path);
|
||||
|
||||
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
|
||||
|
||||
void taosGetTmpfilePath(const char *inputTmpDir, const char *fileNamePrefix, char *dstPath);
|
||||
|
||||
int64_t taosFSendFile(TdFilePtr pFileOut, TdFilePtr pFileIn, int64_t *offset, int64_t size);
|
||||
|
||||
|
|
|
@ -20,6 +20,12 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
// If the error is in a third-party library, place this header file under the third-party library header file.
|
||||
// When you want to use this feature, you should find or add the same function in the following sectio
|
||||
#ifndef ALLOW_FORBID_FUNC
|
||||
#define qsort QSORT_FUNC_TAOS_FORBID
|
||||
#endif
|
||||
|
||||
#define TPOW2(x) ((x) * (x))
|
||||
#define TABS(x) ((x) > 0 ? (x) : -(x))
|
||||
|
||||
|
@ -60,6 +66,13 @@ extern "C" {
|
|||
})
|
||||
#endif
|
||||
|
||||
#ifndef __COMPAR_FN_T
|
||||
#define __COMPAR_FN_T
|
||||
typedef int32_t (*__compar_fn_t)(const void *, const void *);
|
||||
#endif
|
||||
|
||||
void taosSort(void* arr, int64_t sz, int64_t width, __compar_fn_t compar);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -24,7 +24,9 @@ extern "C" {
|
|||
|
||||
#if defined(_TD_DARWIN_64)
|
||||
|
||||
typedef struct tsem_s *tsem_t;
|
||||
// typedef struct tsem_s *tsem_t;
|
||||
typedef struct bosal_sem_t *tsem_t;
|
||||
|
||||
|
||||
int tsem_init(tsem_t *sem, int pshared, unsigned int value);
|
||||
int tsem_wait(tsem_t *sem);
|
||||
|
@ -51,11 +53,11 @@ int tsem_timewait(tsem_t *sim, int64_t nanosecs);
|
|||
// #define taosThreadRwlockRdlock(lock) taosThreadMutexLock(lock)
|
||||
// #define taosThreadRwlockUnlock(lock) taosThreadMutexUnlock(lock)
|
||||
|
||||
#define TdThreadSpinlock TdThreadMutex
|
||||
#define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
|
||||
#define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
|
||||
#define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
|
||||
#define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
|
||||
// #define TdThreadSpinlock TdThreadMutex
|
||||
// #define taosThreadSpinInit(lock, NULL) taosThreadMutexInit(lock, NULL)
|
||||
// #define taosThreadSpinDestroy(lock) taosThreadMutexDestroy(lock)
|
||||
// #define taosThreadSpinLock(lock) taosThreadMutexLock(lock)
|
||||
// #define taosThreadSpinUnlock(lock) taosThreadMutexUnlock(lock)
|
||||
#endif
|
||||
|
||||
bool taosCheckPthreadValid(TdThread thread);
|
||||
|
|
|
@ -64,7 +64,6 @@
|
|||
#include <osEok.h>
|
||||
#else
|
||||
#include <netinet/in.h>
|
||||
#include <sys/epoll.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
@ -77,15 +76,12 @@ typedef int socklen_t;
|
|||
#define TAOS_EPOLL_WAIT_TIME 100
|
||||
typedef SOCKET eventfd_t;
|
||||
#define eventfd(a, b) -1
|
||||
#define EpollClose(pollFd) epoll_close(pollFd)
|
||||
#ifndef EPOLLWAKEUP
|
||||
#define EPOLLWAKEUP (1u << 29)
|
||||
#endif
|
||||
#elif defined(_TD_DARWIN_64)
|
||||
#define TAOS_EPOLL_WAIT_TIME 500
|
||||
typedef int32_t SOCKET;
|
||||
typedef SOCKET EpollFd;
|
||||
#define EpollClose(pollFd) epoll_close(pollFd)
|
||||
#else
|
||||
#define TAOS_EPOLL_WAIT_TIME 500
|
||||
typedef int32_t SOCKET;
|
||||
|
@ -122,14 +118,6 @@ typedef SOCKET EpollFd;
|
|||
typedef int32_t SocketFd;
|
||||
typedef SocketFd EpollFd;
|
||||
|
||||
typedef struct TdSocket {
|
||||
#if SOCKET_WITH_LOCK
|
||||
TdThreadRwlock rwlock;
|
||||
#endif
|
||||
int refId;
|
||||
SocketFd fd;
|
||||
} * TdSocketPtr, TdSocket;
|
||||
|
||||
typedef struct TdSocketServer *TdSocketServerPtr;
|
||||
typedef struct TdSocket * TdSocketPtr;
|
||||
typedef struct TdEpoll * TdEpollPtr;
|
||||
|
@ -181,11 +169,6 @@ void taosSetMaskSIGPIPE();
|
|||
uint32_t taosInetAddr(const char *ipAddr);
|
||||
const char *taosInetNtoa(struct in_addr ipInt);
|
||||
|
||||
TdEpollPtr taosCreateEpoll(int32_t size);
|
||||
int32_t taosCtlEpoll(TdEpollPtr pEpoll, int32_t epollOperate, TdSocketPtr pSocket, struct epoll_event *event);
|
||||
int32_t taosWaitEpoll(TdEpollPtr pEpoll, struct epoll_event *event, int32_t maxEvents, int32_t timeout);
|
||||
int32_t taosCloseEpoll(TdEpollPtr *ppEpoll);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -67,7 +67,7 @@ bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs
|
|||
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
|
||||
TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);
|
||||
bool taosValidateEncodec(const char *encodec);
|
||||
int32_t taosHexEncode(const char *src, char *dst, int32_t len);
|
||||
int32_t taosHexEncode(const unsigned char *src, char *dst, int32_t len);
|
||||
int32_t taosHexDecode(const char *src, char *dst, int32_t len);
|
||||
|
||||
int32_t taosWcharWidth(TdWchar wchar);
|
||||
|
|
|
@ -188,27 +188,27 @@ int32_t taosThreadJoin(TdThread thread, void **valuePtr);
|
|||
int32_t taosThreadKeyCreate(TdThreadKey * key, void(*destructor)(void *));
|
||||
int32_t taosThreadKeyDelete(TdThreadKey key);
|
||||
int32_t taosThreadKill(TdThread thread, int32_t sig);
|
||||
int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
|
||||
// int32_t taosThreadMutexConsistent(TdThreadMutex* mutex);
|
||||
int32_t taosThreadMutexDestroy(TdThreadMutex * mutex);
|
||||
int32_t taosThreadMutexInit(TdThreadMutex * mutex, const TdThreadMutexAttr * attr);
|
||||
int32_t taosThreadMutexLock(TdThreadMutex * mutex);
|
||||
int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
|
||||
// int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime);
|
||||
int32_t taosThreadMutexTryLock(TdThreadMutex * mutex);
|
||||
int32_t taosThreadMutexUnlock(TdThreadMutex * mutex);
|
||||
int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr * attr);
|
||||
int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *pshared);
|
||||
int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
|
||||
// int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust);
|
||||
int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind);
|
||||
int32_t taosThreadMutexAttrInit(TdThreadMutexAttr * attr);
|
||||
int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared);
|
||||
int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
|
||||
// int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust);
|
||||
int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind);
|
||||
int32_t taosThreadOnce(TdThreadOnce * onceControl, void(*initRoutine)(void));
|
||||
int32_t taosThreadRwlockDestroy(TdThreadRwlock * rwlock);
|
||||
int32_t taosThreadRwlockInit(TdThreadRwlock * rwlock, const TdThreadRwlockAttr * attr);
|
||||
int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock);
|
||||
int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
|
||||
int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
|
||||
// int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
|
||||
// int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime);
|
||||
int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock);
|
||||
int32_t taosThreadRwlockTryWrlock(TdThreadRwlock * rwlock);
|
||||
int32_t taosThreadRwlockUnlock(TdThreadRwlock * rwlock);
|
||||
|
|
|
@ -388,6 +388,10 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0719)
|
||||
#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x071A)
|
||||
#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x071B)
|
||||
//json
|
||||
#define TSDB_CODE_QRY_JSON_IN_ERROR TAOS_DEF_ERROR_CODE(0, 0x071C)
|
||||
#define TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x071D)
|
||||
#define TSDB_CODE_QRY_JSON_IN_GROUP_ERROR TAOS_DEF_ERROR_CODE(0, 0x071E)
|
||||
|
||||
// grant
|
||||
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)
|
||||
|
@ -573,6 +577,7 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC TAOS_DEF_ERROR_CODE(0, 0x265B)
|
||||
#define TSDB_CODE_PAR_INVALID_TABLE_OPTION TAOS_DEF_ERROR_CODE(0, 0x265C)
|
||||
#define TSDB_CODE_PAR_INVALID_INTERP_CLAUSE TAOS_DEF_ERROR_CODE(0, 0x265D)
|
||||
#define TSDB_CODE_PAR_NO_VALID_FUNC_IN_WIN TAOS_DEF_ERROR_CODE(0, 0x265E)
|
||||
|
||||
//planner
|
||||
#define TSDB_CODE_PLAN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2700)
|
||||
|
@ -618,9 +623,12 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150)
|
||||
#define TSDB_CODE_RSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3151)
|
||||
#define TSDB_CODE_RSMA_QTASKINFO_CREATE TAOS_DEF_ERROR_CODE(0, 0x3152)
|
||||
#define TSDB_CODE_RSMA_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x3153)
|
||||
|
||||
//index
|
||||
#define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200)
|
||||
#define TSDB_CODE_INDEX_INVALID_FILE TAOS_DEF_ERROR_CODE(0, 0x3201)
|
||||
|
||||
|
||||
//tmq
|
||||
#define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000)
|
||||
|
|
|
@ -287,7 +287,7 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta
|
||||
|
||||
#define TSDB_MIN_VNODES_PER_DB 1
|
||||
#define TSDB_MAX_VNODES_PER_DB 4096
|
||||
#define TSDB_MAX_VNODES_PER_DB 1024
|
||||
#define TSDB_DEFAULT_VN_PER_DB 2
|
||||
#define TSDB_MIN_BUFFER_PER_VNODE 3 // unit MB
|
||||
#define TSDB_MAX_BUFFER_PER_VNODE 16384 // unit MB
|
||||
|
|
|
@ -44,6 +44,8 @@ uint32_t ip2uint(const char *const ip_addr);
|
|||
void taosIp2String(uint32_t ip, char *str);
|
||||
void taosIpPort2String(uint32_t ip, uint16_t port, char *str);
|
||||
|
||||
void *tmemmem(const char *haystack, int hlen, const char *needle, int nlen);
|
||||
|
||||
static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) {
|
||||
T_MD5_CTX context;
|
||||
tMD5Init(&context);
|
||||
|
@ -59,10 +61,10 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
|
|||
tMD5Final(&context);
|
||||
char buf[TSDB_PASSWORD_LEN + 1];
|
||||
|
||||
sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0],
|
||||
context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5],
|
||||
context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10],
|
||||
context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]);
|
||||
sprintf(buf, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1],
|
||||
context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6],
|
||||
context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11],
|
||||
context.digest[12], context.digest[13], context.digest[14], context.digest[15]);
|
||||
memcpy(target, buf, TSDB_PASSWORD_LEN);
|
||||
}
|
||||
|
||||
|
|
|
@ -83,6 +83,7 @@ typedef uint16_t VarDataLenT; // maxVarDataLen: 32767
|
|||
|
||||
#define varDataLen(v) ((VarDataLenT *)(v))[0]
|
||||
#define varDataVal(v) ((char *)(v) + VARSTR_HEADER_SIZE)
|
||||
#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
|
||||
|
||||
#define NCHAR_WIDTH_TO_BYTES(n) ((n) * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE)
|
||||
|
||||
|
|
|
@ -786,6 +786,7 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) {
|
|||
void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
|
||||
SRequestObj* pRequest = (SRequestObj*)param;
|
||||
pRequest->code = code;
|
||||
pRequest->body.resInfo.execRes = pResult->res;
|
||||
|
||||
if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type ||
|
||||
TDMT_VND_CREATE_TABLE == pRequest->type) {
|
||||
|
@ -797,6 +798,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) {
|
|||
}
|
||||
}
|
||||
|
||||
taosMemoryFree(pResult);
|
||||
|
||||
tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code,
|
||||
tstrerror(code), pRequest->requestId);
|
||||
|
||||
|
|
|
@ -1609,8 +1609,8 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) {
|
|||
int64_t transporterId = 0;
|
||||
/*printf("send poll\n");*/
|
||||
|
||||
char offsetFormatBuf[50];
|
||||
tFormatOffset(offsetFormatBuf, 50, &pVg->currentOffsetNew);
|
||||
char offsetFormatBuf[80];
|
||||
tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffsetNew);
|
||||
tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId,
|
||||
pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId);
|
||||
/*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/
|
||||
|
|
|
@ -108,13 +108,11 @@ static const SSysDbTableSchema userFuncSchema[] = {
|
|||
};
|
||||
|
||||
static const SSysDbTableSchema userIdxSchema[] = {
|
||||
{.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "index_database", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "index_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "column_name", .bytes = SYSTABLE_SCH_COL_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "index_type", .bytes = 10, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "index_extensions", .bytes = 256, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
|
||||
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
|
||||
};
|
||||
|
||||
static const SSysDbTableSchema userStbsSchema[] = {
|
||||
|
@ -313,7 +311,7 @@ static const SSysDbTableSchema querySchema[] = {
|
|||
{.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT},
|
||||
{.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
|
||||
{.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "end_point", .bytes = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
|
@ -329,17 +327,17 @@ static const SSysDbTableSchema appSchema[] = {
|
|||
{.name = "app_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "ip", .bytes = TSDB_IPv4ADDR_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT},
|
||||
{.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "start_time", .bytes = 8 , .type = TSDB_DATA_TYPE_TIMESTAMP},
|
||||
{.name = "insert_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "insert_row", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "insert_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "insert_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "fetch_bytes", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "query_time", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "show_query", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "total_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "current_req", .bytes = 8 , .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "name", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR},
|
||||
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
|
||||
{.name = "insert_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "insert_row", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "insert_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "insert_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "fetch_bytes", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "query_time", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "show_query", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "total_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "current_req", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT},
|
||||
{.name = "last_access", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP},
|
||||
};
|
||||
|
||||
|
@ -353,8 +351,7 @@ static const SSysTableMeta perfsMeta[] = {
|
|||
{TSDB_PERFS_TABLE_TRANS, transSchema, tListLen(transSchema)},
|
||||
{TSDB_PERFS_TABLE_SMAS, smaSchema, tListLen(smaSchema)},
|
||||
{TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)},
|
||||
{TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}
|
||||
};
|
||||
{TSDB_PERFS_TABLE_APPS, appSchema, tListLen(appSchema)}};
|
||||
|
||||
void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size) {
|
||||
*pInfosTableMeta = infosMeta;
|
||||
|
|
|
@ -716,7 +716,12 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
|
|||
|
||||
void* left1 = colDataGetData(pColInfoData, left);
|
||||
void* right1 = colDataGetData(pColInfoData, right);
|
||||
|
||||
if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) {
|
||||
if (tTagIsJson(left1) || tTagIsJson(right1)) {
|
||||
terrno = TSDB_CODE_QRY_JSON_NOT_SUPPORT_ERROR;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
__compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
|
||||
|
||||
int ret = fn(left1, right1);
|
||||
|
@ -890,9 +895,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
|||
SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0);
|
||||
|
||||
int64_t p0 = taosGetTimestampUs();
|
||||
|
||||
|
||||
__compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
|
||||
qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
|
||||
taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn);
|
||||
|
||||
int64_t p1 = taosGetTimestampUs();
|
||||
uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows);
|
||||
|
@ -919,6 +924,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
|||
}
|
||||
|
||||
taosqsort(index, rows, sizeof(int32_t), &helper, dataBlockCompar);
|
||||
if(terrno) return terrno;
|
||||
|
||||
int64_t p1 = taosGetTimestampUs();
|
||||
|
||||
|
@ -1218,7 +1224,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
|
|||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
|
||||
SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
|
||||
if (pSrc->pData == NULL) {
|
||||
if (pSrc->pData == NULL && (!IS_VAR_DATA_TYPE(pSrc->info.type))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -1253,6 +1259,7 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
|
|||
|
||||
dst->info.rows = src->info.rows;
|
||||
dst->info.window = src->info.window;
|
||||
dst->info.type = src->info.type;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -1431,9 +1438,39 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
|
|||
}
|
||||
}
|
||||
|
||||
static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, size_t end){
|
||||
int32_t dataOffset = -1;
|
||||
int32_t dataLen = 0;
|
||||
int32_t beigin = start;
|
||||
while(beigin < end){
|
||||
int32_t offset = pColInfoData->varmeta.offset[beigin];
|
||||
if(offset == -1) {
|
||||
beigin++;
|
||||
continue;
|
||||
}
|
||||
if(start != 0) {
|
||||
pColInfoData->varmeta.offset[beigin] = dataLen;
|
||||
}
|
||||
char *data = pColInfoData->pData + offset;
|
||||
if(dataOffset == -1) dataOffset = offset; // mark the begin of data
|
||||
int32_t type = pColInfoData->info.type;
|
||||
if (type == TSDB_DATA_TYPE_JSON) {
|
||||
dataLen += getJsonValueLen(data);
|
||||
} else {
|
||||
dataLen += varDataTLen(data);
|
||||
}
|
||||
beigin++;
|
||||
}
|
||||
if(dataOffset > 0){
|
||||
memmove(pColInfoData->pData, pColInfoData->pData + dataOffset, dataLen);
|
||||
memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[start], (end - start) * sizeof(int32_t));
|
||||
}
|
||||
return dataLen;
|
||||
}
|
||||
|
||||
static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
|
||||
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
|
||||
memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n) * sizeof(int32_t));
|
||||
pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, n, total);
|
||||
memset(&pColInfoData->varmeta.offset[total - n], 0, n);
|
||||
} else {
|
||||
int32_t bytes = pColInfoData->info.bytes;
|
||||
|
@ -1461,6 +1498,33 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void colDataKeepFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) {
|
||||
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
|
||||
pColInfoData->varmeta.length = colDataMoveVarData(pColInfoData, 0, n);
|
||||
memset(&pColInfoData->varmeta.offset[n], 0, total - n);
|
||||
}
|
||||
}
|
||||
|
||||
int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
|
||||
if (n == 0) {
|
||||
blockDataCleanup(pBlock);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
if (pBlock->info.rows <= n) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
|
||||
colDataKeepFirstNRows(pColInfoData, n, pBlock->info.rows);
|
||||
}
|
||||
|
||||
pBlock->info.rows = n;
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
|
||||
int64_t tbUid = pBlock->info.uid;
|
||||
int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
|
@ -1605,6 +1669,13 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) {
|
|||
return buf;
|
||||
}
|
||||
|
||||
void blockDebugShowDataBlock(SSDataBlock* pBlock, const char* flag) {
|
||||
SArray* dataBlocks = taosArrayInit(1, sizeof(SSDataBlock));
|
||||
taosArrayPush(dataBlocks, pBlock);
|
||||
blockDebugShowDataBlocks(dataBlocks, flag);
|
||||
taosArrayDestroy(dataBlocks);
|
||||
}
|
||||
|
||||
void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
|
||||
char pBuf[128] = {0};
|
||||
int32_t sz = taosArrayGetSize(dataBlocks);
|
||||
|
|
|
@ -931,9 +931,9 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) {
|
|||
|
||||
// sort
|
||||
if (isJson) {
|
||||
qsort(pArray->pData, nTag, sizeof(STagVal), tTagValJsonCmprFn);
|
||||
taosSort(pArray->pData, nTag, sizeof(STagVal), tTagValJsonCmprFn);
|
||||
} else {
|
||||
qsort(pArray->pData, nTag, sizeof(STagVal), tTagValCmprFn);
|
||||
taosSort(pArray->pData, nTag, sizeof(STagVal), tTagValCmprFn);
|
||||
}
|
||||
|
||||
// get size
|
||||
|
|
|
@ -503,6 +503,7 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
|
|||
if (tEncodeI32(&encoder, pReq->ttl) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->numOfFuncs) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->commentLen) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->ast1Len) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->ast2Len) < 0) return -1;
|
||||
|
@ -510,21 +511,26 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
|
|||
for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
|
||||
SField *pField = taosArrayGet(pReq->pColumns, i);
|
||||
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pReq->numOfTags; ++i) {
|
||||
SField *pField = taosArrayGet(pReq->pTags, i);
|
||||
if (tEncodeI8(&encoder, pField->type) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pField->bytes) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
|
||||
if (tEncodeI8(&encoder, pField->flags) < 0) return -1;
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
|
||||
const char *pFunc = taosArrayGet(pReq->pFuncs, i);
|
||||
if (tEncodeCStr(&encoder, pFunc) < 0) return -1;
|
||||
}
|
||||
|
||||
if (pReq->commentLen > 0) {
|
||||
if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1;
|
||||
}
|
||||
if (pReq->ast1Len > 0) {
|
||||
if (tEncodeBinary(&encoder, pReq->pAst1, pReq->ast1Len) < 0) return -1;
|
||||
|
@ -533,13 +539,6 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq
|
|||
if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1;
|
||||
}
|
||||
|
||||
int32_t numOfFuncs = taosArrayGetSize(pReq->pFuncs);
|
||||
if (tEncodeI32(&encoder, numOfFuncs) < 0) return -1;
|
||||
for (int32_t i = 0; i < numOfFuncs; ++i) {
|
||||
const char *pFunc = taosArrayGet(pReq->pFuncs, i);
|
||||
if (tEncodeCStr(&encoder, pFunc) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
@ -561,13 +560,15 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
|
|||
if (tDecodeI32(&decoder, &pReq->ttl) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->numOfFuncs) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->commentLen) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->ast1Len) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->ast2Len) < 0) return -1;
|
||||
|
||||
pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField));
|
||||
pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField));
|
||||
if (pReq->pColumns == NULL || pReq->pTags == NULL) {
|
||||
pReq->pFuncs = taosArrayInit(pReq->numOfFuncs, TSDB_FUNC_NAME_LEN);
|
||||
if (pReq->pColumns == NULL || pReq->pTags == NULL || pReq->pFuncs == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
@ -575,9 +576,9 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
|
|||
for (int32_t i = 0; i < pReq->numOfColumns; ++i) {
|
||||
SField field = {0};
|
||||
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
|
||||
if (taosArrayPush(pReq->pColumns, &field) == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
|
@ -587,19 +588,28 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
|
|||
for (int32_t i = 0; i < pReq->numOfTags; ++i) {
|
||||
SField field = {0};
|
||||
if (tDecodeI8(&decoder, &field.type) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &field.bytes) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, field.name) < 0) return -1;
|
||||
if (tDecodeI8(&decoder, &field.flags) < 0) return -1;
|
||||
if (taosArrayPush(pReq->pTags, &field) == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pReq->numOfFuncs; ++i) {
|
||||
char pFunc[TSDB_FUNC_NAME_LEN] = {0};
|
||||
if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1;
|
||||
if (taosArrayPush(pReq->pFuncs, pFunc) == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
if (pReq->commentLen > 0) {
|
||||
pReq->comment = taosMemoryMalloc(pReq->commentLen + 1);
|
||||
if (pReq->comment == NULL) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1;
|
||||
pReq->pComment = taosMemoryMalloc(pReq->commentLen + 1);
|
||||
if (pReq->pComment == NULL) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1;
|
||||
}
|
||||
|
||||
if (pReq->ast1Len > 0) {
|
||||
|
@ -614,23 +624,7 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
|
|||
if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1;
|
||||
}
|
||||
|
||||
int32_t numOfFuncs = 0;
|
||||
if (tDecodeI32(&decoder, &numOfFuncs) < 0) return -1;
|
||||
if (numOfFuncs > 0) {
|
||||
pReq->pFuncs = taosArrayInit(numOfFuncs, TSDB_FUNC_NAME_LEN);
|
||||
if (NULL == pReq->pFuncs) return -1;
|
||||
}
|
||||
for (int32_t i = 0; i < numOfFuncs; ++i) {
|
||||
char pFunc[TSDB_FUNC_NAME_LEN] = {0};
|
||||
if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1;
|
||||
if (taosArrayPush(pReq->pFuncs, pFunc) == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
@ -638,10 +632,10 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR
|
|||
void tFreeSMCreateStbReq(SMCreateStbReq *pReq) {
|
||||
taosArrayDestroy(pReq->pColumns);
|
||||
taosArrayDestroy(pReq->pTags);
|
||||
taosMemoryFreeClear(pReq->comment);
|
||||
taosArrayDestroy(pReq->pFuncs);
|
||||
taosMemoryFreeClear(pReq->pComment);
|
||||
taosMemoryFreeClear(pReq->pAst1);
|
||||
taosMemoryFreeClear(pReq->pAst2);
|
||||
taosArrayDestroy(pReq->pFuncs);
|
||||
}
|
||||
|
||||
int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) {
|
||||
|
|
|
@ -589,7 +589,7 @@ int32_t convertStringToTimestamp(int16_t type, char *inputData, int64_t timePrec
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
newColData[len] = 0;
|
||||
int32_t ret = taosParseTime(newColData, timeVal, len + 1, (int32_t)timePrec, tsDaylight);
|
||||
int32_t ret = taosParseTime(newColData, timeVal, len, (int32_t)timePrec, tsDaylight);
|
||||
if (ret != TSDB_CODE_SUCCESS) {
|
||||
taosMemoryFree(newColData);
|
||||
return ret;
|
||||
|
|
|
@ -845,11 +845,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) {
|
|||
|
||||
int64_t offset = getDataStartOffset();
|
||||
int32_t size = (int32_t)pSrcBuf->fileSize - (int32_t)offset;
|
||||
#if defined(_TD_DARWIN_64)
|
||||
int64_t written = taosFSendFile(pDestBuf->pFile->fp, pSrcBuf->pFile->fp, &offset, size);
|
||||
#else
|
||||
int64_t written = taosFSendFile(pDestBuf->pFile, pSrcBuf->pFile, &offset, size);
|
||||
#endif
|
||||
|
||||
if (written == -1 || written != size) {
|
||||
return -1;
|
||||
|
|
|
@ -155,7 +155,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin
|
|||
void taosVariantDestroy(SVariant *pVar) {
|
||||
if (pVar == NULL) return;
|
||||
|
||||
if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) {
|
||||
if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR
|
||||
|| pVar->nType == TSDB_DATA_TYPE_JSON) {
|
||||
taosMemoryFreeClear(pVar->pz);
|
||||
pVar->nLen = 0;
|
||||
}
|
||||
|
@ -184,7 +185,8 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) {
|
|||
if (pSrc == NULL || pDst == NULL) return;
|
||||
|
||||
pDst->nType = pSrc->nType;
|
||||
if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) {
|
||||
if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR
|
||||
|| pSrc->nType == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE;
|
||||
char *p = taosMemoryRealloc(pDst->pz, len);
|
||||
assert(p);
|
||||
|
@ -976,6 +978,7 @@ char *taosVariantGet(SVariant *pVar, int32_t type) {
|
|||
case TSDB_DATA_TYPE_FLOAT:
|
||||
return (char *)&pVar->d;
|
||||
case TSDB_DATA_TYPE_BINARY:
|
||||
case TSDB_DATA_TYPE_JSON:
|
||||
return (char *)pVar->pz;
|
||||
case TSDB_DATA_TYPE_NCHAR:
|
||||
return (char *)pVar->ucs4;
|
||||
|
|
|
@ -21,7 +21,7 @@ SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) {
|
|||
|
||||
taosThreadRwlockRdlock(&pMgmt->lock);
|
||||
taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
|
||||
if (pVnode == NULL) {
|
||||
if (pVnode == NULL || pVnode->dropped) {
|
||||
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
|
||||
} else {
|
||||
int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
|
||||
|
@ -81,16 +81,18 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) {
|
|||
|
||||
vmReleaseVnode(pMgmt, pVnode);
|
||||
while (pVnode->refCount > 0) taosMsleep(10);
|
||||
dTrace("vgId:%d, wait for vnode queue is empty", pVnode->vgId);
|
||||
|
||||
while (!taosQueueEmpty(pVnode->pWriteQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pSyncQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pApplyQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pQueryQ)) taosMsleep(10);
|
||||
while (!taosQueueEmpty(pVnode->pFetchQ)) taosMsleep(10);
|
||||
dTrace("vgId:%d, vnode-fetch queue is empty", pVnode->vgId);
|
||||
|
||||
vmFreeQueue(pMgmt, pVnode);
|
||||
vnodeClose(pVnode->pImpl);
|
||||
pVnode->pImpl = NULL;
|
||||
|
||||
dDebug("vgId:%d, vnode is closed", pVnode->vgId);
|
||||
|
||||
if (pVnode->dropped) {
|
||||
|
|
|
@ -107,7 +107,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf
|
|||
const STraceId *trace = &pMsg->info.traceId;
|
||||
dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg);
|
||||
|
||||
int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here
|
||||
int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here
|
||||
dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code);
|
||||
rpcFreeCont(pMsg->pCont);
|
||||
taosFreeQitem(pMsg);
|
||||
|
@ -146,8 +146,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp
|
|||
|
||||
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId);
|
||||
if (pVnode == NULL) {
|
||||
dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(),
|
||||
TMSG_INFO(pMsg->msgType));
|
||||
dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, msgtype:%s qtype:%d", pHead->vgId, pMsg,
|
||||
terrstr(), TMSG_INFO(pMsg->msgType), qtype);
|
||||
return terrno != 0 ? terrno : -1;
|
||||
}
|
||||
|
||||
|
|
|
@ -341,11 +341,12 @@ typedef struct {
|
|||
int32_t colVer;
|
||||
int32_t smaVer;
|
||||
int32_t nextColId;
|
||||
int64_t watermark[2];
|
||||
int64_t maxdelay[2];
|
||||
int64_t watermark[2];
|
||||
int32_t ttl;
|
||||
int32_t numOfColumns;
|
||||
int32_t numOfTags;
|
||||
int32_t numOfFuncs;
|
||||
int32_t commentLen;
|
||||
int32_t ast1Len;
|
||||
int32_t ast2Len;
|
||||
|
@ -555,7 +556,7 @@ typedef struct {
|
|||
int64_t uid;
|
||||
int8_t status;
|
||||
// config
|
||||
int8_t dropPolicy;
|
||||
int8_t igExpired;
|
||||
int8_t trigger;
|
||||
int64_t triggerParam;
|
||||
int64_t watermark;
|
||||
|
|
|
@ -87,7 +87,7 @@ typedef struct {
|
|||
} STelemMgmt;
|
||||
|
||||
typedef struct {
|
||||
sem_t syncSem;
|
||||
tsem_t syncSem;
|
||||
int64_t sync;
|
||||
bool standby;
|
||||
SReplica replica;
|
||||
|
|
|
@ -38,6 +38,8 @@ int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
|
|||
int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
|
||||
int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
|
||||
|
||||
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -47,6 +47,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, S
|
|||
void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *cntlen, bool standby);
|
||||
void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
|
||||
void *mndBuildAlterVnodeReq(SMnode *, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen);
|
||||
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -21,6 +21,7 @@
|
|||
#include "mndShow.h"
|
||||
#include "mndSma.h"
|
||||
#include "mndStb.h"
|
||||
#include "mndStream.h"
|
||||
#include "mndSubscribe.h"
|
||||
#include "mndTopic.h"
|
||||
#include "mndTrans.h"
|
||||
|
@ -548,18 +549,33 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
|
|||
terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED;
|
||||
|
||||
if (pAlter->buffer > 0 && pAlter->buffer != pDb->cfg.buffer) {
|
||||
#if 1
|
||||
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
return terrno;
|
||||
#else
|
||||
pDb->cfg.buffer = pAlter->buffer;
|
||||
terrno = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (pAlter->pages > 0 && pAlter->pages != pDb->cfg.pages) {
|
||||
#if 1
|
||||
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
return terrno;
|
||||
#else
|
||||
pDb->cfg.pages = pAlter->pages;
|
||||
terrno = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (pAlter->pageSize > 0 && pAlter->pageSize != pDb->cfg.pageSize) {
|
||||
#if 1
|
||||
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
return terrno;
|
||||
#else
|
||||
pDb->cfg.pageSize = pAlter->pageSize;
|
||||
terrno = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (pAlter->daysPerFile > 0 && pAlter->daysPerFile != pDb->cfg.daysPerFile) {
|
||||
|
@ -593,8 +609,12 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
|
|||
}
|
||||
|
||||
if (pAlter->strict >= 0 && pAlter->strict != pDb->cfg.strict) {
|
||||
#if 1
|
||||
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
#else
|
||||
pDb->cfg.strict = pAlter->strict;
|
||||
terrno = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) {
|
||||
|
@ -603,9 +623,13 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
|
|||
}
|
||||
|
||||
if (pAlter->replications > 0 && pAlter->replications != pDb->cfg.replications) {
|
||||
#if 1
|
||||
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
#else
|
||||
pDb->cfg.replications = pAlter->replications;
|
||||
pDb->vgVersion++;
|
||||
terrno = 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
return terrno;
|
||||
|
@ -645,7 +669,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
|
|||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
|
||||
if (pVgroup->dbUid == pNew->uid) {
|
||||
if (mndVgroupInDb(pVgroup, pNew->uid)) {
|
||||
if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup, pArray) != 0) {
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
|
@ -927,6 +951,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
|
|||
if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
|
||||
if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
|
||||
if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;
|
||||
if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
|
||||
if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
|
||||
if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER;
|
||||
|
||||
|
@ -947,7 +972,6 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
|
|||
mndTransSetRpcRsp(pTrans, pRsp, rspLen);
|
||||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
@ -1006,7 +1030,7 @@ static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) {
|
|||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
|
||||
if (pVgroup->dbUid == pDb->uid) {
|
||||
if (mndVgroupInDb(pVgroup, pDb->uid)) {
|
||||
numOfTables += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT;
|
||||
vindex++;
|
||||
}
|
||||
|
|
|
@ -28,7 +28,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
|
|||
if (tEncodeI64(pEncoder, pObj->uid) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pObj->status) < 0) return -1;
|
||||
|
||||
if (tEncodeI8(pEncoder, pObj->dropPolicy) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pObj->igExpired) < 0) return -1;
|
||||
if (tEncodeI8(pEncoder, pObj->trigger) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pObj->triggerParam) < 0) return -1;
|
||||
if (tEncodeI64(pEncoder, pObj->watermark) < 0) return -1;
|
||||
|
@ -73,7 +73,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj) {
|
|||
if (tDecodeI64(pDecoder, &pObj->uid) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pObj->status) < 0) return -1;
|
||||
|
||||
if (tDecodeI8(pDecoder, &pObj->dropPolicy) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pObj->igExpired) < 0) return -1;
|
||||
if (tDecodeI8(pDecoder, &pObj->trigger) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pObj->triggerParam) < 0) return -1;
|
||||
if (tDecodeI64(pDecoder, &pObj->watermark) < 0) return -1;
|
||||
|
|
|
@ -758,6 +758,11 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) {
|
|||
}
|
||||
}
|
||||
|
||||
if (numOfVnodes > 0) {
|
||||
terrno = TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes);
|
||||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
|
|
|
@ -225,10 +225,11 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
|
|||
SVgObj* pVgroup;
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (strcmp(pVgroup->dbName, pStream->targetDb) != 0) {
|
||||
if (!mndVgroupInDb(pVgroup, pStream->targetDbUid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
||||
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
|
||||
if (pTask == NULL) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
|
@ -420,10 +421,11 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
|
|||
SVgObj* pVgroup;
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pStream->sourceDbUid) {
|
||||
if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
||||
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
|
||||
if (pInnerTask == NULL) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
@ -483,10 +485,11 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
|
|||
SVgObj* pVgroup;
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pStream->sourceDbUid) {
|
||||
if (!mndVgroupInDb(pVgroup, pStream->sourceDbUid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
||||
SStreamTask* pTask = tNewSStreamTask(pStream->uid);
|
||||
if (pTask == NULL) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
|
@ -559,7 +562,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
|
|||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void**)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pTopic->dbUid) {
|
||||
if (!mndVgroupInDb(pVgroup, pTopic->dbUid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
|
|
@ -71,7 +71,7 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
|
|||
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_FUNCTIONS, len) == 0) {
|
||||
type = TSDB_MGMT_TABLE_FUNC;
|
||||
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_INDEXES, len) == 0) {
|
||||
// type = TSDB_MGMT_TABLE_INDEX;
|
||||
type = TSDB_MGMT_TABLE_INDEX;
|
||||
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_STABLES, len) == 0) {
|
||||
type = TSDB_MGMT_TABLE_STB;
|
||||
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) {
|
||||
|
|
|
@ -396,6 +396,8 @@ static int32_t mndSetUpdateSmaStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
|
|||
stbObj.pColumns = NULL;
|
||||
stbObj.numOfTags = 0;
|
||||
stbObj.pTags = NULL;
|
||||
stbObj.numOfFuncs = 0;
|
||||
stbObj.pFuncs = NULL;
|
||||
stbObj.updateTime = taosGetTimestampMs();
|
||||
stbObj.lock = 0;
|
||||
stbObj.smaVer++;
|
||||
|
@ -408,47 +410,6 @@ static int32_t mndSetUpdateSmaStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStb
|
|||
return 0;
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int32_t mndSetCreateSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SVgObj *pVgroup = NULL;
|
||||
void *pIter = NULL;
|
||||
int32_t contLen;
|
||||
|
||||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pDb->uid) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
||||
void *pReq = mndBuildVCreateSmaReq(pMnode, pVgroup, pSma, &contLen);
|
||||
if (pReq == NULL) {
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
STransAction action = {0};
|
||||
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
|
||||
action.pCont = pReq;
|
||||
action.contLen = contLen;
|
||||
action.msgType = TDMT_VND_CREATE_SMA;
|
||||
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
|
||||
taosMemoryFree(pReq);
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
return -1;
|
||||
}
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int32_t mndSetCreateSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup,
|
||||
SSmaObj *pSma) {
|
||||
SVnodeGid *pVgid = pVgroup->vnodeGid + 0;
|
||||
|
@ -562,6 +523,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
|
|||
streamObj.updateTime = streamObj.createTime;
|
||||
streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name));
|
||||
streamObj.sourceDbUid = pDb->uid;
|
||||
streamObj.targetDbUid = pDb->uid;
|
||||
streamObj.version = 1;
|
||||
streamObj.sql = pCreate->sql;
|
||||
streamObj.smaId = smaObj.uid;
|
||||
|
@ -621,7 +583,6 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
|
|||
if (mndSetCreateSmaCommitLogs(pMnode, pTrans, &smaObj) != 0) goto _OVER;
|
||||
if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER;
|
||||
if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
|
||||
// if (mndSetCreateSmaRedoActions(pMnode, pTrans, pDb, &smaObj) != 0) goto _OVER;
|
||||
if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER;
|
||||
if (mndScheduleStream(pMnode, &streamObj) != 0) goto _OVER;
|
||||
if (mndPersistStream(pMnode, pTrans, &streamObj) != 0) goto _OVER;
|
||||
|
@ -770,49 +731,6 @@ static int32_t mndSetDropSmaVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVg
|
|||
return 0;
|
||||
}
|
||||
|
||||
#if 0
|
||||
static int32_t mndSetDropSmaRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SSmaObj *pSma) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SVgObj *pVgroup = NULL;
|
||||
void *pIter = NULL;
|
||||
int32_t contLen;
|
||||
|
||||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pDb->uid) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
||||
int32_t contLen = 0;
|
||||
void *pReq = mndBuildVDropSmaReq(pMnode, pVgroup, pSma, &contLen);
|
||||
if (pReq == NULL) {
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
STransAction action = {0};
|
||||
action.epSet = mndGetVgroupEpset(pMnode, pVgroup);
|
||||
action.pCont = pReq;
|
||||
action.contLen = contLen;
|
||||
action.msgType = TDMT_VND_DROP_SMA;
|
||||
action.acceptableCode = TSDB_CODE_VND_SMA_NOT_EXIST;
|
||||
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
|
||||
taosMemoryFree(pReq);
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
return -1;
|
||||
}
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
static int32_t mndSetDropSmaVgroupRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) {
|
||||
SVnodeGid *pVgid = pVgroup->vnodeGid + 0;
|
||||
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId);
|
||||
|
@ -879,7 +797,6 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p
|
|||
if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
|
||||
if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
|
||||
if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
|
||||
// if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER;
|
||||
if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER;
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
|
||||
|
||||
|
@ -906,10 +823,20 @@ int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p
|
|||
if (pSma->stbUid == pStb->uid) {
|
||||
pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId);
|
||||
if (pVgroup == NULL) goto _OVER;
|
||||
|
||||
SStreamObj *pStream = mndAcquireStream(pMnode, pSma->name);
|
||||
if (pStream != NULL && pStream->smaId == pSma->uid) {
|
||||
if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) {
|
||||
mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
|
||||
goto _OVER;
|
||||
}
|
||||
if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
|
||||
goto _OVER;
|
||||
}
|
||||
}
|
||||
if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
|
||||
if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER;
|
||||
if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
|
||||
// if (mndSetDropSmaRedoActions(pMnode, pTrans, pDb, pSma) != 0) goto _OVER;
|
||||
mndReleaseVgroup(pMnode, pVgroup);
|
||||
pVgroup = NULL;
|
||||
}
|
||||
|
@ -927,36 +854,26 @@ _OVER:
|
|||
}
|
||||
|
||||
int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SSmaObj *pSma = NULL;
|
||||
void *pIter = NULL;
|
||||
SVgObj *pVgroup = NULL;
|
||||
int32_t code = -1;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
void *pIter = NULL;
|
||||
|
||||
while (1) {
|
||||
SSmaObj *pSma = NULL;
|
||||
pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma);
|
||||
if (pIter == NULL) break;
|
||||
|
||||
if (pSma->dbUid == pDb->uid) {
|
||||
pVgroup = mndAcquireVgroup(pMnode, pSma->dstVgId);
|
||||
if (pVgroup == NULL) goto _OVER;
|
||||
if (mndSetDropSmaVgroupCommitLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER;
|
||||
if (mndSetDropSmaVgroupRedoActions(pMnode, pTrans, pDb, pVgroup) != 0) goto _OVER;
|
||||
if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER;
|
||||
mndReleaseVgroup(pMnode, pVgroup);
|
||||
pVgroup = NULL;
|
||||
if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) {
|
||||
sdbRelease(pSdb, pSma);
|
||||
sdbCancelFetch(pSdb, pSma);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
sdbRelease(pSdb, pSma);
|
||||
}
|
||||
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
sdbRelease(pSdb, pSma);
|
||||
mndReleaseVgroup(pMnode, pVgroup);
|
||||
return code;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) {
|
||||
|
@ -1230,29 +1147,32 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
|
|||
|
||||
SName smaName = {0};
|
||||
tNameFromString(&smaName, pSma->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
|
||||
char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_TO_VARSTR(n1, (char *)tNameGetTableName(&smaName));
|
||||
|
||||
char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_TO_VARSTR(n, (char *)tNameGetTableName(&smaName));
|
||||
cols++;
|
||||
char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_TO_VARSTR(n2, (char *)mndGetDbStr(pDb->name));
|
||||
|
||||
SName stbName = {0};
|
||||
tNameFromString(&stbName, pSma->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
|
||||
|
||||
char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_TO_VARSTR(n1, (char *)tNameGetTableName(&stbName));
|
||||
char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
|
||||
STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName));
|
||||
|
||||
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)n, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)n1, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)n2, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)n3, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&pSma->dstVgId, false);
|
||||
|
||||
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
|
||||
colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false);
|
||||
|
||||
numOfRows++;
|
||||
sdbRelease(pSdb, pSma);
|
||||
}
|
||||
|
|
|
@ -273,6 +273,9 @@ _OVER:
|
|||
}
|
||||
|
||||
static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) {
|
||||
#if 1
|
||||
return TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
#else
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
int32_t code = -1;
|
||||
SSnodeObj *pObj = NULL;
|
||||
|
@ -315,6 +318,7 @@ _OVER:
|
|||
mndReleaseSnode(pMnode, pObj);
|
||||
mndReleaseDnode(pMnode, pDnode);
|
||||
return code;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int32_t mndSetDropSnodeRedoLogs(STrans *pTrans, SSnodeObj *pObj) {
|
||||
|
@ -386,9 +390,12 @@ _OVER:
|
|||
}
|
||||
|
||||
static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) {
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
int32_t code = -1;
|
||||
SSnodeObj *pObj = NULL;
|
||||
#if 1
|
||||
return TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
#else
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
int32_t code = -1;
|
||||
SSnodeObj *pObj = NULL;
|
||||
SMDropSnodeReq dropReq = {0};
|
||||
|
||||
if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
|
||||
|
@ -422,6 +429,7 @@ _OVER:
|
|||
|
||||
mndReleaseSnode(pMnode, pObj);
|
||||
return code;
|
||||
#endif
|
||||
}
|
||||
|
||||
static int32_t mndRetrieveSnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
|
||||
|
|
|
@ -78,7 +78,7 @@ void mndCleanupStb(SMnode *pMnode) {}
|
|||
SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
||||
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen +
|
||||
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + pStb->commentLen +
|
||||
pStb->ast1Len + pStb->ast2Len + STB_RESERVE_SIZE + taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN;
|
||||
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size);
|
||||
if (pRaw == NULL) goto _OVER;
|
||||
|
@ -92,6 +92,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
|
|||
SDB_SET_INT64(pRaw, dataPos, pStb->dbUid, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->tagVer, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->colVer, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->smaVer, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->nextColId, _OVER)
|
||||
SDB_SET_INT64(pRaw, dataPos, pStb->maxdelay[0], _OVER)
|
||||
SDB_SET_INT64(pRaw, dataPos, pStb->maxdelay[1], _OVER)
|
||||
|
@ -100,17 +101,11 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
|
|||
SDB_SET_INT32(pRaw, dataPos, pStb->ttl, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->numOfColumns, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->numOfTags, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->numOfFuncs, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->commentLen, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER)
|
||||
SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER)
|
||||
|
||||
int32_t funcNum = taosArrayGetSize(pStb->pFuncs);
|
||||
SDB_SET_INT32(pRaw, dataPos, funcNum, _OVER)
|
||||
for (int32_t i = 0; i < funcNum; ++i) {
|
||||
char *func = taosArrayGet(pStb->pFuncs, i);
|
||||
SDB_SET_BINARY(pRaw, dataPos, func, TSDB_FUNC_NAME_LEN, _OVER)
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pStb->numOfColumns; ++i) {
|
||||
SSchema *pSchema = &pStb->pColumns[i];
|
||||
SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER)
|
||||
|
@ -129,15 +124,23 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
|
|||
SDB_SET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pStb->numOfFuncs; ++i) {
|
||||
char *func = taosArrayGet(pStb->pFuncs, i);
|
||||
SDB_SET_BINARY(pRaw, dataPos, func, TSDB_FUNC_NAME_LEN, _OVER)
|
||||
}
|
||||
|
||||
if (pStb->commentLen > 0) {
|
||||
SDB_SET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen + 1, _OVER)
|
||||
}
|
||||
|
||||
if (pStb->ast1Len > 0) {
|
||||
SDB_SET_BINARY(pRaw, dataPos, pStb->pAst1, pStb->ast1Len, _OVER)
|
||||
}
|
||||
|
||||
if (pStb->ast2Len > 0) {
|
||||
SDB_SET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER)
|
||||
}
|
||||
|
||||
SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER)
|
||||
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
|
||||
|
||||
|
@ -180,6 +183,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
|
|||
SDB_GET_INT64(pRaw, dataPos, &pStb->dbUid, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->tagVer, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->colVer, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->smaVer, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->nextColId, _OVER)
|
||||
SDB_GET_INT64(pRaw, dataPos, &pStb->maxdelay[0], _OVER)
|
||||
SDB_GET_INT64(pRaw, dataPos, &pStb->maxdelay[1], _OVER)
|
||||
|
@ -188,27 +192,15 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
|
|||
SDB_GET_INT32(pRaw, dataPos, &pStb->ttl, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfColumns, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfTags, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->numOfFuncs, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->commentLen, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER)
|
||||
SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER)
|
||||
|
||||
int32_t funcNum = 0;
|
||||
SDB_GET_INT32(pRaw, dataPos, &funcNum, _OVER)
|
||||
if (funcNum > 0) {
|
||||
pStb->pFuncs = taosArrayInit(funcNum, TSDB_FUNC_NAME_LEN);
|
||||
if (NULL == pStb->pFuncs) {
|
||||
goto _OVER;
|
||||
}
|
||||
char funcName[TSDB_FUNC_NAME_LEN];
|
||||
for (int32_t i = 0; i < funcNum; ++i) {
|
||||
SDB_GET_BINARY(pRaw, dataPos, funcName, TSDB_FUNC_NAME_LEN, _OVER)
|
||||
taosArrayPush(pStb->pFuncs, funcName);
|
||||
}
|
||||
}
|
||||
|
||||
pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema));
|
||||
pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema));
|
||||
if (pStb->pColumns == NULL || pStb->pTags == NULL) {
|
||||
pStb->pFuncs = taosArrayInit(pStb->numOfFuncs, TSDB_FUNC_NAME_LEN);
|
||||
if (pStb->pColumns == NULL || pStb->pTags == NULL || pStb->pFuncs == NULL) {
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
|
@ -230,16 +222,24 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
|
|||
SDB_GET_BINARY(pRaw, dataPos, pSchema->name, TSDB_COL_NAME_LEN, _OVER)
|
||||
}
|
||||
|
||||
for (int32_t i = 0; i < pStb->numOfFuncs; ++i) {
|
||||
char funcName[TSDB_FUNC_NAME_LEN] = {0};
|
||||
SDB_GET_BINARY(pRaw, dataPos, funcName, TSDB_FUNC_NAME_LEN, _OVER)
|
||||
taosArrayPush(pStb->pFuncs, funcName);
|
||||
}
|
||||
|
||||
if (pStb->commentLen > 0) {
|
||||
pStb->comment = taosMemoryCalloc(pStb->commentLen + 1, 1);
|
||||
if (pStb->comment == NULL) goto _OVER;
|
||||
SDB_GET_BINARY(pRaw, dataPos, pStb->comment, pStb->commentLen + 1, _OVER)
|
||||
}
|
||||
|
||||
if (pStb->ast1Len > 0) {
|
||||
pStb->pAst1 = taosMemoryCalloc(pStb->ast1Len, 1);
|
||||
if (pStb->pAst1 == NULL) goto _OVER;
|
||||
SDB_GET_BINARY(pRaw, dataPos, pStb->pAst1, pStb->ast1Len, _OVER)
|
||||
}
|
||||
|
||||
if (pStb->ast2Len > 0) {
|
||||
pStb->pAst2 = taosMemoryCalloc(pStb->ast2Len, 1);
|
||||
if (pStb->pAst2 == NULL) goto _OVER;
|
||||
|
@ -273,6 +273,7 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) {
|
|||
taosMemoryFreeClear(pStb->pColumns);
|
||||
taosMemoryFreeClear(pStb->pTags);
|
||||
taosMemoryFreeClear(pStb->comment);
|
||||
taosMemoryFreeClear(pStb->pFuncs);
|
||||
taosMemoryFreeClear(pStb->pAst1);
|
||||
taosMemoryFreeClear(pStb->pAst2);
|
||||
taosArrayDestroy(pStb->pFuncs);
|
||||
|
@ -322,7 +323,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
|
|||
pOld->commentLen = pNew->commentLen;
|
||||
|
||||
if (pOld->ast1Len < pNew->ast1Len) {
|
||||
void *pAst1 = taosMemoryMalloc(pNew->ast1Len);
|
||||
void *pAst1 = taosMemoryMalloc(pNew->ast1Len + 1);
|
||||
if (pAst1 != NULL) {
|
||||
taosMemoryFree(pOld->pAst1);
|
||||
pOld->pAst1 = pAst1;
|
||||
|
@ -334,7 +335,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
|
|||
}
|
||||
|
||||
if (pOld->ast2Len < pNew->ast2Len) {
|
||||
void *pAst2 = taosMemoryMalloc(pNew->ast2Len);
|
||||
void *pAst2 = taosMemoryMalloc(pNew->ast2Len + 1);
|
||||
if (pAst2 != NULL) {
|
||||
taosMemoryFree(pOld->pAst2);
|
||||
pOld->pAst2 = pAst2;
|
||||
|
@ -361,12 +362,15 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
|
|||
}
|
||||
if (pNew->commentLen > 0) {
|
||||
memcpy(pOld->comment, pNew->comment, pNew->commentLen + 1);
|
||||
pOld->commentLen = pNew->commentLen;
|
||||
}
|
||||
if (pNew->ast1Len != 0) {
|
||||
memcpy(pOld->pAst1, pNew->pAst1, pNew->ast1Len);
|
||||
pOld->ast1Len = pNew->ast1Len;
|
||||
}
|
||||
if (pNew->ast2Len != 0) {
|
||||
memcpy(pOld->pAst2, pNew->pAst2, pNew->ast2Len);
|
||||
pOld->ast2Len = pNew->ast2Len;
|
||||
}
|
||||
taosWUnLockLatch(&pOld->lock);
|
||||
return 0;
|
||||
|
@ -575,7 +579,10 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
|
|||
static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
|
||||
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
|
||||
if (pRedoRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
|
||||
sdbFreeRaw(pRedoRaw);
|
||||
return -1;
|
||||
}
|
||||
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
|
@ -584,7 +591,10 @@ static int32_t mndSetCreateStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
|
|||
static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
|
||||
SSdbRaw *pUndoRaw = mndStbActionEncode(pStb);
|
||||
if (pUndoRaw == NULL) return -1;
|
||||
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
|
||||
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
|
||||
sdbFreeRaw(pUndoRaw);
|
||||
return -1;
|
||||
}
|
||||
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
|
@ -593,7 +603,10 @@ static int32_t mndSetCreateStbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
|
|||
static int32_t mndSetCreateStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
|
||||
SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
|
||||
if (pCommitRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
|
||||
sdbFreeRaw(pCommitRaw);
|
||||
return -1;
|
||||
}
|
||||
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
|
@ -608,7 +621,7 @@ static int32_t mndSetCreateStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
|
|||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pDb->uid) {
|
||||
if (!mndVgroupInDb(pVgroup, pDb->uid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
@ -646,7 +659,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
|
|||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pDb->uid) {
|
||||
if (!mndVgroupInDb(pVgroup, pDb->uid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
@ -697,6 +710,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
|
|||
pDst->dbUid = pDb->uid;
|
||||
pDst->tagVer = 1;
|
||||
pDst->colVer = 1;
|
||||
pDst->smaVer = 1;
|
||||
pDst->nextColId = 1;
|
||||
pDst->maxdelay[0] = pCreate->delay1;
|
||||
pDst->maxdelay[1] = pCreate->delay2;
|
||||
|
@ -705,6 +719,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
|
|||
pDst->ttl = pCreate->ttl;
|
||||
pDst->numOfColumns = pCreate->numOfColumns;
|
||||
pDst->numOfTags = pCreate->numOfTags;
|
||||
pDst->numOfFuncs = pCreate->numOfFuncs;
|
||||
pDst->commentLen = pCreate->commentLen;
|
||||
pDst->pFuncs = pCreate->pFuncs;
|
||||
pCreate->pFuncs = NULL;
|
||||
|
@ -715,7 +730,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
|
|||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
memcpy(pDst->comment, pCreate->comment, pDst->commentLen + 1);
|
||||
memcpy(pDst->comment, pCreate->pComment, pDst->commentLen + 1);
|
||||
}
|
||||
|
||||
pDst->ast1Len = pCreate->ast1Len;
|
||||
|
@ -770,20 +785,15 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
|
|||
|
||||
static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
|
||||
SStbObj stbObj = {0};
|
||||
|
||||
int32_t code = -1;
|
||||
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq);
|
||||
if (pTrans == NULL) goto _OVER;
|
||||
|
||||
mDebug("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
|
||||
|
||||
if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
|
||||
|
||||
if (mndAddStbToTrans(pMnode, pTrans, pDb, &stbObj) < 0) goto _OVER;
|
||||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
|
||||
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
@ -906,7 +916,8 @@ _OVER:
|
|||
}
|
||||
|
||||
static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
|
||||
if (pAlter->commentLen >= 0 || pAlter->ttl != 0) return 0;
|
||||
if (pAlter->commentLen >= 0) return 0;
|
||||
if (pAlter->ttl != 0) return 0;
|
||||
|
||||
if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) {
|
||||
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
|
||||
|
@ -969,6 +980,7 @@ static int32_t mndUpdateStbCommentAndTTL(const SStbObj *pOld, SStbObj *pNew, cha
|
|||
memcpy(pNew->comment, pComment, commentLen + 1);
|
||||
} else if (commentLen == 0) {
|
||||
pNew->commentLen = 0;
|
||||
} else {
|
||||
}
|
||||
|
||||
if (ttl >= 0) {
|
||||
|
@ -1245,7 +1257,10 @@ static int32_t mndAlterStbColumnBytes(SMnode *pMnode, const SStbObj *pOld, SStbO
|
|||
static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
|
||||
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
|
||||
if (pRedoRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
|
||||
sdbFreeRaw(pRedoRaw);
|
||||
return -1;
|
||||
}
|
||||
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
|
@ -1254,7 +1269,10 @@ static int32_t mndSetAlterStbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
|
|||
static int32_t mndSetAlterStbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) {
|
||||
SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
|
||||
if (pCommitRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
|
||||
sdbFreeRaw(pCommitRaw);
|
||||
return -1;
|
||||
}
|
||||
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
|
@ -1269,7 +1287,7 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj
|
|||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pDb->uid) {
|
||||
if (!mndVgroupInDb(pVgroup, pDb->uid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
@ -1388,7 +1406,7 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
|
|||
pSchema->bytes = pSrcSchema->bytes;
|
||||
}
|
||||
|
||||
if (pStb->pFuncs) {
|
||||
if (pStb->numOfFuncs > 0) {
|
||||
pRsp->pFuncs = taosArrayDup(pStb->pFuncs);
|
||||
}
|
||||
|
||||
|
@ -1626,7 +1644,10 @@ _OVER:
|
|||
static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
|
||||
SSdbRaw *pRedoRaw = mndStbActionEncode(pStb);
|
||||
if (pRedoRaw == NULL) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
|
||||
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
|
||||
sdbFreeRaw(pRedoRaw);
|
||||
return -1;
|
||||
}
|
||||
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
|
@ -1635,7 +1656,10 @@ static int32_t mndSetDropStbRedoLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pS
|
|||
static int32_t mndSetDropStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) {
|
||||
SSdbRaw *pCommitRaw = mndStbActionEncode(pStb);
|
||||
if (pCommitRaw == NULL) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
|
||||
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
|
||||
sdbFreeRaw(pCommitRaw);
|
||||
return -1;
|
||||
}
|
||||
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
|
||||
|
||||
return 0;
|
||||
|
@ -1649,7 +1673,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
|
|||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
|
||||
if (pIter == NULL) break;
|
||||
if (pVgroup->dbUid != pDb->uid) {
|
||||
if (!mndVgroupInDb(pVgroup, pDb->uid)) {
|
||||
sdbRelease(pSdb, pVgroup);
|
||||
continue;
|
||||
}
|
||||
|
@ -1683,7 +1707,7 @@ static int32_t mndSetDropStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *
|
|||
|
||||
static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb) {
|
||||
int32_t code = -1;
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq);
|
||||
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq);
|
||||
if (pTrans == NULL) goto _OVER;
|
||||
|
||||
mDebug("trans:%d, used to drop stb:%s", pTrans->id, pStb->name);
|
||||
|
|
|
@ -247,8 +247,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
|
|||
pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name));
|
||||
pObj->status = 0;
|
||||
|
||||
// TODO
|
||||
pObj->dropPolicy = 0;
|
||||
pObj->igExpired = pCreate->igExpired;
|
||||
pObj->trigger = pCreate->triggerType;
|
||||
pObj->triggerParam = pCreate->maxDelay;
|
||||
pObj->watermark = pCreate->watermark;
|
||||
|
@ -301,6 +300,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
|
|||
.streamQuery = true,
|
||||
.triggerType = pObj->trigger == STREAM_TRIGGER_MAX_DELAY ? STREAM_TRIGGER_WINDOW_CLOSE : pObj->trigger,
|
||||
.watermark = pObj->watermark,
|
||||
.igExpired = pObj->igExpired,
|
||||
};
|
||||
|
||||
// using ast and param to build physical plan
|
||||
|
@ -673,27 +673,29 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
|
|||
|
||||
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
void *pIter = NULL;
|
||||
|
||||
void *pIter = NULL;
|
||||
SStreamObj *pStream = NULL;
|
||||
while (1) {
|
||||
SStreamObj *pStream = NULL;
|
||||
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
|
||||
if (pIter == NULL) break;
|
||||
|
||||
if (pStream->sourceDbUid == pDb->uid || pStream->targetDbUid == pDb->uid) {
|
||||
if (pStream->sourceDbUid != pStream->targetDbUid) {
|
||||
sdbRelease(pSdb, pStream);
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64,
|
||||
pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid);
|
||||
terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
|
||||
return -1;
|
||||
} else {
|
||||
// TODO drop all task on snode
|
||||
if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) {
|
||||
sdbRelease(pSdb, pStream);
|
||||
sdbCancelFetch(pSdb, pIter);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
sdbRelease(pSdb, pStream);
|
||||
continue;
|
||||
}
|
||||
|
||||
#if 0
|
||||
|
|
|
@ -90,7 +90,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
|
|||
for (int32_t i = 0; i < actionNum; ++i) {
|
||||
STransAction *pAction = taosArrayGet(pArray, i);
|
||||
if (pAction->actionType == TRANS_ACTION_RAW) {
|
||||
rawDataLen += (sdbGetRawTotalSize(pAction->pRaw) + sizeof(int32_t));
|
||||
rawDataLen += (sizeof(STransAction) + sdbGetRawTotalSize(pAction->pRaw));
|
||||
} else if (pAction->actionType == TRANS_ACTION_MSG) {
|
||||
rawDataLen += (sizeof(STransAction) + pAction->contLen);
|
||||
} else {
|
||||
|
@ -105,7 +105,7 @@ static int32_t mndTransGetActionsSize(SArray *pArray) {
|
|||
static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
|
||||
int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE;
|
||||
int32_t rawDataLen = sizeof(STrans) + TRANS_RESERVE_SIZE + pTrans->paramLen;
|
||||
rawDataLen += mndTransGetActionsSize(pTrans->redoActions);
|
||||
rawDataLen += mndTransGetActionsSize(pTrans->undoActions);
|
||||
rawDataLen += mndTransGetActionsSize(pTrans->commitActions);
|
||||
|
@ -226,7 +226,8 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) {
|
|||
|
||||
_OVER:
|
||||
if (terrno != 0) {
|
||||
mError("trans:%d, failed to encode to raw:%p len:%d since %s", pTrans->id, pRaw, dataPos, terrstr());
|
||||
mError("trans:%d, failed to encode to raw:%p maxlen:%d len:%d since %s", pTrans->id, pRaw, sdbGetRawTotalSize(pRaw),
|
||||
dataPos, terrstr());
|
||||
sdbFreeRaw(pRaw);
|
||||
return NULL;
|
||||
}
|
||||
|
@ -1025,7 +1026,7 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction
|
|||
pTrans->lastAction = pAction->id;
|
||||
pTrans->lastMsgType = pAction->msgType;
|
||||
pTrans->lastEpset = pAction->epSet;
|
||||
pTrans->lastErrorNo == 0;
|
||||
pTrans->lastErrorNo = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -1146,13 +1147,13 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans)
|
|||
} else {
|
||||
code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
}
|
||||
}
|
||||
if (pAction->rawWritten) {
|
||||
} else if (pAction->rawWritten) {
|
||||
if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) {
|
||||
code = pAction->errCode;
|
||||
} else {
|
||||
mDebug("trans:%d, %s:%d write successfully", pTrans->id, mndTransStr(pAction->stage), action);
|
||||
}
|
||||
} else {
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -426,7 +426,15 @@ static int32_t mndCompareDnodeId(int32_t *dnode1Id, int32_t *dnode2Id) { return
|
|||
static int32_t mndCompareDnodeVnodes(SDnodeObj *pDnode1, SDnodeObj *pDnode2) {
|
||||
float d1Score = (float)pDnode1->numOfVnodes / pDnode1->numOfSupportVnodes;
|
||||
float d2Score = (float)pDnode2->numOfVnodes / pDnode2->numOfSupportVnodes;
|
||||
#if 0
|
||||
if (d1Score == d2Score) {
|
||||
return pDnode2->id - pDnode1->id;
|
||||
} else {
|
||||
return d1Score >= d2Score ? 1 : 0;
|
||||
}
|
||||
#else
|
||||
return d1Score >= d2Score ? 1 : 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
void mndSortVnodeGid(SVgObj *pVgroup) {
|
||||
|
@ -1211,6 +1219,9 @@ _OVER:
|
|||
}
|
||||
|
||||
static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) {
|
||||
#if 1
|
||||
return TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
#else
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
SDnodeObj *pNew1 = NULL;
|
||||
SDnodeObj *pNew2 = NULL;
|
||||
|
@ -1404,6 +1415,7 @@ _OVER:
|
|||
mndReleaseDb(pMnode, pDb);
|
||||
|
||||
return code;
|
||||
#endif
|
||||
}
|
||||
|
||||
int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, SArray *pArray) {
|
||||
|
@ -1703,6 +1715,9 @@ _OVER:
|
|||
}
|
||||
|
||||
static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) {
|
||||
#if 1
|
||||
return TSDB_CODE_OPS_NOT_SUPPORT;
|
||||
#else
|
||||
SMnode *pMnode = pReq->info.node;
|
||||
int32_t code = -1;
|
||||
SArray *pArray = NULL;
|
||||
|
@ -1751,4 +1766,7 @@ _OVER:
|
|||
|
||||
taosArrayDestroy(pArray);
|
||||
return code;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid) { return !pVgroup->isTsma && pVgroup->dbUid == dbUid; }
|
|
@ -93,7 +93,7 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) {
|
|||
|
||||
SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen);
|
||||
ASSERT_NE(pRsp, nullptr);
|
||||
ASSERT_EQ(pRsp->code, 0);
|
||||
ASSERT_EQ(pRsp->code, TSDB_CODE_OPS_NOT_SUPPORT);
|
||||
}
|
||||
|
||||
test.SendShowReq(TSDB_MGMT_TABLE_DB, "user_databases", "");
|
||||
|
|
|
@ -31,6 +31,8 @@ target_sources(
|
|||
"src/sma/smaEnv.c"
|
||||
"src/sma/smaUtil.c"
|
||||
"src/sma/smaOpen.c"
|
||||
"src/sma/smaCommit.c"
|
||||
"src/sma/smaSnapshot.c"
|
||||
"src/sma/smaRollup.c"
|
||||
"src/sma/smaTimeRange.c"
|
||||
|
||||
|
|
|
@ -62,12 +62,10 @@ struct STSmaStat {
|
|||
|
||||
struct SRSmaStat {
|
||||
SSma *pSma;
|
||||
int64_t refId; // shared by persistence/fetch tasks
|
||||
void *tmrHandle; // for persistence task
|
||||
tmr_h tmrId; // for persistence task
|
||||
int32_t tmrSeconds; // for persistence task
|
||||
int8_t triggerStat; // for persistence task
|
||||
int8_t runningStat; // for persistence task
|
||||
int64_t refId; // shared by fetch tasks
|
||||
void *tmrHandle; // shared by fetch tasks
|
||||
int8_t triggerStat; // shared by fetch tasks
|
||||
int8_t runningStat; // for persistence task
|
||||
SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo;
|
||||
};
|
||||
|
||||
|
@ -82,7 +80,6 @@ struct SSmaStat {
|
|||
#define SMA_TSMA_STAT(s) (&(s)->tsmaStat)
|
||||
#define SMA_RSMA_STAT(s) (&(s)->rsmaStat)
|
||||
#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash)
|
||||
#define RSMA_TMR_ID(r) ((r)->tmrId)
|
||||
#define RSMA_TMR_HANDLE(r) ((r)->tmrHandle)
|
||||
#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat)
|
||||
#define RSMA_RUNNING_STAT(r) (&(r)->runningStat)
|
||||
|
@ -92,8 +89,9 @@ enum {
|
|||
TASK_TRIGGER_STAT_INIT = 0,
|
||||
TASK_TRIGGER_STAT_ACTIVE = 1,
|
||||
TASK_TRIGGER_STAT_INACTIVE = 2,
|
||||
TASK_TRIGGER_STAT_CANCELLED = 3,
|
||||
TASK_TRIGGER_STAT_FINISHED = 4,
|
||||
TASK_TRIGGER_STAT_PAUSED = 3,
|
||||
TASK_TRIGGER_STAT_CANCELLED = 4,
|
||||
TASK_TRIGGER_STAT_FINISHED = 5,
|
||||
};
|
||||
void tdDestroySmaEnv(SSmaEnv *pSmaEnv);
|
||||
void *tdFreeSmaEnv(SSmaEnv *pSmaEnv);
|
||||
|
@ -184,9 +182,11 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) {
|
|||
static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType);
|
||||
void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType);
|
||||
void *tdFreeRSmaInfo(SRSmaInfo *pInfo);
|
||||
int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat);
|
||||
|
||||
int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName);
|
||||
int32_t tdProcessRSmaRestoreImpl(SSma *pSma);
|
||||
|
||||
int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg);
|
||||
int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg);
|
||||
int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
|
||||
|
@ -214,25 +214,22 @@ struct STFInfo {
|
|||
};
|
||||
|
||||
struct STFile {
|
||||
STFInfo info;
|
||||
STfsFile f;
|
||||
TdFilePtr pFile;
|
||||
uint8_t state;
|
||||
STFInfo info;
|
||||
char *fname;
|
||||
TdFilePtr pFile;
|
||||
};
|
||||
|
||||
#define TD_TFILE_F(tf) (&((tf)->f))
|
||||
#define TD_TFILE_PFILE(tf) ((tf)->pFile)
|
||||
#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL)
|
||||
#define TD_TFILE_FULL_NAME(tf) (TD_TFILE_F(tf)->aname)
|
||||
#define TD_TFILE_REL_NAME(tf) (TD_TFILE_F(tf)->rname)
|
||||
#define TD_TFILE_FULL_NAME(tf) ((tf)->fname)
|
||||
#define TD_TFILE_OPENED(tf) (TD_TFILE_PFILE(tf) != NULL)
|
||||
#define TD_TFILE_CLOSED(tf) (!TD_TFILE_OPENED(tf))
|
||||
#define TD_TFILE_SET_CLOSED(f) (TD_TFILE_PFILE(f) = NULL)
|
||||
#define TD_TFILE_SET_STATE(tf, s) ((tf)->state = (s))
|
||||
#define TD_TFILE_DID(tf) (TD_TFILE_F(tf)->did)
|
||||
|
||||
int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname);
|
||||
int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType);
|
||||
int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname);
|
||||
int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType);
|
||||
int32_t tdOpenTFile(STFile *pTFile, int flags);
|
||||
int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte);
|
||||
int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence);
|
||||
|
@ -244,8 +241,10 @@ int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo);
|
|||
int32_t tdUpdateTFileHeader(STFile *pTFile);
|
||||
void tdUpdateTFileMagic(STFile *pTFile, void *pCksm);
|
||||
void tdCloseTFile(STFile *pTFile);
|
||||
void tdDestroyTFile(STFile *pTFile);
|
||||
|
||||
void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName);
|
||||
void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version, char *outputName);
|
||||
void tdGetVndDirName(int32_t vgId,const char *pdname, const char *dname, bool endWithSep, char *outputName);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
@ -157,7 +157,7 @@ typedef struct {
|
|||
static STqMgmt tqMgmt = {0};
|
||||
|
||||
// tqRead
|
||||
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** pHeadWithCkSum);
|
||||
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum);
|
||||
|
||||
// tqExec
|
||||
int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId);
|
||||
|
@ -178,6 +178,7 @@ STqOffsetStore* tqOffsetOpen();
|
|||
void tqOffsetClose(STqOffsetStore*);
|
||||
STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey);
|
||||
int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset);
|
||||
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey);
|
||||
int32_t tqOffsetSnapshot(STqOffsetStore* pStore);
|
||||
|
||||
// tqSink
|
||||
|
|
|
@ -163,8 +163,11 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool
|
|||
|
||||
// sma
|
||||
int32_t smaOpen(SVnode* pVnode);
|
||||
int32_t smaCloseEnv(SSma* pSma);
|
||||
int32_t smaCloseEx(SSma* pSma);
|
||||
int32_t smaClose(SSma* pSma);
|
||||
int32_t smaBegin(SSma* pSma);
|
||||
int32_t smaPreCommit(SSma* pSma);
|
||||
int32_t smaCommit(SSma* pSma);
|
||||
int32_t smaPostCommit(SSma* pSma);
|
||||
|
||||
int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg);
|
||||
int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg);
|
||||
|
@ -238,7 +241,7 @@ struct SVnode {
|
|||
tsem_t canCommit;
|
||||
int64_t sync;
|
||||
int32_t syncCount;
|
||||
sem_t syncSem;
|
||||
tsem_t syncSem;
|
||||
SQHandle* pQuery;
|
||||
};
|
||||
|
||||
|
|
|
@ -0,0 +1,225 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "sma.h"
|
||||
|
||||
static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma);
|
||||
static int32_t tdProcessRSmaCommitImpl(SSma *pSma);
|
||||
static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma);
|
||||
|
||||
/**
|
||||
* @brief Only applicable to Rollup SMA
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); }
|
||||
|
||||
/**
|
||||
* @brief Only applicable to Rollup SMA
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); }
|
||||
|
||||
/**
|
||||
* @brief Only applicable to Rollup SMA
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); }
|
||||
|
||||
/**
|
||||
* @brief set rsma trigger stat active
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
int32_t smaBegin(SSma *pSma) {
|
||||
SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
|
||||
if (!pSmaEnv) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv);
|
||||
SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
|
||||
|
||||
int8_t rsmaTriggerStat =
|
||||
atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED, TASK_TRIGGER_STAT_ACTIVE);
|
||||
switch (rsmaTriggerStat) {
|
||||
case TASK_TRIGGER_STAT_PAUSED: {
|
||||
smaDebug("vgId:%d rsma trigger stat from paused to active", SMA_VID(pSma));
|
||||
break;
|
||||
}
|
||||
case TASK_TRIGGER_STAT_INIT: {
|
||||
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE);
|
||||
smaDebug("vgId:%d rsma trigger stat from init to active", SMA_VID(pSma));
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE);
|
||||
smaWarn("vgId:%d rsma trigger stat %" PRIi8 " is unexpected", SMA_VID(pSma), rsmaTriggerStat);
|
||||
ASSERT(0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief pre-commit for rollup sma.
|
||||
* 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED.
|
||||
* 2) perform persist task for qTaskInfo
|
||||
* 3) wait all triggered fetch tasks finished
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) {
|
||||
SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
|
||||
if (!pSmaEnv) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv);
|
||||
SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat);
|
||||
|
||||
|
||||
// step 1: set persistence task paused
|
||||
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED);
|
||||
|
||||
// step 2: perform persist task for qTaskInfo
|
||||
tdRSmaPersistExecImpl(pRSmaStat);
|
||||
|
||||
// step 3: wait all triggered fetch tasks finished
|
||||
int32_t nLoops = 0;
|
||||
while (1) {
|
||||
if (T_REF_VAL_GET(pStat) == 0) {
|
||||
smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
|
||||
break;
|
||||
} else {
|
||||
smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
|
||||
}
|
||||
++nLoops;
|
||||
if (nLoops > 1000) {
|
||||
sched_yield();
|
||||
nLoops = 0;
|
||||
}
|
||||
}
|
||||
|
||||
smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma));
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief commit for rollup sma
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
static int32_t tdProcessRSmaCommitImpl(SSma *pSma) {
|
||||
SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma);
|
||||
if (!pSmaEnv) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief post-commit for rollup sma
|
||||
* 1) clean up the outdated qtaskinfo files
|
||||
*
|
||||
* @param pSma
|
||||
* @return int32_t
|
||||
*/
|
||||
static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) {
|
||||
SVnode *pVnode = pSma->pVnode;
|
||||
|
||||
if (!VND_IS_RSMA(pVnode)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int64_t committed = pVnode->state.committed;
|
||||
TdDirPtr pDir = NULL;
|
||||
TdDirEntryPtr pDirEntry = NULL;
|
||||
char dir[TSDB_FILENAME_LEN];
|
||||
const char *pattern = "v[0-9]+qtaskinfo\\.ver([0-9]+)?$";
|
||||
regex_t regex;
|
||||
int code = 0;
|
||||
|
||||
tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir);
|
||||
|
||||
// Resource allocation and init
|
||||
if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) {
|
||||
char errbuf[128];
|
||||
regerror(code, ®ex, errbuf, sizeof(errbuf));
|
||||
smaWarn("vgId:%d, rsma post commit, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
if ((pDir = taosOpenDir(dir)) == NULL) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr());
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
int32_t dirLen = strlen(dir);
|
||||
char *dirEnd = POINTER_SHIFT(dir, dirLen);
|
||||
regmatch_t regMatch[2];
|
||||
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
|
||||
char *entryName = taosGetDirEntryName(pDirEntry);
|
||||
if (!entryName) {
|
||||
continue;
|
||||
}
|
||||
|
||||
code = regexec(®ex, entryName, 2, regMatch, 0);
|
||||
|
||||
if (code == 0) {
|
||||
// match
|
||||
int64_t version = -1;
|
||||
sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &version);
|
||||
if ((version < committed) && (version > -1)) {
|
||||
strncpy(dirEnd, entryName, TSDB_FILENAME_LEN - dirLen);
|
||||
if (taosRemoveFile(dir) != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
smaWarn("vgId:%d, committed version:%" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed,
|
||||
dir, terrstr());
|
||||
} else {
|
||||
smaDebug("vgId:%d, committed version:%" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, dir);
|
||||
}
|
||||
}
|
||||
} else if (code == REG_NOMATCH) {
|
||||
// not match
|
||||
smaTrace("vgId:%d, rsma post commit, not match %s", TD_VID(pVnode), entryName);
|
||||
continue;
|
||||
} else {
|
||||
// has other error
|
||||
char errbuf[128];
|
||||
regerror(code, ®ex, errbuf, sizeof(errbuf));
|
||||
smaWarn("vgId:%d, rsma post commit, regexec failed since %s", TD_VID(pVnode), errbuf);
|
||||
|
||||
taosCloseDir(&pDir);
|
||||
regfree(®ex);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
taosCloseDir(&pDir);
|
||||
regfree(®ex);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
|
@ -132,6 +132,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS
|
|||
if (smaType == TSDB_SMA_TYPE_ROLLUP) {
|
||||
SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat);
|
||||
pRSmaStat->pSma = (SSma *)pSma;
|
||||
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT);
|
||||
|
||||
// init smaMgmt
|
||||
smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat);
|
||||
|
@ -192,22 +193,20 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) {
|
|||
static void tdDestroyRSmaStat(void *pRSmaStat) {
|
||||
if (pRSmaStat) {
|
||||
SRSmaStat *pStat = (SRSmaStat *)pRSmaStat;
|
||||
smaDebug("vgId:%d %s:%d destroy rsma stat %p", SMA_VID(pStat->pSma), __func__, __LINE__, pRSmaStat);
|
||||
// step 1: set persistence task cancelled
|
||||
SSma *pSma = pStat->pSma;
|
||||
smaDebug("vgId:%d, destroy rsma stat %p", SMA_VID(pSma), pRSmaStat);
|
||||
// step 1: set rsma trigger stat cancelled
|
||||
atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED);
|
||||
|
||||
// step 2: stop the persistence timer
|
||||
taosTmrStopA(&RSMA_TMR_ID(pStat));
|
||||
|
||||
// step 3: wait the persistence thread to finish
|
||||
// step 2: wait the persistence thread to finish
|
||||
int32_t nLoops = 0;
|
||||
if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) {
|
||||
while (1) {
|
||||
if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) {
|
||||
smaDebug("rsma, persist task finished already");
|
||||
smaDebug("vgId:%d, rsma persist task finished already", SMA_VID(pSma));
|
||||
break;
|
||||
} else {
|
||||
smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8,
|
||||
smaDebug("vgId:%d, rsma persist task not finished yet since rsma stat in %" PRIi8, SMA_VID(pSma),
|
||||
atomic_load_8(RSMA_TRIGGER_STAT(pStat)));
|
||||
}
|
||||
++nLoops;
|
||||
|
@ -218,13 +217,15 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
|
|||
}
|
||||
}
|
||||
|
||||
// step 4: destroy the rsma info and associated fetch tasks
|
||||
// step 3: destroy the rsma info and associated fetch tasks
|
||||
// TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready.
|
||||
void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
|
||||
while (infoHash) {
|
||||
SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
|
||||
tdFreeRSmaInfo(pSmaInfo);
|
||||
infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
|
||||
if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) {
|
||||
void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL);
|
||||
while (infoHash) {
|
||||
SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash;
|
||||
tdFreeRSmaInfo(pSmaInfo);
|
||||
infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash);
|
||||
}
|
||||
}
|
||||
taosHashCleanup(RSMA_INFO_HASH(pStat));
|
||||
|
||||
|
@ -232,10 +233,10 @@ static void tdDestroyRSmaStat(void *pRSmaStat) {
|
|||
nLoops = 0;
|
||||
while (1) {
|
||||
if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) {
|
||||
smaDebug("rsma, all fetch task finished already");
|
||||
smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma));
|
||||
break;
|
||||
} else {
|
||||
smaDebug("rsma, fetch tasks not all finished yet");
|
||||
smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma));
|
||||
}
|
||||
++nLoops;
|
||||
if (nLoops > 1000) {
|
||||
|
@ -275,7 +276,7 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) {
|
|||
} else if (smaType == TSDB_SMA_TYPE_ROLLUP) {
|
||||
SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat);
|
||||
if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) {
|
||||
smaError("remove refId from smaRef failed, refId:0x%" PRIx64, RSMA_REF_ID(pRSmaStat));
|
||||
smaError("remove refId from rsmaRef:0x%" PRIx64 " failed since %s", RSMA_REF_ID(pRSmaStat), terrstr());
|
||||
}
|
||||
} else {
|
||||
ASSERT(0);
|
||||
|
|
|
@ -135,17 +135,11 @@ _err:
|
|||
return -1;
|
||||
}
|
||||
|
||||
int32_t smaCloseEnv(SSma *pSma) {
|
||||
if (pSma) {
|
||||
SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
|
||||
SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t smaCloseEx(SSma *pSma) {
|
||||
int32_t smaClose(SSma *pSma) {
|
||||
if (pSma) {
|
||||
taosThreadMutexDestroy(&pSma->mutex);
|
||||
SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
|
||||
SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
|
||||
if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
|
||||
if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
|
||||
if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));
|
||||
|
|
|
@ -15,16 +15,14 @@
|
|||
|
||||
#include "sma.h"
|
||||
|
||||
#define RSMA_QTASKINFO_PERSIST_MS 7200000
|
||||
#define RSMA_QTASKINFO_BUFSIZE 32768
|
||||
#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid
|
||||
#define RSMA_QTASKINFO_BUFSIZE 32768
|
||||
#define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid
|
||||
|
||||
SSmaMgmt smaMgmt = {
|
||||
.smaRef = -1,
|
||||
};
|
||||
|
||||
typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T;
|
||||
static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"};
|
||||
#define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver"
|
||||
typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem;
|
||||
typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter;
|
||||
|
||||
|
@ -37,16 +35,16 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType
|
|||
static void tdRSmaFetchTrigger(void *param, void *tmrId);
|
||||
static void tdRSmaPersistTrigger(void *param, void *tmrId);
|
||||
static void *tdRSmaPersistExec(void *param);
|
||||
static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName);
|
||||
static void tdRSmaQTaskInfoGetFName(int32_t vid, int64_t version, char *outputName);
|
||||
|
||||
static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile);
|
||||
static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish);
|
||||
static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter);
|
||||
static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *infoItem);
|
||||
|
||||
static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma);
|
||||
static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma);
|
||||
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma);
|
||||
static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables);
|
||||
static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed);
|
||||
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed);
|
||||
|
||||
struct SRSmaInfoItem {
|
||||
SRSmaInfo *pRsmaInfo;
|
||||
|
@ -88,8 +86,8 @@ struct SRSmaQTaskInfoIter {
|
|||
int32_t nBufPos;
|
||||
};
|
||||
|
||||
static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) {
|
||||
tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName);
|
||||
static void tdRSmaQTaskInfoGetFName(int32_t vgId, int64_t version, char *outputName) {
|
||||
tdGetVndFileName(vgId, NULL, VNODE_RSMA_DIR, TD_QTASKINFO_FNAME_PREFIX, version, outputName);
|
||||
}
|
||||
|
||||
static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) {
|
||||
|
@ -115,12 +113,14 @@ void *tdFreeRSmaInfo(SRSmaInfo *pInfo) {
|
|||
for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
|
||||
SRSmaInfoItem *pItem = &pInfo->items[i];
|
||||
if (pItem->taskInfo) {
|
||||
smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
|
||||
i + 1);
|
||||
taosTmrStopA(&pItem->tmrId);
|
||||
if (pItem->tmrId) {
|
||||
smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId,
|
||||
i + 1);
|
||||
taosTmrStopA(&pItem->tmrId);
|
||||
}
|
||||
tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1);
|
||||
} else {
|
||||
smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
|
||||
smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
|
||||
pInfo->suid, i + 1);
|
||||
}
|
||||
}
|
||||
|
@ -359,13 +359,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
|
|||
goto _err;
|
||||
}
|
||||
|
||||
smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), suid);
|
||||
|
||||
// start the persist timer
|
||||
if (TASK_TRIGGER_STAT_INIT ==
|
||||
atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) {
|
||||
taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat));
|
||||
}
|
||||
smaDebug("vgId:%d, register rsma info succeed for table %" PRIi64, SMA_VID(pSma), suid);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
|
@ -493,7 +487,6 @@ static int32_t tdProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) {
|
|||
}
|
||||
|
||||
static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
|
||||
ASSERT(pMsg != NULL);
|
||||
SSubmitMsgIter msgIter = {0};
|
||||
SSubmitBlk *pBlock = NULL;
|
||||
SSubmitBlkIter blkIter = {0};
|
||||
|
@ -501,19 +494,26 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) {
|
|||
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
|
||||
if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) return -1;
|
||||
if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) {
|
||||
return -1;
|
||||
}
|
||||
while (true) {
|
||||
if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
|
||||
if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!pBlock) break;
|
||||
tdUidStorePut(pStore, msgIter.suid, NULL);
|
||||
}
|
||||
|
||||
if (terrno != TSDB_CODE_SUCCESS) return -1;
|
||||
if (terrno != TSDB_CODE_SUCCESS) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void tdDestroySDataBlockArray(SArray *pArray) {
|
||||
// TODO
|
||||
#if 0
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
|
||||
SSDataBlock *pDataBlock = taosArrayGet(pArray, i);
|
||||
|
@ -598,33 +598,54 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) {
|
|||
|
||||
pSma = RSMA_INFO_SMA(pItem->pRsmaInfo);
|
||||
|
||||
// if rsma trigger stat in cancelled or finished, not start fetch task anymore
|
||||
// if rsma trigger stat in paused, cancelled or finished, not start fetch task
|
||||
int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat));
|
||||
if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) {
|
||||
taosReleaseRef(smaMgmt.smaRef, pItem->refId);
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
return;
|
||||
switch (rsmaTriggerStat) {
|
||||
case TASK_TRIGGER_STAT_PAUSED:
|
||||
case TASK_TRIGGER_STAT_CANCELLED:
|
||||
case TASK_TRIGGER_STAT_FINISHED: {
|
||||
taosReleaseRef(smaMgmt.smaRef, pItem->refId);
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
return;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
int8_t fetchTriggerStat =
|
||||
atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE);
|
||||
if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) {
|
||||
smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
|
||||
pItem->level, pItem->pRsmaInfo->suid);
|
||||
switch (fetchTriggerStat) {
|
||||
case TASK_TRIGGER_STAT_ACTIVE: {
|
||||
smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma),
|
||||
pItem->level, pItem->pRsmaInfo->suid);
|
||||
|
||||
tdRefSmaStat(pSma, (SSmaStat *)pStat);
|
||||
tdRefSmaStat(pSma, (SSmaStat *)pStat);
|
||||
|
||||
SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
|
||||
qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
|
||||
tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
|
||||
SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL};
|
||||
qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false);
|
||||
tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK);
|
||||
|
||||
tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
|
||||
|
||||
} else {
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
tdUnRefSmaStat(pSma, (SSmaStat *)pStat);
|
||||
} break;
|
||||
case TASK_TRIGGER_STAT_PAUSED: {
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is paused",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
} break;
|
||||
case TASK_TRIGGER_STAT_INACTIVE: {
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
} break;
|
||||
case TASK_TRIGGER_STAT_INIT: {
|
||||
smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is init", SMA_VID(pSma),
|
||||
pItem->level, pItem->pRsmaInfo->suid);
|
||||
} break;
|
||||
default: {
|
||||
smaWarn("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is unknown",
|
||||
SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid);
|
||||
} break;
|
||||
}
|
||||
|
||||
_end:
|
||||
taosReleaseRef(smaMgmt.smaRef, pItem->refId);
|
||||
}
|
||||
|
@ -722,7 +743,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
|
||||
static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) {
|
||||
SVnode *pVnode = pSma->pVnode;
|
||||
|
||||
SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t));
|
||||
|
@ -732,7 +753,12 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
int32_t arrSize = taosArrayGetSize(suidList);
|
||||
int64_t arrSize = taosArrayGetSize(suidList);
|
||||
|
||||
if (nTables) {
|
||||
*nTables = arrSize;
|
||||
}
|
||||
|
||||
if (arrSize == 0) {
|
||||
taosArrayDestroy(suidList);
|
||||
smaDebug("vgId:%d, no need to restore rsma env since empty stb id list", TD_VID(pVnode));
|
||||
|
@ -741,9 +767,9 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma) {
|
|||
|
||||
SMetaReader mr = {0};
|
||||
metaReaderInit(&mr, SMA_META(pSma), 0);
|
||||
for (int32_t i = 0; i < arrSize; ++i) {
|
||||
for (int64_t i = 0; i < arrSize; ++i) {
|
||||
tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i);
|
||||
smaDebug("vgId:%d, rsma restore, suid[%d] is %" PRIi64, TD_VID(pVnode), i, suid);
|
||||
smaDebug("vgId:%d, rsma restore, suid is %" PRIi64, TD_VID(pVnode), suid);
|
||||
if (metaGetTableEntryByUid(&mr, suid) < 0) {
|
||||
smaError("vgId:%d, rsma restore, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), suid,
|
||||
terrstr());
|
||||
|
@ -777,17 +803,24 @@ _err:
|
|||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
|
||||
static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed) {
|
||||
SVnode *pVnode = pSma->pVnode;
|
||||
STFile tFile = {0};
|
||||
char qTaskInfoFName[TSDB_FILENAME_LEN];
|
||||
char qTaskInfoFName[TSDB_FILENAME_LEN] = {0};
|
||||
|
||||
tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName);
|
||||
if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) {
|
||||
tdRSmaQTaskInfoGetFName(TD_VID(pVnode), pVnode->state.committed, qTaskInfoFName);
|
||||
if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
if (!taosCheckExistFile(TD_TFILE_FULL_NAME(&tFile))) {
|
||||
if (pVnode->state.committed > 0) {
|
||||
smaWarn("vgId:%d, rsma restore for version %" PRIi64 ", not start as %s not exist", TD_VID(pVnode),
|
||||
pVnode->state.committed, TD_TFILE_FULL_NAME(&tFile));
|
||||
} else {
|
||||
smaDebug("vgId:%d, rsma restore for version %" PRIi64 ", no need as %s not exist", TD_VID(pVnode),
|
||||
pVnode->state.committed, TD_TFILE_FULL_NAME(&tFile));
|
||||
}
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -799,20 +832,28 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) {
|
|||
if (tdRSmaQTaskInfoIterInit(&fIter, &tFile) < 0) {
|
||||
tdRSmaQTaskInfoIterDestroy(&fIter);
|
||||
tdCloseTFile(&tFile);
|
||||
tdDestroyTFile(&tFile);
|
||||
goto _err;
|
||||
}
|
||||
|
||||
if (tdRSmaQTaskInfoRestore(pSma, &fIter) < 0) {
|
||||
tdRSmaQTaskInfoIterDestroy(&fIter);
|
||||
tdCloseTFile(&tFile);
|
||||
tdDestroyTFile(&tFile);
|
||||
goto _err;
|
||||
}
|
||||
|
||||
tdRSmaQTaskInfoIterDestroy(&fIter);
|
||||
tdCloseTFile(&tFile);
|
||||
tdDestroyTFile(&tFile);
|
||||
|
||||
// restored successfully from committed
|
||||
*committed = pVnode->state.committed;
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
smaError("rsma restore, qtaskinfo reload failed since %s", terrstr());
|
||||
smaError("vgId:%d, rsma restore for version %" PRIi64 ", qtaskinfo reload failed since %s", TD_VID(pVnode),
|
||||
pVnode->state.committed, terrstr());
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
|
@ -820,35 +861,45 @@ _err:
|
|||
* @brief reload ts data from checkpoint
|
||||
*
|
||||
* @param pSma
|
||||
* @param committed restore from committed version
|
||||
* @return int32_t
|
||||
*/
|
||||
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) {
|
||||
static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed) {
|
||||
// TODO
|
||||
smaDebug("vgId:%d, rsma restore from %" PRIi64 ", ts data reload success", SMA_VID(pSma), committed);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
smaError("rsma restore, ts data reload failed since %s", terrstr());
|
||||
smaError("vgId:%d, rsma restore from %" PRIi64 ", ts data reload failed since %s", SMA_VID(pSma), committed,
|
||||
terrstr());
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
int32_t tdProcessRSmaRestoreImpl(SSma *pSma) {
|
||||
// step 1: iterate all stables to restore the rsma env
|
||||
if (tdRSmaRestoreQTaskInfoInit(pSma) < 0) {
|
||||
int64_t nTables = 0;
|
||||
if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
if (nTables <= 0) {
|
||||
smaDebug("vgId:%d, no need to restore rsma task since no tables", SMA_VID(pSma));
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
// step 2: retrieve qtaskinfo items from the persistence file(rsma/qtaskinfo) and restore
|
||||
if (tdRSmaRestoreQTaskInfoReload(pSma) < 0) {
|
||||
int64_t committed = -1;
|
||||
if (tdRSmaRestoreQTaskInfoReload(pSma, &committed) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
// step 3: reload ts data from checkpoint
|
||||
if (tdRSmaRestoreTSDataReload(pSma) < 0) {
|
||||
if (tdRSmaRestoreTSDataReload(pSma, committed) < 0) {
|
||||
goto _err;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
smaError("failed to restore rsma task since %s", terrstr());
|
||||
smaError("vgId:%d failed to restore rsma task since %s", SMA_VID(pSma), terrstr());
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
|
@ -931,19 +982,21 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
|
|||
}
|
||||
|
||||
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) {
|
||||
ASSERT(0);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) {
|
||||
ASSERT(0);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
int32_t infoLen = 0;
|
||||
taosDecodeFixedI32(pIter->qBuf, &infoLen);
|
||||
if (infoLen > nBytes) {
|
||||
ASSERT(infoLen > RSMA_QTASKINFO_BUFSIZE);
|
||||
if (infoLen <= RSMA_QTASKINFO_BUFSIZE) {
|
||||
terrno = TSDB_CODE_RSMA_FILE_CORRUPTED;
|
||||
smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
pIter->nAlloc = infoLen;
|
||||
void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen);
|
||||
if (!pBuf) {
|
||||
|
@ -955,12 +1008,10 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF
|
|||
nBytes = infoLen;
|
||||
|
||||
if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) {
|
||||
ASSERT(0);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) {
|
||||
ASSERT(0);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
}
|
||||
|
@ -977,7 +1028,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
|
|||
// block iter
|
||||
bool isFinish = false;
|
||||
if (tdRSmaQTaskInfoIterNextBlock(pIter, &isFinish) < 0) {
|
||||
ASSERT(0);
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
if (isFinish) {
|
||||
|
@ -989,6 +1039,8 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
|
|||
pIter->qBuf = taosDecodeFixedI32(pIter->qBuf, &qTaskInfoLenWithHead);
|
||||
if (qTaskInfoLenWithHead < RSMA_QTASKINFO_HEAD_LEN) {
|
||||
terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
|
||||
smaError("vgId:%d, restore rsma qtaskinfo file %s failed since %s", SMA_VID(pSma),
|
||||
TD_TFILE_FULL_NAME(pIter->pTFile), terrstr());
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
|
@ -1025,22 +1077,20 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void *tdRSmaPersistExec(void *param) {
|
||||
setThreadName("rsma-task-persist");
|
||||
SRSmaStat *pRSmaStat = param;
|
||||
SSma *pSma = pRSmaStat->pSma;
|
||||
STfs *pTfs = pSma->pVnode->pTfs;
|
||||
int32_t vid = SMA_VID(pSma);
|
||||
int64_t toffset = 0;
|
||||
bool isFileCreated = false;
|
||||
int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) {
|
||||
SSma *pSma = pRSmaStat->pSma;
|
||||
SVnode *pVnode = pSma->pVnode;
|
||||
int32_t vid = SMA_VID(pSma);
|
||||
int64_t toffset = 0;
|
||||
bool isFileCreated = false;
|
||||
|
||||
if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) {
|
||||
goto _end;
|
||||
if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL);
|
||||
if (!infoHash) {
|
||||
goto _end;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
STFile tFile = {0};
|
||||
|
@ -1074,9 +1124,17 @@ static void *tdRSmaPersistExec(void *param) {
|
|||
|
||||
if (!isFileCreated) {
|
||||
char qTaskInfoFName[TSDB_FILENAME_LEN];
|
||||
tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName);
|
||||
tdInitTFile(&tFile, pTfs, qTaskInfoFName);
|
||||
tdCreateTFile(&tFile, pTfs, true, -1);
|
||||
tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName);
|
||||
if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) {
|
||||
smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr());
|
||||
goto _err;
|
||||
}
|
||||
if (tdCreateTFile(&tFile, true, -1) < 0) {
|
||||
smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr());
|
||||
goto _err;
|
||||
}
|
||||
smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid,
|
||||
i + 1, TD_TFILE_FULL_NAME(&tFile));
|
||||
|
||||
isFileCreated = true;
|
||||
}
|
||||
|
@ -1101,49 +1159,56 @@ static void *tdRSmaPersistExec(void *param) {
|
|||
|
||||
infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash);
|
||||
}
|
||||
_normal:
|
||||
|
||||
if (isFileCreated) {
|
||||
if (tdUpdateTFileHeader(&tFile) < 0) {
|
||||
smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile),
|
||||
tstrerror(terrno));
|
||||
tdCloseTFile(&tFile);
|
||||
tdRemoveTFile(&tFile);
|
||||
goto _err;
|
||||
} else {
|
||||
smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile));
|
||||
}
|
||||
|
||||
tdCloseTFile(&tFile);
|
||||
|
||||
char newFName[TSDB_FILENAME_LEN];
|
||||
strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN);
|
||||
char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]);
|
||||
strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName));
|
||||
if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) {
|
||||
smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
|
||||
goto _err;
|
||||
} else {
|
||||
smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName);
|
||||
}
|
||||
tdDestroyTFile(&tFile);
|
||||
}
|
||||
goto _end;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
_err:
|
||||
smaError("vgId:%d, rsma persit failed since %s", vid, terrstr());
|
||||
if (isFileCreated) {
|
||||
tdRemoveTFile(&tFile);
|
||||
tdDestroyTFile(&tFile);
|
||||
}
|
||||
return TSDB_CODE_FAILED;
|
||||
}
|
||||
|
||||
static void *tdRSmaPersistExec(void *param) {
|
||||
setThreadName("rsma-task-persist");
|
||||
SRSmaStat *pRSmaStat = param;
|
||||
SSma *pSma = pRSmaStat->pSma;
|
||||
|
||||
int8_t triggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat));
|
||||
|
||||
if (TASK_TRIGGER_STAT_CANCELLED == triggerStat || TASK_TRIGGER_STAT_PAUSED == triggerStat) {
|
||||
goto _end;
|
||||
}
|
||||
|
||||
// execution
|
||||
tdRSmaPersistExecImpl(pRSmaStat);
|
||||
|
||||
_end:
|
||||
if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
|
||||
TASK_TRIGGER_STAT_INACTIVE,
|
||||
TASK_TRIGGER_STAT_ACTIVE)) {
|
||||
smaDebug("vgId:%d, rsma persist task is active again", vid);
|
||||
smaDebug("vgId:%d, rsma persist task is active again", SMA_VID(pSma));
|
||||
} else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat),
|
||||
TASK_TRIGGER_STAT_CANCELLED,
|
||||
TASK_TRIGGER_STAT_FINISHED)) {
|
||||
smaDebug("vgId:%d, rsma persist task is cancelled", vid);
|
||||
smaDebug("vgId:%d, rsma persist task is cancelled", SMA_VID(pSma));
|
||||
} else {
|
||||
smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
|
||||
ASSERT(0);
|
||||
smaWarn("vgId:%d, rsma persist task in stat %" PRIi8, SMA_VID(pSma), atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
|
||||
}
|
||||
|
||||
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
|
||||
taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId);
|
||||
taosThreadExit(NULL);
|
||||
|
@ -1166,8 +1231,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) {
|
|||
TASK_TRIGGER_STAT_FINISHED)) {
|
||||
smaDebug("vgId:%d, persist task is cancelled and set finished", SMA_VID(pRSmaStat->pSma));
|
||||
} else {
|
||||
smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)),
|
||||
SMA_VID(pRSmaStat->pSma));
|
||||
smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, SMA_VID(pRSmaStat->pSma),
|
||||
atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)));
|
||||
ASSERT(0);
|
||||
}
|
||||
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
|
||||
|
@ -1205,8 +1270,8 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
|
|||
// start persist task
|
||||
tdRSmaPersistTask(pRSmaStat);
|
||||
|
||||
taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASKINFO_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle,
|
||||
&pRSmaStat->tmrId);
|
||||
// taosTmrReset(tdRSmaPersistTrigger, 5000, pRSmaStat, pRSmaStat->tmrHandle,
|
||||
// RSMA_TMR_ID(pRSmaStat));
|
||||
} else {
|
||||
atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0);
|
||||
}
|
||||
|
@ -1216,6 +1281,9 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) {
|
|||
atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED);
|
||||
smaDebug("rsma persistence not start since cancelled and finished");
|
||||
} break;
|
||||
case TASK_TRIGGER_STAT_PAUSED: {
|
||||
smaDebug("rsma persistence not start since paused");
|
||||
} break;
|
||||
case TASK_TRIGGER_STAT_INACTIVE: {
|
||||
smaDebug("rsma persistence not start since inactive");
|
||||
} break;
|
||||
|
|
|
@ -0,0 +1,16 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "sma.h"
|
|
@ -140,7 +140,7 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset)
|
|||
return -1;
|
||||
}
|
||||
|
||||
#if 1
|
||||
#if 0
|
||||
smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile),
|
||||
toffset, nbyte, toffset + nbyte);
|
||||
#endif
|
||||
|
@ -179,52 +179,85 @@ void tdCloseTFile(STFile *pTFile) {
|
|||
}
|
||||
}
|
||||
|
||||
void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname);
|
||||
void tdDestroyTFile(STFile *pTFile) { taosMemoryFreeClear(TD_TFILE_FULL_NAME(pTFile)); }
|
||||
|
||||
void tdGetVndFileName(int32_t vgId, const char *pdname, const char *dname, const char *fname, int64_t version,
|
||||
char *outputName) {
|
||||
if (version < 0) {
|
||||
if (pdname) {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId,
|
||||
TD_DIRSEP, dname, TD_DIRSEP, vgId, fname);
|
||||
} else {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP,
|
||||
vgId, fname);
|
||||
}
|
||||
} else {
|
||||
if (pdname) {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%sv%d%s%" PRIi64, pdname, TD_DIRSEP, TD_DIRSEP,
|
||||
vgId, TD_DIRSEP, dname, TD_DIRSEP, vgId, fname, version);
|
||||
} else {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%sv%d%s%" PRIi64, TD_DIRSEP, vgId, TD_DIRSEP, dname,
|
||||
TD_DIRSEP, vgId, fname, version);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) {
|
||||
char fullname[TSDB_FILENAME_LEN];
|
||||
SDiskID did = {0};
|
||||
void tdGetVndDirName(int32_t vgId, const char *pdname, const char *dname, bool endWithSep, char *outputName) {
|
||||
if (pdname) {
|
||||
if (endWithSep) {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP,
|
||||
dname, TD_DIRSEP);
|
||||
} else {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "%s%svnode%svnode%d%s%s", pdname, TD_DIRSEP, TD_DIRSEP, vgId, TD_DIRSEP,
|
||||
dname);
|
||||
}
|
||||
} else {
|
||||
if (endWithSep) {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname, TD_DIRSEP);
|
||||
} else {
|
||||
snprintf(outputName, TSDB_FILENAME_LEN, "vnode%svnode%d%s%s", TD_DIRSEP, vgId, TD_DIRSEP, dname);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int32_t tdInitTFile(STFile *pTFile, const char *dname, const char *fname) {
|
||||
TD_TFILE_SET_STATE(pTFile, TD_FILE_STATE_OK);
|
||||
TD_TFILE_SET_CLOSED(pTFile);
|
||||
|
||||
memset(&(pTFile->info), 0, sizeof(pTFile->info));
|
||||
pTFile->info.magic = TD_FILE_INIT_MAGIC;
|
||||
|
||||
if (tfsAllocDisk(pTfs, 0, &did) < 0) {
|
||||
terrno = TSDB_CODE_NO_AVAIL_DISK;
|
||||
char tmpName[TSDB_FILENAME_LEN * 2 + 32] = {0};
|
||||
snprintf(tmpName, TSDB_FILENAME_LEN * 2 + 32, "%s%s%s", dname, TD_DIRSEP, fname);
|
||||
int32_t tmpNameLen = strlen(tmpName) + 1;
|
||||
pTFile->fname = taosMemoryMalloc(tmpNameLen);
|
||||
if (!pTFile->fname) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
return -1;
|
||||
}
|
||||
|
||||
tfsInitFile(pTfs, &(pTFile->f), did, fname);
|
||||
tstrncpy(pTFile->fname, tmpName, tmpNameLen);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) {
|
||||
int32_t tdCreateTFile(STFile *pTFile, bool updateHeader, int8_t fType) {
|
||||
ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC);
|
||||
|
||||
pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
|
||||
if (pTFile->pFile == NULL) {
|
||||
if (errno == ENOENT) {
|
||||
// Try to create directory recursively
|
||||
char *s = strdup(TD_TFILE_REL_NAME(pTFile));
|
||||
if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_TFILE_DID(pTFile)) < 0) {
|
||||
taosMemoryFreeClear(s);
|
||||
char *s = strdup(TD_TFILE_FULL_NAME(pTFile));
|
||||
if (taosMulMkDir(taosDirName(s)) != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
taosMemoryFree(s);
|
||||
return -1;
|
||||
}
|
||||
taosMemoryFreeClear(s);
|
||||
|
||||
taosMemoryFree(s);
|
||||
pTFile->pFile = taosOpenFile(TD_TFILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC);
|
||||
if (pTFile->pFile == NULL) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
} else {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -244,7 +277,13 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp
|
|||
return 0;
|
||||
}
|
||||
|
||||
int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_TFILE_F(pTFile)); }
|
||||
int32_t tdRemoveTFile(STFile *pTFile) {
|
||||
if (taosRemoveFile(TD_TFILE_FULL_NAME(pTFile)) != 0) {
|
||||
terrno = TAOS_SYSTEM_ERROR(errno);
|
||||
return -1;
|
||||
};
|
||||
return 0;
|
||||
}
|
||||
|
||||
// smaXXXUtil ================
|
||||
// ...
|
|
@ -183,13 +183,15 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) {
|
|||
} else {
|
||||
ASSERT(0);
|
||||
}
|
||||
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);
|
||||
if (pOffset == NULL || pOffset->val.version < offset.val.version) {
|
||||
if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
/*STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey);*/
|
||||
/*if (pOffset != NULL) {*/
|
||||
/*if (pOffset->val.type == TMQ_OFFSET__LOG && pOffset->val.version < offset.val.version) {*/
|
||||
if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) {
|
||||
ASSERT(0);
|
||||
return -1;
|
||||
}
|
||||
/*}*/
|
||||
/*}*/
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -271,12 +273,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
|
||||
if (pOffset != NULL) {
|
||||
fetchOffsetNew = pOffset->val;
|
||||
char formatBuf[50];
|
||||
tFormatOffset(formatBuf, 50, &fetchOffsetNew);
|
||||
char formatBuf[80];
|
||||
tFormatOffset(formatBuf, 80, &fetchOffsetNew);
|
||||
tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf);
|
||||
} else {
|
||||
if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) {
|
||||
if (pReq->useSnapshot) {
|
||||
if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
if (!pHandle->fetchMeta) {
|
||||
tqOffsetResetToData(&fetchOffsetNew, 0, 0);
|
||||
} else {
|
||||
|
@ -302,9 +304,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType);
|
||||
|
||||
if (fetchOffsetNew.type == TMQ_OFFSET__LOG) {
|
||||
int64_t fetchVer = fetchOffsetNew.version + 1;
|
||||
SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
|
||||
if (pHeadWithCkSum == NULL) {
|
||||
int64_t fetchVer = fetchOffsetNew.version + 1;
|
||||
SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048);
|
||||
if (pCkHead == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
@ -318,7 +320,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
break;
|
||||
}
|
||||
|
||||
if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) {
|
||||
if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) {
|
||||
// TODO add push mgr
|
||||
|
||||
tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer);
|
||||
|
@ -329,7 +331,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
goto OVER;
|
||||
}
|
||||
|
||||
SWalReadHead* pHead = &pHeadWithCkSum->head;
|
||||
SWalCont* pHead = &pCkHead->head;
|
||||
|
||||
tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
|
||||
TD_VID(pTq->pVnode), fetchVer, pHead->msgType);
|
||||
|
@ -373,13 +375,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
|||
}
|
||||
}
|
||||
|
||||
taosMemoryFree(pHeadWithCkSum);
|
||||
taosMemoryFree(pCkHead);
|
||||
} else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) {
|
||||
// 1. set uid and ts
|
||||
// 2. get data (rebuild reader if needed)
|
||||
// 3. get new uid and ts
|
||||
|
||||
tqInfo("retrieve using snapshot req offset: uid %ld ts %ld", dataRsp.reqOffset.uid, dataRsp.reqOffset.ts);
|
||||
tqInfo("retrieve using snapshot actual offset: uid %ld ts %ld", fetchOffsetNew.uid, fetchOffsetNew.ts);
|
||||
if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
@ -408,199 +406,14 @@ OVER:
|
|||
return code;
|
||||
}
|
||||
|
||||
#if 0
|
||||
int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
|
||||
SMqPollReq* pReq = pMsg->pCont;
|
||||
int64_t consumerId = pReq->consumerId;
|
||||
int64_t timeout = pReq->timeout;
|
||||
int32_t reqEpoch = pReq->epoch;
|
||||
int64_t fetchOffset;
|
||||
int32_t code = 0;
|
||||
|
||||
// get offset to fetch message
|
||||
if (pReq->currentOffset >= 0) {
|
||||
fetchOffset = pReq->currentOffset + 1;
|
||||
} else {
|
||||
STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey);
|
||||
if (pOffset != NULL) {
|
||||
ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);
|
||||
tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey,
|
||||
TD_VID(pTq->pVnode), pOffset->val.version);
|
||||
fetchOffset = pOffset->val.version + 1;
|
||||
} else {
|
||||
if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) {
|
||||
fetchOffset = walGetFirstVer(pTq->pWal);
|
||||
} else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__LATEST) {
|
||||
fetchOffset = walGetCommittedVer(pTq->pWal);
|
||||
} else if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__NONE) {
|
||||
tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
|
||||
pReq->subKey);
|
||||
terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET;
|
||||
return -1;
|
||||
}
|
||||
tqDebug("consumer %ld, restore offset of %s on vg %d failed, config is %ld, set to %ld", consumerId, pReq->subKey,
|
||||
TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
|
||||
}
|
||||
}
|
||||
|
||||
tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %ld fetch offset %ld", consumerId,
|
||||
pReq->epoch, TD_VID(pTq->pVnode), pReq->currentOffset, fetchOffset);
|
||||
|
||||
STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey));
|
||||
/*ASSERT(pHandle);*/
|
||||
if (pHandle == NULL) {
|
||||
tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode),
|
||||
pReq->subKey);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (pHandle->consumerId != consumerId) {
|
||||
tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld",
|
||||
consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId);
|
||||
return -1;
|
||||
}
|
||||
|
||||
int32_t consumerEpoch = atomic_load_32(&pHandle->epoch);
|
||||
while (consumerEpoch < reqEpoch) {
|
||||
consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch);
|
||||
}
|
||||
|
||||
SMqDataBlkRsp rsp = {0};
|
||||
rsp.reqOffset = pReq->currentOffset;
|
||||
|
||||
rsp.blockData = taosArrayInit(0, sizeof(void*));
|
||||
rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
|
||||
|
||||
if (rsp.blockData == NULL || rsp.blockDataLen == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
rsp.withTbName = pReq->withTbName;
|
||||
if (rsp.withTbName) {
|
||||
rsp.blockTbName = taosArrayInit(0, sizeof(void*));
|
||||
}
|
||||
|
||||
if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) {
|
||||
rsp.withSchema = false;
|
||||
} else {
|
||||
rsp.withSchema = true;
|
||||
rsp.blockSchema = taosArrayInit(0, sizeof(void*));
|
||||
}
|
||||
|
||||
#if 1
|
||||
if (pReq->useSnapshot) {
|
||||
// TODO set ver into snapshot
|
||||
int64_t lastVer = walGetCommittedVer(pTq->pWal);
|
||||
if (rsp.reqOffset < lastVer) {
|
||||
tqInfo("retrieve using snapshot req offset %ld last ver %ld", rsp.reqOffset, lastVer);
|
||||
tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId);
|
||||
|
||||
if (rsp.blockNum != 0) {
|
||||
rsp.withTbName = false;
|
||||
rsp.rspOffset = lastVer;
|
||||
tqInfo("direct send by snapshot req offset %ld rsp offset %ld", rsp.reqOffset, rsp.rspOffset);
|
||||
fetchOffset = lastVer;
|
||||
goto SEND_RSP;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
|
||||
if (pHeadWithCkSum == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
walSetReaderCapacity(pHandle->pWalReader, 2048);
|
||||
|
||||
while (1) {
|
||||
consumerEpoch = atomic_load_32(&pHandle->epoch);
|
||||
if (consumerEpoch > reqEpoch) {
|
||||
tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d",
|
||||
consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, consumerEpoch, reqEpoch);
|
||||
break;
|
||||
}
|
||||
|
||||
if (tqFetchLog(pTq, pHandle, &fetchOffset, &pHeadWithCkSum) < 0) {
|
||||
// TODO add push mgr
|
||||
break;
|
||||
}
|
||||
|
||||
SWalReadHead* pHead = &pHeadWithCkSum->head;
|
||||
|
||||
tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
|
||||
TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
|
||||
|
||||
if (pHead->msgType == TDMT_VND_SUBMIT) {
|
||||
SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
|
||||
|
||||
if (tqDataExec(pTq, &pHandle->execHandle, pCont, &rsp, workerId) < 0) {
|
||||
/*ASSERT(0);*/
|
||||
}
|
||||
} else {
|
||||
ASSERT(pHandle->fetchMeta);
|
||||
ASSERT(IS_META_MSG(pHead->msgType));
|
||||
tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType);
|
||||
SMqMetaRsp metaRsp = {0};
|
||||
metaRsp.reqOffset = pReq->currentOffset;
|
||||
metaRsp.rspOffset = fetchOffset;
|
||||
metaRsp.resMsgType = pHead->msgType;
|
||||
metaRsp.metaRspLen = pHead->bodyLen;
|
||||
metaRsp.metaRsp = pHead->body;
|
||||
if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) {
|
||||
code = -1;
|
||||
goto OVER;
|
||||
}
|
||||
code = 0;
|
||||
goto OVER;
|
||||
}
|
||||
|
||||
// TODO batch optimization:
|
||||
// TODO continue scan until meeting batch requirement
|
||||
if (rsp.blockNum > 0 /* threshold */) {
|
||||
break;
|
||||
} else {
|
||||
fetchOffset++;
|
||||
}
|
||||
}
|
||||
|
||||
taosMemoryFree(pHeadWithCkSum);
|
||||
|
||||
SEND_RSP:
|
||||
ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum);
|
||||
ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum);
|
||||
if (rsp.withSchema) {
|
||||
ASSERT(taosArrayGetSize(rsp.blockSchema) == rsp.blockNum);
|
||||
}
|
||||
|
||||
rsp.rspOffset = fetchOffset;
|
||||
|
||||
if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) {
|
||||
code = -1;
|
||||
}
|
||||
OVER:
|
||||
// TODO wrap in destroy func
|
||||
taosArrayDestroy(rsp.blockDataLen);
|
||||
taosArrayDestroyP(rsp.blockData, (FDelete)taosMemoryFree);
|
||||
|
||||
if (rsp.withSchema) {
|
||||
taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
|
||||
}
|
||||
|
||||
if (rsp.withTbName) {
|
||||
taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) {
|
||||
SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg;
|
||||
|
||||
int32_t code = taosHashRemove(pTq->handles, pReq->subKey, strlen(pReq->subKey));
|
||||
ASSERT(code == 0);
|
||||
|
||||
tqOffsetDelete(pTq->pOffsetStore, pReq->subKey);
|
||||
|
||||
if (tqMetaDeleteHandle(pTq, pReq->subKey) < 0) {
|
||||
ASSERT(0);
|
||||
}
|
||||
|
|
|
@ -97,6 +97,10 @@ int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) {
|
|||
return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset));
|
||||
}
|
||||
|
||||
int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) {
|
||||
return taosHashRemove(pStore->pHash, subscribeKey, strlen(subscribeKey));
|
||||
}
|
||||
|
||||
int32_t tqOffsetSnapshot(STqOffsetStore* pStore) {
|
||||
// open file
|
||||
// TODO file name should be with a version
|
||||
|
|
|
@ -15,13 +15,13 @@
|
|||
|
||||
#include "tq.h"
|
||||
|
||||
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** ppHeadWithCkSum) {
|
||||
int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) {
|
||||
int32_t code = 0;
|
||||
taosThreadMutexLock(&pHandle->pWalReader->mutex);
|
||||
int64_t offset = *fetchOffset;
|
||||
|
||||
while (1) {
|
||||
if (walFetchHead(pHandle->pWalReader, offset, *ppHeadWithCkSum) < 0) {
|
||||
if (walFetchHead(pHandle->pWalReader, offset, *ppCkHead) < 0) {
|
||||
tqDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", pHandle->consumerId,
|
||||
pHandle->epoch, TD_VID(pTq->pVnode), offset);
|
||||
*fetchOffset = offset - 1;
|
||||
|
@ -29,8 +29,8 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead*
|
|||
goto END;
|
||||
}
|
||||
|
||||
if ((*ppHeadWithCkSum)->head.msgType == TDMT_VND_SUBMIT) {
|
||||
code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum);
|
||||
if ((*ppCkHead)->head.msgType == TDMT_VND_SUBMIT) {
|
||||
code = walFetchBody(pHandle->pWalReader, ppCkHead);
|
||||
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
|
@ -43,9 +43,9 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead*
|
|||
goto END;
|
||||
} else {
|
||||
if (pHandle->fetchMeta) {
|
||||
SWalReadHead* pHead = &((*ppHeadWithCkSum)->head);
|
||||
SWalCont* pHead = &((*ppCkHead)->head);
|
||||
if (IS_META_MSG(pHead->msgType)) {
|
||||
code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum);
|
||||
code = walFetchBody(pHandle->pWalReader, ppCkHead);
|
||||
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
|
@ -58,7 +58,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead*
|
|||
goto END;
|
||||
}
|
||||
}
|
||||
code = walSkipFetchBody(pHandle->pWalReader, *ppHeadWithCkSum);
|
||||
code = walSkipFetchBody(pHandle->pWalReader, *ppCkHead);
|
||||
if (code < 0) {
|
||||
ASSERT(0);
|
||||
*fetchOffset = offset;
|
||||
|
@ -120,7 +120,9 @@ bool tqNextDataBlock(SStreamReader* pHandle) {
|
|||
return true;
|
||||
}
|
||||
void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->msgIter.uid, sizeof(int64_t));
|
||||
/*tqDebug("search uid %ld", pHandle->msgIter.uid);*/
|
||||
if (ret != NULL) {
|
||||
/*tqDebug("find uid %ld", pHandle->msgIter.uid);*/
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -185,5 +185,7 @@ void tqTableSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data) {
|
|||
.contLen = ntohl(pReq->length),
|
||||
};
|
||||
|
||||
ASSERT(tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) == 0);
|
||||
if (tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &msg) != 0) {
|
||||
tqDebug("failed to put into write-queue since %s", terrstr());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -28,12 +28,12 @@ const SVnodeCfg vnodeCfgDefault = {
|
|||
.update = 1,
|
||||
.compression = 2,
|
||||
.slLevel = 5,
|
||||
.days = 10,
|
||||
.days = 14400,
|
||||
.minRows = 100,
|
||||
.maxRows = 4096,
|
||||
.keep2 = 3650,
|
||||
.keep0 = 3650,
|
||||
.keep1 = 3650},
|
||||
.keep2 = 5256000,
|
||||
.keep0 = 5256000,
|
||||
.keep1 = 5256000},
|
||||
.walCfg =
|
||||
{.vgId = -1, .fsyncPeriod = 0, .retentionPeriod = 0, .rollPeriod = 0, .segSize = 0, .level = TAOS_WAL_WRITE},
|
||||
.hashBegin = 0,
|
||||
|
|
|
@ -69,6 +69,9 @@ int vnodeBegin(SVnode *pVnode) {
|
|||
}
|
||||
}
|
||||
|
||||
// begin sma
|
||||
smaBegin(pVnode->pSma); // TODO: refactor to include the rsma1/rsma2 tsdbBegin() after tsdb_refact branch merged
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -229,6 +232,9 @@ int vnodeCommit(SVnode *pVnode) {
|
|||
return -1;
|
||||
}
|
||||
|
||||
// preCommit
|
||||
smaPreCommit(pVnode->pSma);
|
||||
|
||||
// commit each sub-system
|
||||
if (metaCommit(pVnode->pMeta) < 0) {
|
||||
ASSERT(0);
|
||||
|
@ -269,6 +275,9 @@ int vnodeCommit(SVnode *pVnode) {
|
|||
|
||||
pVnode->state.committed = info.state.committed;
|
||||
|
||||
// postCommit
|
||||
smaPostCommit(pVnode->pSma);
|
||||
|
||||
// apply the commit (TODO)
|
||||
vnodeBufPoolReset(pVnode->onCommit);
|
||||
pVnode->onCommit->next = pVnode->pPool;
|
||||
|
|
|
@ -152,12 +152,11 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
|
|||
return pVnode;
|
||||
|
||||
_err:
|
||||
if (pVnode->pSma) smaCloseEnv(pVnode->pSma);
|
||||
if (pVnode->pQuery) vnodeQueryClose(pVnode);
|
||||
if (pVnode->pTq) tqClose(pVnode->pTq);
|
||||
if (pVnode->pWal) walClose(pVnode->pWal);
|
||||
if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
|
||||
if (pVnode->pSma) smaCloseEx(pVnode->pSma);
|
||||
if (pVnode->pSma) smaClose(pVnode->pSma);
|
||||
if (pVnode->pMeta) metaClose(pVnode->pMeta);
|
||||
|
||||
tsem_destroy(&(pVnode->canCommit));
|
||||
|
@ -167,14 +166,13 @@ _err:
|
|||
|
||||
void vnodeClose(SVnode *pVnode) {
|
||||
if (pVnode) {
|
||||
smaCloseEnv(pVnode->pSma);
|
||||
vnodeCommit(pVnode);
|
||||
vnodeSyncClose(pVnode);
|
||||
vnodeQueryClose(pVnode);
|
||||
walClose(pVnode->pWal);
|
||||
tqClose(pVnode->pTq);
|
||||
if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb);
|
||||
smaCloseEx(pVnode->pSma);
|
||||
smaClose(pVnode->pSma);
|
||||
metaClose(pVnode->pMeta);
|
||||
vnodeCloseBufPool(pVnode);
|
||||
// destroy handle
|
||||
|
|
|
@ -779,6 +779,7 @@ _exit:
|
|||
taosArrayDestroy(submitRsp.pArray);
|
||||
|
||||
// TODO: the partial success scenario and the error case
|
||||
// => If partial success, extract the success submitted rows and reconstruct a new submit msg, and push to level 1/level 2.
|
||||
// TODO: refactor
|
||||
if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) {
|
||||
tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT);
|
||||
|
|
|
@ -278,7 +278,7 @@ typedef struct SCtgAsyncFps {
|
|||
|
||||
typedef struct SCtgApiStat {
|
||||
|
||||
#ifdef WINDOWS
|
||||
#if defined(WINDOWS) || defined(_TD_DARWIN_64)
|
||||
size_t avoidCompilationErrors;
|
||||
#endif
|
||||
|
||||
|
|
|
@ -1,23 +1,25 @@
|
|||
|
||||
MESSAGE(STATUS "build catalog unit test")
|
||||
|
||||
# GoogleTest requires at least C++11
|
||||
SET(CMAKE_CXX_STANDARD 11)
|
||||
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
|
||||
IF(NOT TD_DARWIN)
|
||||
# GoogleTest requires at least C++11
|
||||
SET(CMAKE_CXX_STANDARD 11)
|
||||
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
|
||||
|
||||
ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
|
||||
TARGET_LINK_LIBRARIES(
|
||||
catalogTest
|
||||
PUBLIC os util common catalog transport gtest qcom taos_static
|
||||
)
|
||||
ADD_EXECUTABLE(catalogTest ${SOURCE_LIST})
|
||||
TARGET_LINK_LIBRARIES(
|
||||
catalogTest
|
||||
PUBLIC os util common catalog transport gtest qcom taos_static
|
||||
)
|
||||
|
||||
TARGET_INCLUDE_DIRECTORIES(
|
||||
catalogTest
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
|
||||
PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
|
||||
)
|
||||
TARGET_INCLUDE_DIRECTORIES(
|
||||
catalogTest
|
||||
PUBLIC "${TD_SOURCE_DIR}/include/libs/catalog/"
|
||||
PRIVATE "${TD_SOURCE_DIR}/source/libs/catalog/inc"
|
||||
)
|
||||
|
||||
# add_test(
|
||||
# NAME catalogTest
|
||||
# COMMAND catalogTest
|
||||
# )
|
||||
# add_test(
|
||||
# NAME catalogTest
|
||||
# COMMAND catalogTest
|
||||
# )
|
||||
ENDIF()
|
||||
|
|
|
@ -297,7 +297,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) {
|
|||
SArray* pTagVals = NULL;
|
||||
STag* pTag = (STag*)pCfg->pTags;
|
||||
|
||||
if (pCfg->pTags && pTag->flags & TD_TAG_JSON) {
|
||||
if (pCfg->pTags && tTagIsJson(pTag)) {
|
||||
char* pJson = parseTagDatatoJson(pTag);
|
||||
if (pJson) {
|
||||
*len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson);
|
||||
|
|
|
@ -287,6 +287,7 @@ typedef struct STableScanInfo {
|
|||
} lastStatus;
|
||||
|
||||
int8_t scanMode;
|
||||
int8_t noTable;
|
||||
} STableScanInfo;
|
||||
|
||||
typedef struct STagScanInfo {
|
||||
|
@ -355,6 +356,7 @@ typedef struct SStreamBlockScanInfo {
|
|||
SUpdateInfo* pUpdateInfo;
|
||||
|
||||
EStreamScanMode scanMode;
|
||||
SOperatorInfo* pStreamScanOp;
|
||||
SOperatorInfo* pSnapshotReadOp;
|
||||
SArray* childIds;
|
||||
SessionWindowSupporter sessionSup;
|
||||
|
@ -426,7 +428,7 @@ typedef struct SIntervalAggOperatorInfo {
|
|||
STimeWindowAggSupp twAggSup;
|
||||
bool invertible;
|
||||
SArray* pPrevValues; // SArray<SGroupKeys> used to keep the previous not null value for interpolation.
|
||||
bool ignoreCloseWindow;
|
||||
bool ignoreExpiredData;
|
||||
} SIntervalAggOperatorInfo;
|
||||
|
||||
typedef struct SStreamFinalIntervalOperatorInfo {
|
||||
|
@ -448,7 +450,7 @@ typedef struct SStreamFinalIntervalOperatorInfo {
|
|||
SArray* pPullWins; // SPullWindowInfo
|
||||
int32_t pullIndex;
|
||||
SSDataBlock* pPullDataRes;
|
||||
bool ignoreCloseWindow;
|
||||
bool ignoreExpiredData;
|
||||
} SStreamFinalIntervalOperatorInfo;
|
||||
|
||||
typedef struct SAggOperatorInfo {
|
||||
|
@ -586,7 +588,7 @@ typedef struct SStreamSessionAggOperatorInfo {
|
|||
SArray* pChildren; // cache for children's result; final stream operator
|
||||
SPhysiNode* pPhyNode; // create new child
|
||||
bool isFinal;
|
||||
bool ignoreCloseWindow;
|
||||
bool ignoreExpiredData;
|
||||
} SStreamSessionAggOperatorInfo;
|
||||
|
||||
typedef struct STimeSliceOperatorInfo {
|
||||
|
@ -630,7 +632,7 @@ typedef struct SStreamStateAggOperatorInfo {
|
|||
void* pDelIterator;
|
||||
SArray* pScanWindow;
|
||||
SArray* pChildren; // cache for children's result;
|
||||
bool ignoreCloseWindow;
|
||||
bool ignoreExpiredData;
|
||||
} SStreamStateAggOperatorInfo;
|
||||
|
||||
typedef struct SSortedMergeOperatorInfo {
|
||||
|
|
|
@ -35,7 +35,7 @@ int32_t dsDataSinkGetCacheSize(SDataSinkStat *pStat) {
|
|||
|
||||
|
||||
int32_t dsCreateDataSinker(const SDataSinkNode *pDataSink, DataSinkHandle* pHandle, void* pParam) {
|
||||
switch (nodeType(pDataSink)) {
|
||||
switch ((int)nodeType(pDataSink)) {
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DISPATCH:
|
||||
return createDataDispatcher(&gDataSinkManager, pDataSink, pHandle);
|
||||
case QUERY_NODE_PHYSICAL_PLAN_DELETE:
|
||||
|
|
|
@ -63,6 +63,7 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
|
|||
rowSize += pCtx[i].resDataInfo.interBufSize;
|
||||
}
|
||||
|
||||
rowSize += (numOfOutput * sizeof(bool)); // expand rowSize to mark if col is null for top/bottom result(saveTupleData)
|
||||
return rowSize;
|
||||
}
|
||||
|
||||
|
@ -112,13 +113,15 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int
|
|||
p->groupId = *(uint64_t*)key;
|
||||
p->pos = *(SResultRowPosition*)pData;
|
||||
memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_groupRes, groupId:%"PRIu64",pageId:%d,offset:%d\n", p->groupId, p->pos.pageId, p->pos.offset);
|
||||
#endif
|
||||
taosArrayPush(pGroupResInfo->pRows, &p);
|
||||
}
|
||||
|
||||
if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) {
|
||||
__compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc;
|
||||
qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn);
|
||||
taosSort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn);
|
||||
}
|
||||
|
||||
pGroupResInfo->index = 0;
|
||||
|
@ -271,6 +274,7 @@ static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) {
|
|||
SNode* pNew = NULL;
|
||||
int32_t code = scalarCalculateConstants(pTagCondTmp, &pNew);
|
||||
if (TSDB_CODE_SUCCESS != code) {
|
||||
terrno = code;
|
||||
nodesDestroyNode(pTagCondTmp);
|
||||
return false;
|
||||
}
|
||||
|
@ -323,12 +327,19 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo
|
|||
code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList);
|
||||
}
|
||||
|
||||
if (pTagCond) {
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid);
|
||||
terrno = code;
|
||||
return code;
|
||||
}
|
||||
|
||||
if(pTagCond){
|
||||
int32_t i = 0;
|
||||
while (i < taosArrayGetSize(pListInfo->pTableList)) {
|
||||
STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i);
|
||||
bool isOk = isTableOk(info, pTagCond, metaHandle);
|
||||
if (!isOk) {
|
||||
bool isOk = isTableOk(info, pTagCond, metaHandle);
|
||||
if(terrno) return terrno;
|
||||
if(!isOk){
|
||||
taosArrayRemove(pListInfo->pTableList, i);
|
||||
continue;
|
||||
}
|
||||
|
@ -586,13 +597,16 @@ static int32_t setSelectValueColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutpu
|
|||
}
|
||||
|
||||
for (int32_t i = 0; i < numOfOutput; ++i) {
|
||||
if (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
|
||||
if (strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_select_value") == 0 ||
|
||||
strcmp(pCtx[i].pExpr->pExpr->_function.functionName, "_group_key") == 0) {
|
||||
pValCtx[num++] = &pCtx[i];
|
||||
} else if (fmIsSelectFunc(pCtx[i].functionId)) {
|
||||
p = &pCtx[i];
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_setSelect num:%d", num);
|
||||
#endif
|
||||
if (p != NULL) {
|
||||
p->subsidiaries.pCtx = pValCtx;
|
||||
p->subsidiaries.num = num;
|
||||
|
|
|
@ -145,10 +145,12 @@ static SArray* filterQualifiedChildTables(const SStreamBlockScanInfo* pScanInfo,
|
|||
continue;
|
||||
}
|
||||
|
||||
// TODO handle ntb case
|
||||
if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) {
|
||||
continue;
|
||||
}
|
||||
// TODO handle ntb case
|
||||
/*pScanInfo->pStreamScanOp->pTaskInfo->tableqinfoList.*/
|
||||
// handle multiple partition
|
||||
|
||||
taosArrayPush(qa, id);
|
||||
}
|
||||
|
|
|
@ -235,9 +235,11 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) {
|
|||
SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
|
||||
|
||||
if (uid == 0) {
|
||||
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
|
||||
uid = pTableInfo->uid;
|
||||
ts = INT64_MIN;
|
||||
if (taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList) != 0) {
|
||||
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0);
|
||||
uid = pTableInfo->uid;
|
||||
ts = INT64_MIN;
|
||||
}
|
||||
}
|
||||
|
||||
return doPrepareScan(pTaskInfo->pRoot, uid, ts);
|
||||
|
|
|
@ -274,6 +274,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
|
|||
|
||||
// 1. close current opened time window
|
||||
if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId))) {
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_1");
|
||||
#endif
|
||||
SResultRowPosition pos = pResultRowInfo->cur;
|
||||
SFilePage* pPage = getBufPage(pResultBuf, pos.pageId);
|
||||
releaseBufPage(pResultBuf, pPage);
|
||||
|
@ -281,6 +284,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR
|
|||
|
||||
// allocate a new buffer page
|
||||
if (pResult == NULL) {
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_2");
|
||||
#endif
|
||||
ASSERT(pSup->resultRowSize > 0);
|
||||
pResult = getNewResultRow(pResultBuf, groupId, pSup->resultRowSize);
|
||||
|
||||
|
@ -538,7 +544,9 @@ static int32_t doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunct
|
|||
if (pCtx[k].fpSet.process == NULL) {
|
||||
continue;
|
||||
}
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_process");
|
||||
#endif
|
||||
int32_t code = pCtx[k].fpSet.process(&pCtx[k]);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code));
|
||||
|
@ -1413,7 +1421,9 @@ void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t
|
|||
if (pAggInfo->groupId != INT32_MIN && pAggInfo->groupId == groupId) {
|
||||
return;
|
||||
}
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_setbuf, groupId:%" PRIu64, groupId);
|
||||
#endif
|
||||
doSetTableGroupOutputBuf(pOperator, pAggInfo, numOfOutput, groupId);
|
||||
|
||||
// record the current active group id
|
||||
|
@ -1489,11 +1499,15 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
|
|||
int32_t numOfExprs) {
|
||||
int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
|
||||
int32_t start = pGroupResInfo->index;
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("\npage_copytoblock rows:%d", numOfRows);
|
||||
#endif
|
||||
for (int32_t i = start; i < numOfRows; i += 1) {
|
||||
SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i);
|
||||
SFilePage* page = getBufPage(pBuf, pPos->pos.pageId);
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_copytoblock pos pageId:%d, offset:%d", pPos->pos.pageId, pPos->pos.offset);
|
||||
#endif
|
||||
SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset);
|
||||
|
||||
doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset);
|
||||
|
@ -1525,6 +1539,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
|
|||
|
||||
pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowCellOffset);
|
||||
if (pCtx[j].fpSet.finalize) {
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("\npage_finalize %d", numOfExprs);
|
||||
#endif
|
||||
int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
|
||||
if (TAOS_FAILED(code)) {
|
||||
qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code));
|
||||
|
@ -1553,9 +1570,9 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI
|
|||
|
||||
releaseBufPage(pBuf, page);
|
||||
pBlock->info.rows += pRow->numOfRows;
|
||||
if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full
|
||||
break;
|
||||
}
|
||||
// if (pBlock->info.rows >= pBlock->info.capacity) { // output buffer is full
|
||||
// break;
|
||||
// }
|
||||
}
|
||||
|
||||
qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
|
||||
|
@ -2010,8 +2027,9 @@ static int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInf
|
|||
|
||||
ASSERT(pDataInfo->status == EX_SOURCE_DATA_NOT_READY);
|
||||
|
||||
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo),
|
||||
pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources);
|
||||
qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId,
|
||||
sourceIndex, totalSources);
|
||||
|
||||
pMsg->header.vgId = htonl(pSource->addr.nodeId);
|
||||
pMsg->sId = htobe64(pSource->schedId);
|
||||
|
@ -2146,8 +2164,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
|
|||
SSDataBlock* pRes = pExchangeInfo->pResult;
|
||||
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
|
||||
if (pRsp->numOfRows == 0) {
|
||||
qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
|
||||
", completed:%d try next %d/%" PRIzu,
|
||||
qDebug("%s vgId:%d, taskId:0x%" PRIx64 " execId:%d index:%d completed, rowsOfSource:%" PRIu64
|
||||
", totalRows:%" PRIu64 ", completed:%d try next %d/%" PRIzu,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pDataInfo->totalRows,
|
||||
pExchangeInfo->loadInfo.totalRows, completed + 1, i + 1, totalSources);
|
||||
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
|
||||
|
@ -2166,18 +2184,19 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx
|
|||
}
|
||||
|
||||
if (pRsp->completed == 1) {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d"
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64
|
||||
" execId:%d"
|
||||
" index:%d completed, numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64
|
||||
", completed:%d try next %d/%" PRIzu,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows, pDataInfo->totalRows,
|
||||
pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, i, pRes->info.rows,
|
||||
pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, completed + 1, i + 1, totalSources);
|
||||
completed += 1;
|
||||
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
|
||||
} else {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
|
||||
", totalBytes:%" PRIu64,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
|
||||
pLoadInfo->totalSize);
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
|
||||
pLoadInfo->totalRows, pLoadInfo->totalSize);
|
||||
}
|
||||
|
||||
taosMemoryFreeClear(pDataInfo->pRsp);
|
||||
|
@ -2250,8 +2269,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
SDownstreamSourceNode* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current);
|
||||
|
||||
if (pDataInfo->code != TSDB_CODE_SUCCESS) {
|
||||
qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo), pSource->addr.nodeId,
|
||||
pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
|
||||
qError("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d error happens, code:%s", GET_TASKID(pTaskInfo),
|
||||
pSource->addr.nodeId, pSource->taskId, pSource->execId, tstrerror(pDataInfo->code));
|
||||
pOperator->pTaskInfo->code = pDataInfo->code;
|
||||
return NULL;
|
||||
}
|
||||
|
@ -2259,8 +2278,8 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
SRetrieveTableRsp* pRsp = pDataInfo->pRsp;
|
||||
SLoadRemoteDataInfo* pLoadInfo = &pExchangeInfo->loadInfo;
|
||||
if (pRsp->numOfRows == 0) {
|
||||
qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64
|
||||
" try next",
|
||||
qDebug("%s vgId:%d, taskID:0x%" PRIx64 " execId:%d %d of total completed, rowsOfSource:%" PRIu64
|
||||
", totalRows:%" PRIu64 " try next",
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pExchangeInfo->current + 1,
|
||||
pDataInfo->totalRows, pLoadInfo->totalRows);
|
||||
|
||||
|
@ -2279,16 +2298,17 @@ static SSDataBlock* seqLoadRemoteData(SOperatorInfo* pOperator) {
|
|||
if (pRsp->completed == 1) {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64
|
||||
", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pDataInfo->totalRows,
|
||||
pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1, totalSources);
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
|
||||
pDataInfo->totalRows, pLoadInfo->totalRows, pLoadInfo->totalSize, pExchangeInfo->current + 1,
|
||||
totalSources);
|
||||
|
||||
pDataInfo->status = EX_SOURCE_DATA_EXHAUSTED;
|
||||
pExchangeInfo->current += 1;
|
||||
} else {
|
||||
qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, totalRows:%" PRIu64
|
||||
", totalBytes:%" PRIu64,
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows, pLoadInfo->totalRows,
|
||||
pLoadInfo->totalSize);
|
||||
GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pSource->execId, pRes->info.rows,
|
||||
pLoadInfo->totalRows, pLoadInfo->totalSize);
|
||||
}
|
||||
|
||||
pOperator->resultInfo.totalRows += pRes->info.rows;
|
||||
|
@ -2373,8 +2393,7 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo*
|
|||
}
|
||||
|
||||
pInfo->pSources = taosArrayInit(numOfSources, sizeof(SDownstreamSourceNode));
|
||||
pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo));
|
||||
if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) {
|
||||
if (pInfo->pSources == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
|
||||
|
@ -2829,22 +2848,46 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
|
|||
pScanInfo->blockType = STREAM_INPUT__DATA_SCAN;
|
||||
|
||||
STableScanInfo* pInfo = pScanInfo->pSnapshotReadOp->info;
|
||||
if (uid == 0) {
|
||||
pInfo->noTable = 1;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
/*if (pSnapShotScanInfo->dataReader == NULL) {*/
|
||||
/*pSnapShotScanInfo->dataReader = tsdbReaderOpen(pHandle->vnode, &pSTInfo->cond, tableList, 0, 0);*/
|
||||
/*pSnapShotScanInfo->scanMode = TABLE_SCAN__TABLE_ORDER;*/
|
||||
/*}*/
|
||||
|
||||
pInfo->noTable = 0;
|
||||
|
||||
if (pInfo->lastStatus.uid != uid || pInfo->lastStatus.ts != ts) {
|
||||
tsdbSetTableId(pInfo->dataReader, uid);
|
||||
int64_t oldSkey = pInfo->cond.twindows[0].skey;
|
||||
pInfo->cond.twindows[0].skey = ts;
|
||||
pInfo->cond.twindows[0].skey = ts + 1;
|
||||
tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0);
|
||||
pInfo->cond.twindows[0].skey = oldSkey;
|
||||
pInfo->scanTimes = 0;
|
||||
pInfo->curTWinIdx = 0;
|
||||
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
int32_t tableSz = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList);
|
||||
bool found = false;
|
||||
for (int32_t i = 0; i < tableSz; i++) {
|
||||
STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, i);
|
||||
if (pTableInfo->uid == uid) {
|
||||
found = true;
|
||||
pInfo->currentTable = i;
|
||||
}
|
||||
}
|
||||
// TODO after processing drop,
|
||||
ASSERT(found);
|
||||
qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts,
|
||||
pInfo->currentTable, tableSz);
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
||||
} else {
|
||||
if (pOperator->numOfDownstream == 1) {
|
||||
return doPrepareScan(pOperator->pDownstream[0], uid, ts);
|
||||
|
@ -2856,8 +2899,6 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) {
|
|||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts) {
|
||||
|
@ -3160,8 +3201,9 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock)
|
|||
}
|
||||
|
||||
// check for the limitation in each group
|
||||
if (pProjectInfo->limit.limit > 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) {
|
||||
pRes->info.rows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput);
|
||||
if (pProjectInfo->limit.limit >= 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) {
|
||||
int32_t keepRows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput);
|
||||
blockDataKeepFirstNRows(pRes, keepRows);
|
||||
if (pProjectInfo->slimit.limit > 0 && pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) {
|
||||
pOperator->status = OP_EXEC_DONE;
|
||||
}
|
||||
|
@ -3228,6 +3270,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
|
|||
doSetOperatorCompleted(pOperator);
|
||||
break;
|
||||
}
|
||||
if (pBlock->info.type == STREAM_RETRIEVE) {
|
||||
// for stream interval
|
||||
return pBlock;
|
||||
}
|
||||
|
||||
// the pDataBlock are always the same one, no need to call this again
|
||||
int32_t code = getTableScanInfo(pOperator->pDownstream[0], &order, &scanFlag);
|
||||
|
@ -3235,8 +3281,6 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
|
|||
longjmp(pTaskInfo->env, code);
|
||||
}
|
||||
|
||||
doFilter(pProjectInfo->pFilterNode, pBlock);
|
||||
|
||||
setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, scanFlag, false);
|
||||
blockDataEnsureCapacity(pInfo->pRes, pInfo->pRes->info.rows + pBlock->info.rows);
|
||||
|
||||
|
@ -3247,6 +3291,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
|
|||
}
|
||||
|
||||
int32_t status = handleLimitOffset(pOperator, pBlock);
|
||||
|
||||
// filter shall be applied after apply functions and limit/offset on the result
|
||||
doFilter(pProjectInfo->pFilterNode, pInfo->pRes);
|
||||
|
||||
if (status == PROJECT_RETRIEVE_CONTINUE) {
|
||||
continue;
|
||||
} else if (status == PROJECT_RETRIEVE_DONE) {
|
||||
|
@ -3473,11 +3521,12 @@ int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInf
|
|||
}
|
||||
|
||||
void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows) {
|
||||
ASSERT(numOfRows != 0);
|
||||
pOperator->resultInfo.capacity = numOfRows;
|
||||
pOperator->resultInfo.threshold = numOfRows * 0.75;
|
||||
|
||||
if (pOperator->resultInfo.threshold == 0) {
|
||||
pOperator->resultInfo.capacity = numOfRows;
|
||||
pOperator->resultInfo.threshold = numOfRows;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3977,12 +4026,12 @@ static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum)
|
|||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
if (p == NULL) {
|
||||
if (taosArrayPush(sortSupport, groupId) != NULL) {
|
||||
if (taosArrayPush(sortSupport, groupId) == NULL) {
|
||||
qError("taos push support array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
}
|
||||
if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL) {
|
||||
if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) == NULL) {
|
||||
qError("taos push group array error");
|
||||
taosArrayDestroy(sortSupport);
|
||||
return TSDB_CODE_QRY_APP_ERROR;
|
||||
|
@ -4062,19 +4111,27 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle,
|
|||
} else {
|
||||
taosMemoryFree(keyBuf);
|
||||
nodesClearList(groupNew);
|
||||
metaReaderClear(&mr);
|
||||
return code;
|
||||
}
|
||||
|
||||
ASSERT(nodeType(pNew) == QUERY_NODE_VALUE);
|
||||
SValueNode* pValue = (SValueNode*)pNew;
|
||||
|
||||
if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) {
|
||||
if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL || pValue->isNull) {
|
||||
isNull[index++] = 1;
|
||||
continue;
|
||||
} else {
|
||||
isNull[index++] = 0;
|
||||
char* data = nodesGetValueFromNode(pValue);
|
||||
if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) {
|
||||
if (tTagIsJson(data)) {
|
||||
terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
|
||||
taosMemoryFree(keyBuf);
|
||||
nodesClearList(groupNew);
|
||||
metaReaderClear(&mr);
|
||||
return terrno;
|
||||
}
|
||||
int32_t len = getJsonValueLen(data);
|
||||
memcpy(pStart, data, len);
|
||||
pStart += len;
|
||||
|
@ -4134,6 +4191,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
|
|||
STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode;
|
||||
int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
|
||||
if (code) {
|
||||
pTaskInfo->code = code;
|
||||
return NULL;
|
||||
}
|
||||
code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo);
|
||||
|
@ -4160,7 +4218,11 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
|
|||
.maxTs = INT64_MIN,
|
||||
};
|
||||
if (pHandle) {
|
||||
createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
|
||||
int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId);
|
||||
if (code) {
|
||||
pTaskInfo->code = code;
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
SOperatorInfo* pOperator =
|
||||
|
|
|
@ -141,6 +141,10 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData
|
|||
pkey->isNull = false;
|
||||
char* val = colDataGetData(pColInfoData, rowIndex);
|
||||
if (pkey->type == TSDB_DATA_TYPE_JSON) {
|
||||
if(tTagIsJson(val)){
|
||||
terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR;
|
||||
return;
|
||||
}
|
||||
int32_t dataLen = getJsonValueLen(val);
|
||||
memcpy(pkey->pData, val, dataLen);
|
||||
} else if (IS_VAR_DATA_TYPE(pkey->type)) {
|
||||
|
@ -227,11 +231,15 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
|
|||
int32_t len = 0;
|
||||
STimeWindow w = TSWINDOW_INITIALIZER;
|
||||
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
int32_t num = 0;
|
||||
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
|
||||
// Compare with the previous row of this column, and do not set the output buffer again if they are identical.
|
||||
if (!pInfo->isInit) {
|
||||
recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j);
|
||||
if (terrno != TSDB_CODE_SUCCESS) { // group by json error
|
||||
longjmp(pTaskInfo->env, terrno);
|
||||
}
|
||||
pInfo->isInit = true;
|
||||
num++;
|
||||
continue;
|
||||
|
@ -247,6 +255,9 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
|
|||
if (j == 0) {
|
||||
num++;
|
||||
recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j);
|
||||
if (terrno != TSDB_CODE_SUCCESS) { // group by json error
|
||||
longjmp(pTaskInfo->env, terrno);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -661,7 +672,11 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
|
|||
}
|
||||
}
|
||||
|
||||
terrno = TSDB_CODE_SUCCESS;
|
||||
doHashPartition(pOperator, pBlock);
|
||||
if (terrno != TSDB_CODE_SUCCESS) { // group by json error
|
||||
longjmp(pTaskInfo->env, terrno);
|
||||
}
|
||||
}
|
||||
|
||||
SArray* groupArray = taosArrayInit(taosHashGetSize(pInfo->pGroupSet), sizeof(SDataGroupInfo));
|
||||
|
|
|
@ -518,6 +518,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
|
|||
|
||||
// if scan table by table
|
||||
if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
|
||||
if (pInfo->noTable) return NULL;
|
||||
while (1) {
|
||||
SSDataBlock* result = doTableScanGroup(pOperator);
|
||||
if (result) {
|
||||
|
@ -806,6 +807,23 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) {
|
|||
return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
|
||||
}
|
||||
|
||||
static void setGroupId(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
|
||||
ASSERT(rowIndex < pBlock->info.rows);
|
||||
switch (pBlock->info.type)
|
||||
{
|
||||
case STREAM_RETRIEVE: {
|
||||
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
|
||||
uint64_t* groupCol = (uint64_t*)pColInfo->pData;
|
||||
pInfo->groupId = groupCol[rowIndex];
|
||||
}
|
||||
break;
|
||||
case STREAM_DELETE_DATA:
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) {
|
||||
STimeWindow win = {
|
||||
.skey = INT64_MIN,
|
||||
|
@ -828,6 +846,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3
|
|||
} else {
|
||||
win =
|
||||
getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL);
|
||||
setGroupId(pInfo, pSDB, 2, *pRowIndex);
|
||||
(*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL,
|
||||
TSDB_ORDER_ASC);
|
||||
}
|
||||
|
@ -1030,10 +1049,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) {
|
|||
}
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
} else {
|
||||
if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) {
|
||||
if (isStateWindow(pInfo)) {
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER;
|
||||
pInfo->updateResIndex = pInfo->pUpdateRes->info.rows;
|
||||
prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
|
||||
if (!prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex)) {
|
||||
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
|
||||
}
|
||||
}
|
||||
if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) {
|
||||
SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex);
|
||||
|
@ -1273,6 +1294,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
|
|||
pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1};
|
||||
pInfo->groupId = 0;
|
||||
pInfo->pPullDataRes = createPullDataBlock();
|
||||
pInfo->pStreamScanOp = pOperator;
|
||||
|
||||
pOperator->name = "StreamBlockScanOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN;
|
||||
|
@ -1307,6 +1329,13 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) {
|
|||
taosArrayDestroy(pInfo->scanCols);
|
||||
}
|
||||
|
||||
static int32_t getSysTableDbNameColId(const char* pTable) {
|
||||
// if (0 == strcmp(TSDB_INS_TABLE_USER_INDEXES, pTable)) {
|
||||
// return 1;
|
||||
// }
|
||||
return TSDB_INS_USER_STABLES_DBNAME_COLID;
|
||||
}
|
||||
|
||||
EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
ENodeType nType = nodeType(pNode);
|
||||
|
@ -1328,7 +1357,7 @@ EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) {
|
|||
}
|
||||
|
||||
SColumnNode* node = (SColumnNode*)pNode;
|
||||
if (TSDB_INS_USER_STABLES_DBNAME_COLID == node->colId) {
|
||||
if (getSysTableDbNameColId(node->tableName) == node->colId) {
|
||||
*(int32_t*)pContext = 2;
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
@ -2089,6 +2118,7 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle
|
|||
qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
pTableListInfo->needSortTableByGroupId = pTableScanNode->groupSort;
|
||||
code = generateGroupIdMap(pTableListInfo, pHandle, pTableScanNode->pGroupTags);
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
return code;
|
||||
|
|
|
@ -838,7 +838,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval,
|
||||
pInfo->interval.precision, &pInfo->win);
|
||||
int32_t ret = TSDB_CODE_SUCCESS;
|
||||
if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
|
||||
if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
|
||||
ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
|
||||
numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
|
||||
if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
|
||||
|
@ -871,7 +871,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
|
||||
}
|
||||
|
||||
if (!pInfo->ignoreCloseWindow || !isCloseWindow(&win, &pInfo->twAggSup)) {
|
||||
if (!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) {
|
||||
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
|
||||
doApplyFunctions(pTaskInfo, pSup->pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols,
|
||||
pBlock->info.rows, numOfOutput, pInfo->order);
|
||||
|
@ -886,7 +886,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
|
|||
if (startPos < 0) {
|
||||
break;
|
||||
}
|
||||
if (pInfo->ignoreCloseWindow && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
|
||||
if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
|
||||
ekey = ascScan ? nextWin.ekey : nextWin.skey;
|
||||
forwardRows =
|
||||
getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order);
|
||||
|
@ -1535,7 +1535,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
|
|||
pInfo->interval = *pInterval;
|
||||
pInfo->execModel = pTaskInfo->execModel;
|
||||
pInfo->twAggSup = *pTwAggSupp;
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
pInfo->ignoreExpiredData = pPhyNode->window.igExpired;
|
||||
|
||||
if (pPhyNode->window.pExprs != NULL) {
|
||||
int32_t numOfScalar = 0;
|
||||
|
@ -2292,7 +2292,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc
|
|||
pInfo->interval.precision, NULL);
|
||||
while (1) {
|
||||
bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup);
|
||||
if (pInfo->ignoreCloseWindow && isClosed) {
|
||||
if (pInfo->ignoreExpiredData && isClosed) {
|
||||
startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin);
|
||||
if (startPos < 0) {
|
||||
break;
|
||||
|
@ -2710,7 +2710,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream,
|
|||
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
|
||||
pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK);
|
||||
pInfo->pPullDataRes = createPullDataBlock();
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
|
||||
|
||||
pOperator->operatorType = pPhyNode->type;
|
||||
pOperator->blocking = true;
|
||||
|
@ -2852,12 +2852,12 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
|
|||
pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
|
||||
pInfo->pDelIterator = NULL;
|
||||
pInfo->pDelRes = createOneDataBlock(pResBlock, false);
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE;
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
|
||||
blockDataEnsureCapacity(pInfo->pDelRes, 64);
|
||||
pInfo->pChildren = NULL;
|
||||
pInfo->isFinal = false;
|
||||
pInfo->pPhyNode = pPhyNode;
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
|
||||
|
||||
pOperator->name = "StreamSessionWindowAggOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION;
|
||||
|
@ -3133,7 +3133,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
|
|||
|
||||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
for (int32_t i = 0; i < pSDataBlock->info.rows;) {
|
||||
if (pInfo->ignoreCloseWindow && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
|
||||
if (pInfo->ignoreExpiredData && isOverdue(endTsCols[i], &pInfo->twAggSup)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
@ -3413,8 +3413,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) {
|
|||
pOperator->status = OP_RES_TO_RETURN;
|
||||
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
|
||||
getResWinForSession, pInfo->ignoreCloseWindow);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
|
||||
getResWinForSession, pInfo->ignoreExpiredData);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
|
||||
copyUpdateResult(pStUpdated, pUpdated);
|
||||
taosHashCleanup(pStUpdated);
|
||||
|
||||
|
@ -3822,7 +3822,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
|
|||
SStreamAggSupporter* pAggSup = &pInfo->streamAggSup;
|
||||
SColumnInfoData* pKeyColInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->stateCol.slotId);
|
||||
for (int32_t i = 0; i < pSDataBlock->info.rows; i += winRows) {
|
||||
if (pInfo->ignoreCloseWindow && isOverdue(tsCols[i], &pInfo->twAggSup)) {
|
||||
if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) {
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
@ -3866,12 +3866,14 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
|
|||
if (pOperator->status == OP_RES_TO_RETURN) {
|
||||
doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, "single state");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
|
||||
if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) {
|
||||
doSetOperatorCompleted(pOperator);
|
||||
}
|
||||
printDataBlock(pBInfo->pRes, "single state");
|
||||
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
|
||||
}
|
||||
|
||||
|
@ -3884,6 +3886,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
|
|||
if (pBlock == NULL) {
|
||||
break;
|
||||
}
|
||||
printDataBlock(pBlock, "single state recv");
|
||||
|
||||
if (pBlock->info.type == STREAM_CLEAR) {
|
||||
doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId,
|
||||
|
@ -3903,8 +3906,8 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
|
|||
pOperator->status = OP_RES_TO_RETURN;
|
||||
|
||||
closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated,
|
||||
getResWinForState, pInfo->ignoreCloseWindow);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreCloseWindow);
|
||||
getResWinForState, pInfo->ignoreExpiredData);
|
||||
closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData);
|
||||
copyUpdateResult(pSeUpdated, pUpdated);
|
||||
taosHashCleanup(pSeUpdated);
|
||||
|
||||
|
@ -3914,9 +3917,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) {
|
|||
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
|
||||
doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator);
|
||||
if (pInfo->pDelRes->info.rows > 0) {
|
||||
printDataBlock(pInfo->pDelRes, "single state");
|
||||
return pInfo->pDelRes;
|
||||
}
|
||||
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf);
|
||||
printDataBlock(pBInfo->pRes, "single state");
|
||||
return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes;
|
||||
}
|
||||
|
||||
|
@ -3975,10 +3980,10 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys
|
|||
pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
|
||||
pInfo->pDelIterator = NULL;
|
||||
pInfo->pDelRes = createOneDataBlock(pResBlock, false);
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE;
|
||||
pInfo->pDelRes->info.type = STREAM_DELETE_RESULT;
|
||||
blockDataEnsureCapacity(pInfo->pDelRes, 64);
|
||||
pInfo->pChildren = NULL;
|
||||
pInfo->ignoreCloseWindow = false;
|
||||
pInfo->ignoreExpiredData = pStateNode->window.igExpired;
|
||||
|
||||
pOperator->name = "StreamStateAggOperator";
|
||||
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE;
|
||||
|
|
|
@ -593,7 +593,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
|
|||
if (size > sortBufSize) {
|
||||
// Perform the in-memory sort and then flush data in the buffer into disk.
|
||||
int64_t p = taosGetTimestampUs();
|
||||
blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
|
||||
code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
|
||||
if (code != 0) {
|
||||
return code;
|
||||
}
|
||||
|
||||
int64_t el = taosGetTimestampUs() - p;
|
||||
pHandle->sortElapsed += el;
|
||||
|
@ -608,7 +611,10 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
|
|||
// Perform the in-memory sort and then flush data in the buffer into disk.
|
||||
int64_t p = taosGetTimestampUs();
|
||||
|
||||
blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
|
||||
int32_t code = blockDataSort(pHandle->pDataBlock, pHandle->pSortInfo);
|
||||
if (code != 0) {
|
||||
return code;
|
||||
}
|
||||
|
||||
int64_t el = taosGetTimestampUs() - p;
|
||||
pHandle->sortElapsed += el;
|
||||
|
|
|
@ -117,6 +117,9 @@ int32_t firstCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
|
|||
int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx);
|
||||
int32_t getFirstLastInfoSize(int32_t resBytes);
|
||||
|
||||
int32_t lastRowFunction(SqlFunctionCtx *pCtx);
|
||||
int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
|
||||
|
||||
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
|
||||
bool getTopBotMergeFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
|
||||
bool topBotFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
|
||||
|
|
|
@ -32,7 +32,7 @@ extern "C" {
|
|||
#define FUNC_MGT_STRING_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(3)
|
||||
#define FUNC_MGT_DATETIME_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
|
||||
#define FUNC_MGT_TIMELINE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
|
||||
#define FUNC_MGT_TIMEORDER_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
|
||||
#define FUNC_MGT_IMPLICIT_TS_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
|
||||
#define FUNC_MGT_PSEUDO_COLUMN_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
|
||||
#define FUNC_MGT_WINDOW_PC_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
|
||||
#define FUNC_MGT_SPECIAL_DATA_REQUIRED FUNC_MGT_FUNC_CLASSIFICATION_MASK(9)
|
||||
|
|
|
@ -39,6 +39,174 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
|
|||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName);
|
||||
}
|
||||
|
||||
#define TIME_UNIT_INVALID 1
|
||||
#define TIME_UNIT_TOO_SMALL 2
|
||||
|
||||
static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
|
||||
if (!pVal->isDuration) {
|
||||
return TIME_UNIT_INVALID;
|
||||
}
|
||||
|
||||
if (TSDB_TIME_PRECISION_MILLI == dbPrec && 0 == strcasecmp(pVal->literal, "1u")) {
|
||||
return TIME_UNIT_TOO_SMALL;
|
||||
}
|
||||
|
||||
if (pVal->literal[0] != '1' || (pVal->literal[1] != 'u' && pVal->literal[1] != 'a' &&
|
||||
pVal->literal[1] != 's' && pVal->literal[1] != 'm' &&
|
||||
pVal->literal[1] != 'h' && pVal->literal[1] != 'd' &&
|
||||
pVal->literal[1] != 'w')) {
|
||||
return TIME_UNIT_INVALID;
|
||||
}
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
/* Following are valid ISO-8601 timezone format:
|
||||
* 1 z/Z
|
||||
* 2 ±hh:mm
|
||||
* 3 ±hhmm
|
||||
* 4 ±hh
|
||||
*
|
||||
*/
|
||||
|
||||
static bool validateHourRange(int8_t hour) {
|
||||
if (hour < 0 || hour > 12) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
|
||||
if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool validateTimestampDigits(const SValueNode* pVal) {
|
||||
if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int64_t tsVal = pVal->datum.i;
|
||||
char fraction[20] = {0};
|
||||
NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
|
||||
int32_t tsDigits = (int32_t)strlen(fraction);
|
||||
|
||||
if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
|
||||
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
|
||||
tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool validateTimezoneFormat(const SValueNode* pVal) {
|
||||
if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
|
||||
return false;
|
||||
}
|
||||
|
||||
char* tz = varDataVal(pVal->datum.p);
|
||||
int32_t len = varDataLen(pVal->datum.p);
|
||||
|
||||
char buf[3] = {0};
|
||||
int8_t hour = -1, minute = -1;
|
||||
if (len == 0) {
|
||||
return false;
|
||||
} else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
|
||||
return true;
|
||||
} else if ((tz[0] == '+' || tz[0] == '-')) {
|
||||
switch (len) {
|
||||
case 3:
|
||||
case 5: {
|
||||
for (int32_t i = 1; i < len; ++i) {
|
||||
if (!isdigit(tz[i])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (i == 2) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
hour = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateHourRange(hour)) {
|
||||
return false;
|
||||
}
|
||||
} else if (i == 4) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
minute = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateMinuteRange(hour, minute, tz[0])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 6: {
|
||||
for (int32_t i = 1; i < len; ++i) {
|
||||
if (i == 3) {
|
||||
if (tz[i] != ':') {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isdigit(tz[i])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (i == 2) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
hour = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateHourRange(hour)) {
|
||||
return false;
|
||||
}
|
||||
} else if (i == 5) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
minute = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateMinuteRange(hour, minute, tz[0])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void static addTimezoneParam(SNodeList* pList) {
|
||||
char buf[6] = {0};
|
||||
time_t t = taosTime(NULL);
|
||||
struct tm* tmInfo = taosLocalTime(&t, NULL);
|
||||
strftime(buf, sizeof(buf), "%z", tmInfo);
|
||||
int32_t len = (int32_t)strlen(buf);
|
||||
|
||||
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
pVal->literal = strndup(buf, len);
|
||||
pVal->isDuration = false;
|
||||
pVal->translate = true;
|
||||
pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
|
||||
pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
|
||||
pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
|
||||
pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
|
||||
varDataSetLen(pVal->datum.p, len);
|
||||
strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
|
||||
|
||||
nodesListAppend(pList, (SNode*)pVal);
|
||||
}
|
||||
|
||||
void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
|
||||
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
pVal->literal = NULL;
|
||||
|
@ -238,7 +406,7 @@ static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t l
|
|||
static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
// pseudo column do not need to check parameters
|
||||
|
||||
//add database precision as param
|
||||
// add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
|
@ -525,9 +693,15 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
if (pValue->datum.i == 0) {
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"ELAPSED function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"ELAPSED function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -634,6 +808,12 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
|
|||
}
|
||||
|
||||
// param1 ~ param3
|
||||
if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
|
||||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
for (int32_t i = 1; i < numOfParams; ++i) {
|
||||
SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
|
||||
if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
|
||||
|
@ -643,12 +823,11 @@ static int32_t translateHistogram(SFunctionNode* pFunc, char* pErrBuf, int32_t l
|
|||
SValueNode* pValue = (SValueNode*)pParamNode;
|
||||
|
||||
pValue->notReserved = true;
|
||||
}
|
||||
|
||||
if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
|
||||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
if (i == 3 && pValue->datum.i != 1 && pValue->datum.i != 0) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"HISTOGRAM function normalized parameter should be 0/1");
|
||||
}
|
||||
}
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = 512, .type = TSDB_DATA_TYPE_BINARY};
|
||||
|
@ -668,6 +847,12 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
|
|||
}
|
||||
|
||||
// param1 ~ param3
|
||||
if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
|
||||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
for (int32_t i = 1; i < numOfParams; ++i) {
|
||||
SNode* pParamNode = nodesListGetNode(pFunc->pParameterList, i);
|
||||
if (QUERY_NODE_VALUE != nodeType(pParamNode)) {
|
||||
|
@ -677,12 +862,11 @@ static int32_t translateHistogramImpl(SFunctionNode* pFunc, char* pErrBuf, int32
|
|||
SValueNode* pValue = (SValueNode*)pParamNode;
|
||||
|
||||
pValue->notReserved = true;
|
||||
}
|
||||
|
||||
if (((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type != TSDB_DATA_TYPE_BINARY ||
|
||||
((SExprNode*)nodesListGetNode(pFunc->pParameterList, 3))->resType.type != TSDB_DATA_TYPE_BIGINT) {
|
||||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
if (i == 3 && pValue->datum.i != 1 && pValue->datum.i != 0) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"HISTOGRAM function normalized parameter should be 0/1");
|
||||
}
|
||||
}
|
||||
|
||||
pFunc->node.resType =
|
||||
|
@ -833,6 +1017,19 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
if (numOfParams == 4) {
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 3));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"STATEDURATION function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"STATEDURATION function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
}
|
||||
|
||||
// set result type
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
@ -1017,7 +1214,7 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
// add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
|
@ -1031,13 +1228,7 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
|
||||
if (QUERY_NODE_COLUMN != nodeType(pPara)) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"The parameters of first/last can only be columns");
|
||||
}
|
||||
|
||||
pFunc->node.resType = ((SExprNode*)pPara)->resType;
|
||||
pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType;
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -1051,11 +1242,6 @@ static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32
|
|||
uint8_t paraType = ((SExprNode*)pPara)->resType.type;
|
||||
int32_t paraBytes = ((SExprNode*)pPara)->resType.bytes;
|
||||
if (isPartial) {
|
||||
if (QUERY_NODE_COLUMN != nodeType(pPara)) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"The parameters of first/last can only be columns");
|
||||
}
|
||||
|
||||
pFunc->node.resType =
|
||||
(SDataType){.bytes = getFirstLastInfoSize(paraBytes) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
|
||||
} else {
|
||||
|
@ -1260,19 +1446,9 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
|
||||
static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
// The number of parameters has been limited by the syntax definition
|
||||
// uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
|
||||
|
||||
// The function return type has been set during syntax parsing
|
||||
uint8_t para2Type = pFunc->node.resType.type;
|
||||
// if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT &&
|
||||
// para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR &&
|
||||
// para2Type != TSDB_DATA_TYPE_TIMESTAMP) {
|
||||
// return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
// }
|
||||
// if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) ||
|
||||
// (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) {
|
||||
// return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
// }
|
||||
|
||||
int32_t para2Bytes = pFunc->node.resType.bytes;
|
||||
if (IS_VAR_DATA_TYPE(para2Type)) {
|
||||
|
@ -1282,155 +1458,14 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
|||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"CAST function converted length should be in range [0, 1000]");
|
||||
}
|
||||
|
||||
// add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
/* Following are valid ISO-8601 timezone format:
|
||||
* 1 z/Z
|
||||
* 2 ±hh:mm
|
||||
* 3 ±hhmm
|
||||
* 4 ±hh
|
||||
*
|
||||
*/
|
||||
|
||||
static bool validateHourRange(int8_t hour) {
|
||||
if (hour < 0 || hour > 12) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) {
|
||||
if (minute == 0 || (minute == 30 && (hour == 3 || hour == 5) && sign == '+')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool validateTimestampDigits(const SValueNode* pVal) {
|
||||
if (!IS_INTEGER_TYPE(pVal->node.resType.type)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int64_t tsVal = pVal->datum.i;
|
||||
char fraction[20] = {0};
|
||||
NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction);
|
||||
int32_t tsDigits = (int32_t)strlen(fraction);
|
||||
|
||||
if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) {
|
||||
if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS ||
|
||||
tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) {
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool validateTimezoneFormat(const SValueNode* pVal) {
|
||||
if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
|
||||
return false;
|
||||
}
|
||||
|
||||
char* tz = varDataVal(pVal->datum.p);
|
||||
int32_t len = varDataLen(pVal->datum.p);
|
||||
|
||||
char buf[3] = {0};
|
||||
int8_t hour = -1, minute = -1;
|
||||
if (len == 0) {
|
||||
return false;
|
||||
} else if (len == 1 && (tz[0] == 'z' || tz[0] == 'Z')) {
|
||||
return true;
|
||||
} else if ((tz[0] == '+' || tz[0] == '-')) {
|
||||
switch (len) {
|
||||
case 3:
|
||||
case 5: {
|
||||
for (int32_t i = 1; i < len; ++i) {
|
||||
if (!isdigit(tz[i])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (i == 2) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
hour = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateHourRange(hour)) {
|
||||
return false;
|
||||
}
|
||||
} else if (i == 4) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
minute = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateMinuteRange(hour, minute, tz[0])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 6: {
|
||||
for (int32_t i = 1; i < len; ++i) {
|
||||
if (i == 3) {
|
||||
if (tz[i] != ':') {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isdigit(tz[i])) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (i == 2) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
hour = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateHourRange(hour)) {
|
||||
return false;
|
||||
}
|
||||
} else if (i == 5) {
|
||||
memcpy(buf, &tz[i - 1], 2);
|
||||
minute = taosStr2Int8(buf, NULL, 10);
|
||||
if (!validateMinuteRange(hour, minute, tz[0])) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void static addTimezoneParam(SNodeList* pList) {
|
||||
char buf[6] = {0};
|
||||
time_t t = taosTime(NULL);
|
||||
struct tm* tmInfo = taosLocalTime(&t, NULL);
|
||||
strftime(buf, sizeof(buf), "%z", tmInfo);
|
||||
int32_t len = (int32_t)strlen(buf);
|
||||
|
||||
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
|
||||
pVal->literal = strndup(buf, len);
|
||||
pVal->isDuration = false;
|
||||
pVal->translate = true;
|
||||
pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
|
||||
pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
|
||||
pVal->node.resType.precision = TSDB_TIME_PRECISION_MILLI;
|
||||
pVal->datum.p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE + 1);
|
||||
varDataSetLen(pVal->datum.p, len);
|
||||
strncpy(varDataVal(pVal->datum.p), pVal->literal, len);
|
||||
|
||||
nodesListAppend(pList, (SNode*)pVal);
|
||||
}
|
||||
|
||||
static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
|
||||
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
|
||||
if (1 != numOfParams && 2 != numOfParams) {
|
||||
|
@ -1477,7 +1512,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
// add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
|
@ -1497,8 +1532,18 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
// add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 1));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMETRUNCATE function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMETRUNCATE function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
pFunc->node.resType =
|
||||
|
@ -1525,8 +1570,20 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le
|
|||
}
|
||||
}
|
||||
|
||||
//add database precision as param
|
||||
// add database precision as param
|
||||
uint8_t dbPrec = pFunc->node.resType.precision;
|
||||
|
||||
if (3 == numOfParams) {
|
||||
int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode *)nodesListGetNode(pFunc->pParameterList, 2));
|
||||
if (ret == TIME_UNIT_TOO_SMALL) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMEDIFF function time unit parameter should be greater than db precision");
|
||||
} else if (ret == TIME_UNIT_INVALID) {
|
||||
return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
|
||||
"TIMEDIFF function time unit parameter should be one of the following: [1u, 1a, 1s, 1m, 1h, 1d, 1w]");
|
||||
}
|
||||
}
|
||||
|
||||
addDbPrecisonParam(&pFunc->pParameterList, dbPrec);
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
|
||||
|
@ -1543,7 +1600,7 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len)
|
|||
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
|
||||
}
|
||||
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BINARY].bytes, .type = TSDB_DATA_TYPE_BINARY};
|
||||
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_JSON].bytes, .type = TSDB_DATA_TYPE_JSON};
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
@ -1924,7 +1981,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "interp",
|
||||
.type = FUNCTION_TYPE_INTERP,
|
||||
.classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC,
|
||||
.classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLast,
|
||||
.getEnvFunc = getSelectivityFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -1934,7 +1991,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "derivative",
|
||||
.type = FUNCTION_TYPE_DERIVATIVE,
|
||||
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateDerivative,
|
||||
.getEnvFunc = getDerivativeFuncEnv,
|
||||
.initFunc = derivativeFuncSetup,
|
||||
|
@ -1944,7 +2001,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "irate",
|
||||
.type = FUNCTION_TYPE_IRATE,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateIrate,
|
||||
.getEnvFunc = getIrateFuncEnv,
|
||||
.initFunc = irateFuncSetup,
|
||||
|
@ -1954,20 +2011,17 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "last_row",
|
||||
.type = FUNCTION_TYPE_LAST_ROW,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLast,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
.processFunc = lastFunction,
|
||||
.finalizeFunc = firstLastFinalize,
|
||||
.pPartialFunc = "_last_partial",
|
||||
.pMergeFunc = "_last_merge",
|
||||
.combineFunc = lastCombine,
|
||||
.processFunc = lastRowFunction,
|
||||
.finalizeFunc = lastRowFinalize,
|
||||
},
|
||||
{
|
||||
.name = "_cache_last_row",
|
||||
.type = FUNCTION_TYPE_CACHE_LAST_ROW,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateLastRow,
|
||||
.getEnvFunc = getMinmaxFuncEnv,
|
||||
.initFunc = minmaxFunctionSetup,
|
||||
|
@ -1977,7 +2031,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "first",
|
||||
.type = FUNCTION_TYPE_FIRST,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLast,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -1990,7 +2044,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "_first_partial",
|
||||
.type = FUNCTION_TYPE_FIRST_PARTIAL,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLastPartial,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -2001,7 +2055,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "_first_merge",
|
||||
.type = FUNCTION_TYPE_FIRST_MERGE,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLastMerge,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -2012,7 +2066,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "last",
|
||||
.type = FUNCTION_TYPE_LAST,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLast,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -2025,7 +2079,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "_last_partial",
|
||||
.type = FUNCTION_TYPE_LAST_PARTIAL,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLastPartial,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -2036,7 +2090,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "_last_merge",
|
||||
.type = FUNCTION_TYPE_LAST_MERGE,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateFirstLastMerge,
|
||||
.getEnvFunc = getFirstLastFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -2047,7 +2101,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "twa",
|
||||
.type = FUNCTION_TYPE_TWA,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateInNumOutDou,
|
||||
.dataRequiredFunc = statisDataRequired,
|
||||
.getEnvFunc = getTwaFuncEnv,
|
||||
|
@ -2144,7 +2198,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "statecount",
|
||||
.type = FUNCTION_TYPE_STATE_COUNT,
|
||||
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC,
|
||||
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC,
|
||||
.translateFunc = translateStateCount,
|
||||
.getEnvFunc = getStateFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -2154,7 +2208,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "stateduration",
|
||||
.type = FUNCTION_TYPE_STATE_DURATION,
|
||||
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC,
|
||||
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC,
|
||||
.translateFunc = translateStateDuration,
|
||||
.getEnvFunc = getStateFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
@ -2194,7 +2248,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "tail",
|
||||
.type = FUNCTION_TYPE_TAIL,
|
||||
.classification = FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC,
|
||||
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_FORBID_STREAM_FUNC |
|
||||
FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateTail,
|
||||
.getEnvFunc = getTailFuncEnv,
|
||||
.initFunc = tailFunctionSetup,
|
||||
|
@ -2205,7 +2260,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
.name = "unique",
|
||||
.type = FUNCTION_TYPE_UNIQUE,
|
||||
.classification = FUNC_MGT_SELECT_FUNC | FUNC_MGT_INDEFINITE_ROWS_FUNC | FUNC_MGT_TIMELINE_FUNC |
|
||||
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC,
|
||||
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_WINDOW_FUNC | FUNC_MGT_FORBID_GROUP_BY_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC,
|
||||
.translateFunc = translateUnique,
|
||||
.getEnvFunc = getUniqueFuncEnv,
|
||||
.initFunc = uniqueFunctionSetup,
|
||||
|
@ -2623,7 +2678,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
|
|||
{
|
||||
.name = "_group_key",
|
||||
.type = FUNCTION_TYPE_GROUP_KEY,
|
||||
.classification = FUNC_MGT_AGG_FUNC,
|
||||
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC,
|
||||
.translateFunc = translateGroupKey,
|
||||
.getEnvFunc = getGroupKeyFuncEnv,
|
||||
.initFunc = functionSetup,
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
#include "builtinsimpl.h"
|
||||
#include "cJSON.h"
|
||||
#include "function.h"
|
||||
#include "query.h"
|
||||
#include "querynodes.h"
|
||||
#include "taggfunction.h"
|
||||
#include "tcompare.h"
|
||||
|
@ -80,6 +81,7 @@ typedef struct STopBotRes {
|
|||
|
||||
typedef struct SFirstLastRes {
|
||||
bool hasResult;
|
||||
bool isNull; // used for last_row function only
|
||||
int32_t bytes;
|
||||
char buf[];
|
||||
} SFirstLastRes;
|
||||
|
@ -208,13 +210,13 @@ typedef struct SMavgInfo {
|
|||
} SMavgInfo;
|
||||
|
||||
typedef struct SSampleInfo {
|
||||
int32_t samples;
|
||||
int32_t totalPoints;
|
||||
int32_t numSampled;
|
||||
uint8_t colType;
|
||||
int16_t colBytes;
|
||||
char* data;
|
||||
STuplePos* tuplePos;
|
||||
int32_t samples;
|
||||
int32_t totalPoints;
|
||||
int32_t numSampled;
|
||||
uint8_t colType;
|
||||
int16_t colBytes;
|
||||
char* data;
|
||||
STuplePos* tuplePos;
|
||||
} SSampleInfo;
|
||||
|
||||
typedef struct STailItem {
|
||||
|
@ -269,20 +271,19 @@ typedef struct SDerivInfo {
|
|||
} SDerivInfo;
|
||||
|
||||
typedef struct SRateInfo {
|
||||
double firstValue;
|
||||
TSKEY firstKey;
|
||||
double lastValue;
|
||||
TSKEY lastKey;
|
||||
int8_t hasResult; // flag to denote has value
|
||||
double firstValue;
|
||||
TSKEY firstKey;
|
||||
double lastValue;
|
||||
TSKEY lastKey;
|
||||
int8_t hasResult; // flag to denote has value
|
||||
} SRateInfo;
|
||||
|
||||
typedef struct SGroupKeyInfo{
|
||||
bool hasResult;
|
||||
bool isNull;
|
||||
char data[];
|
||||
typedef struct SGroupKeyInfo {
|
||||
bool hasResult;
|
||||
bool isNull;
|
||||
char data[];
|
||||
} SGroupKeyInfo;
|
||||
|
||||
|
||||
#define SET_VAL(_info, numOfElem, res) \
|
||||
do { \
|
||||
if ((numOfElem) <= 0) { \
|
||||
|
@ -1471,8 +1472,8 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple
|
|||
int32_t pageId = pTuplePos->pageId;
|
||||
int32_t offset = pTuplePos->offset;
|
||||
|
||||
if (pTuplePos->pageId != -1) {
|
||||
int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock);
|
||||
if (pTuplePos->pageId != -1 && pCtx->subsidiaries.num > 0) {
|
||||
int32_t numOfCols = pCtx->subsidiaries.num;
|
||||
SFilePage* pPage = getBufPage(pCtx->pBuf, pageId);
|
||||
|
||||
bool* nullList = (bool*)((char*)pPage + offset);
|
||||
|
@ -1483,22 +1484,21 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple
|
|||
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
|
||||
|
||||
SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0];
|
||||
int32_t srcSlotId = pFuncParam->pCol->slotId;
|
||||
int32_t dstSlotId = pc->pExpr->base.resSchema.slotId;
|
||||
|
||||
int32_t ps = 0;
|
||||
for (int32_t k = 0; k < srcSlotId; ++k) {
|
||||
SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, k);
|
||||
ps += pSrcCol->info.bytes;
|
||||
}
|
||||
|
||||
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
|
||||
if (nullList[srcSlotId]) {
|
||||
ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes);
|
||||
if (nullList[j]) {
|
||||
colDataAppendNULL(pDstCol, rowIndex);
|
||||
} else {
|
||||
colDataAppend(pDstCol, rowIndex, (pStart + ps), false);
|
||||
colDataAppend(pDstCol, rowIndex, pStart, false);
|
||||
}
|
||||
pStart += pDstCol->info.bytes;
|
||||
}
|
||||
|
||||
releaseBufPage(pCtx->pBuf, pPage);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -2409,7 +2409,9 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx)
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t getFirstLastInfoSize(int32_t resBytes) { return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos); }
|
||||
int32_t getFirstLastInfoSize(int32_t resBytes) {
|
||||
return sizeof(SFirstLastRes) + resBytes + sizeof(int64_t) + sizeof(STuplePos);
|
||||
}
|
||||
|
||||
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
|
||||
SColumnNode* pNode = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0);
|
||||
|
@ -2491,7 +2493,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
memcpy(pInfo->buf, data, bytes);
|
||||
*(TSKEY*)(pInfo->buf + bytes) = cts;
|
||||
//handle selectivity
|
||||
// handle selectivity
|
||||
if (pCtx->subsidiaries.num > 0) {
|
||||
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
|
||||
if (!pInfo->hasResult) {
|
||||
|
@ -2501,7 +2503,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
}
|
||||
pInfo->hasResult = true;
|
||||
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
pResInfo->numOfRes = 1;
|
||||
break;
|
||||
}
|
||||
|
@ -2533,7 +2535,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
memcpy(pInfo->buf, data, bytes);
|
||||
*(TSKEY*)(pInfo->buf + bytes) = cts;
|
||||
//handle selectivity
|
||||
// handle selectivity
|
||||
if (pCtx->subsidiaries.num > 0) {
|
||||
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
|
||||
if (!pInfo->hasResult) {
|
||||
|
@ -2543,7 +2545,7 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
}
|
||||
pInfo->hasResult = true;
|
||||
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
pResInfo->numOfRes = 1;
|
||||
break;
|
||||
}
|
||||
|
@ -2597,7 +2599,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
memcpy(pInfo->buf, data, bytes);
|
||||
*(TSKEY*)(pInfo->buf + bytes) = cts;
|
||||
//handle selectivity
|
||||
// handle selectivity
|
||||
if (pCtx->subsidiaries.num > 0) {
|
||||
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
|
||||
if (!pInfo->hasResult) {
|
||||
|
@ -2607,7 +2609,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
}
|
||||
pInfo->hasResult = true;
|
||||
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
pResInfo->numOfRes = 1;
|
||||
}
|
||||
break;
|
||||
|
@ -2629,7 +2631,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
memcpy(pInfo->buf, data, bytes);
|
||||
*(TSKEY*)(pInfo->buf + bytes) = cts;
|
||||
//handle selectivity
|
||||
// handle selectivity
|
||||
if (pCtx->subsidiaries.num > 0) {
|
||||
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
|
||||
if (!pInfo->hasResult) {
|
||||
|
@ -2640,7 +2642,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
pInfo->hasResult = true;
|
||||
pResInfo->numOfRes = 1;
|
||||
//DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
@ -2652,7 +2654,7 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) {
|
|||
|
||||
static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, SFirstLastRes* pOutput, bool isFirst) {
|
||||
SInputColumnInfoData* pColInfo = &pCtx->input;
|
||||
int32_t start = pColInfo->startRowIndex;
|
||||
int32_t start = pColInfo->startRowIndex;
|
||||
|
||||
pOutput->bytes = pInput->bytes;
|
||||
TSKEY* tsIn = (TSKEY*)(pInput->buf + pInput->bytes);
|
||||
|
@ -2670,7 +2672,7 @@ static void firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput, S
|
|||
}
|
||||
*tsOut = *tsIn;
|
||||
memcpy(pOutput->buf, pInput->buf, pOutput->bytes);
|
||||
//handle selectivity
|
||||
// handle selectivity
|
||||
STuplePos* pTuplePos = (STuplePos*)(pOutput->buf + pOutput->bytes + sizeof(TSKEY));
|
||||
if (pCtx->subsidiaries.num > 0) {
|
||||
if (!pOutput->hasResult) {
|
||||
|
@ -2717,7 +2719,7 @@ int32_t firstLastFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
|
||||
SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
colDataAppend(pCol, pBlock->info.rows, pRes->buf, pResInfo->isNullRes);
|
||||
//handle selectivity
|
||||
// handle selectivity
|
||||
STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
|
||||
setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
|
||||
|
||||
|
@ -2728,8 +2730,8 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
|
||||
SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
|
||||
|
||||
int32_t resultBytes = getFirstLastInfoSize(pRes->bytes);
|
||||
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
|
||||
int32_t resultBytes = getFirstLastInfoSize(pRes->bytes);
|
||||
char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
|
||||
|
||||
memcpy(varDataVal(res), pRes, resultBytes);
|
||||
varDataSetLen(res, resultBytes);
|
||||
|
@ -2738,7 +2740,7 @@ int32_t firstLastPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
|
||||
|
||||
colDataAppend(pCol, pBlock->info.rows, res, false);
|
||||
//handle selectivity
|
||||
// handle selectivity
|
||||
STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
|
||||
setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
|
||||
|
||||
|
@ -2763,6 +2765,111 @@ int32_t lastCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t lastRowFunction(SqlFunctionCtx* pCtx) {
|
||||
int32_t numOfElems = 0;
|
||||
|
||||
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
|
||||
SFirstLastRes* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
|
||||
SInputColumnInfoData* pInput = &pCtx->input;
|
||||
SColumnInfoData* pInputCol = pInput->pData[0];
|
||||
|
||||
int32_t type = pInputCol->info.type;
|
||||
int32_t bytes = pInputCol->info.bytes;
|
||||
pInfo->bytes = bytes;
|
||||
|
||||
SColumnDataAgg* pColAgg = (pInput->colDataAggIsSet) ? pInput->pColumnDataAgg[0] : NULL;
|
||||
|
||||
TSKEY startKey = getRowPTs(pInput->pPTS, 0);
|
||||
TSKEY endKey = getRowPTs(pInput->pPTS, pInput->totalRows - 1);
|
||||
|
||||
int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC;
|
||||
|
||||
if (blockDataOrder == TSDB_ORDER_ASC) {
|
||||
for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) {
|
||||
char* data = colDataGetData(pInputCol, i);
|
||||
TSKEY cts = getRowPTs(pInput->pPTS, i);
|
||||
if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) {
|
||||
if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
|
||||
pInfo->isNull = true;
|
||||
} else {
|
||||
pInfo->isNull = false;
|
||||
if (IS_VAR_DATA_TYPE(type)) {
|
||||
bytes = varDataTLen(data);
|
||||
pInfo->bytes = bytes;
|
||||
}
|
||||
memcpy(pInfo->buf + sizeof(TSKEY), data, bytes);
|
||||
}
|
||||
*(TSKEY*)(pInfo->buf) = cts;
|
||||
numOfElems++;
|
||||
// handle selectivity
|
||||
if (pCtx->subsidiaries.num > 0) {
|
||||
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
|
||||
if (!pInfo->hasResult) {
|
||||
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
|
||||
} else {
|
||||
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
|
||||
}
|
||||
}
|
||||
pInfo->hasResult = true;
|
||||
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
pResInfo->numOfRes = 1;
|
||||
}
|
||||
break;
|
||||
}
|
||||
} else { // descending order
|
||||
for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) {
|
||||
char* data = colDataGetData(pInputCol, i);
|
||||
TSKEY cts = getRowPTs(pInput->pPTS, i);
|
||||
if (pResInfo->numOfRes == 0 || *(TSKEY*)(pInfo->buf) < cts) {
|
||||
if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) {
|
||||
pInfo->isNull = true;
|
||||
} else {
|
||||
pInfo->isNull = false;
|
||||
if (IS_VAR_DATA_TYPE(type)) {
|
||||
bytes = varDataTLen(data);
|
||||
pInfo->bytes = bytes;
|
||||
}
|
||||
memcpy(pInfo->buf + sizeof(TSKEY), data, bytes);
|
||||
}
|
||||
*(TSKEY*)(pInfo->buf) = cts;
|
||||
numOfElems++;
|
||||
// handle selectivity
|
||||
if (pCtx->subsidiaries.num > 0) {
|
||||
STuplePos* pTuplePos = (STuplePos*)(pInfo->buf + bytes + sizeof(TSKEY));
|
||||
if (!pInfo->hasResult) {
|
||||
saveTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
|
||||
} else {
|
||||
copyTupleData(pCtx, i, pCtx->pSrcBlock, pTuplePos);
|
||||
}
|
||||
}
|
||||
pInfo->hasResult = true;
|
||||
pResInfo->numOfRes = 1;
|
||||
// DO_UPDATE_TAG_COLUMNS(pCtx, ts);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
SET_VAL(pResInfo, numOfElems, 1);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
int32_t lastRowFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
||||
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
|
||||
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
|
||||
|
||||
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
|
||||
|
||||
SFirstLastRes* pRes = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
colDataAppend(pCol, pBlock->info.rows, pRes->buf + sizeof(TSKEY), pRes->isNull);
|
||||
// handle selectivity
|
||||
STuplePos* pTuplePos = (STuplePos*)(pRes->buf + pRes->bytes + sizeof(TSKEY));
|
||||
setSelectivityValue(pCtx, pBlock, pTuplePos, pBlock->info.rows);
|
||||
|
||||
return pResInfo->numOfRes;
|
||||
}
|
||||
|
||||
bool getDiffFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
|
||||
pEnv->calcMemSize = sizeof(SDiffInfo);
|
||||
return true;
|
||||
|
@ -2776,7 +2883,11 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
|
|||
SDiffInfo* pDiffInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
pDiffInfo->hasPrev = false;
|
||||
pDiffInfo->prev.i64 = 0;
|
||||
pDiffInfo->ignoreNegative = pCtx->param[1].param.i; // TODO set correct param
|
||||
if (pCtx->numOfParams > 1) {
|
||||
pDiffInfo->ignoreNegative = pCtx->param[1].param.i; // TODO set correct param
|
||||
} else {
|
||||
pDiffInfo->ignoreNegative = false;
|
||||
}
|
||||
pDiffInfo->includeNull = false;
|
||||
pDiffInfo->firstOutput = false;
|
||||
return true;
|
||||
|
@ -2891,10 +3002,8 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
|
|||
SInputColumnInfoData* pInput = &pCtx->input;
|
||||
|
||||
SColumnInfoData* pInputCol = pInput->pData[0];
|
||||
SColumnInfoData* pTsOutput = pCtx->pTsOutput;
|
||||
|
||||
int32_t numOfElems = 0;
|
||||
TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
|
||||
int32_t startOffset = pCtx->offset;
|
||||
|
||||
SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
|
||||
|
@ -2906,9 +3015,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
|
|||
if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
|
||||
if (pDiffInfo->includeNull) {
|
||||
colDataSetNull_f(pOutput->nullbitmap, pos);
|
||||
if (tsList != NULL) {
|
||||
colDataAppendInt64(pTsOutput, pos, &tsList[i]);
|
||||
}
|
||||
|
||||
numOfElems += 1;
|
||||
}
|
||||
|
@ -2919,9 +3025,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
|
|||
|
||||
if (pDiffInfo->hasPrev) {
|
||||
doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order);
|
||||
if (pTsOutput != NULL) {
|
||||
colDataAppendInt64(pTsOutput, pos, &tsList[i]);
|
||||
}
|
||||
|
||||
numOfElems++;
|
||||
} else {
|
||||
|
@ -2937,9 +3040,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
|
|||
if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
|
||||
if (pDiffInfo->includeNull) {
|
||||
colDataSetNull_f(pOutput->nullbitmap, pos);
|
||||
if (tsList != NULL) {
|
||||
colDataAppendInt64(pTsOutput, pos, &tsList[i]);
|
||||
}
|
||||
|
||||
numOfElems += 1;
|
||||
}
|
||||
|
@ -2951,9 +3051,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
|
|||
// there is a row of previous data block to be handled in the first place.
|
||||
if (pDiffInfo->hasPrev) {
|
||||
doHandleDiff(pDiffInfo, pInputCol->info.type, pv, pOutput, pos, pCtx->order);
|
||||
if (pTsOutput != NULL) {
|
||||
colDataAppendInt64(pTsOutput, pos, &pDiffInfo->prevTs);
|
||||
}
|
||||
|
||||
numOfElems++;
|
||||
} else {
|
||||
|
@ -2961,9 +3058,6 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
|
||||
pDiffInfo->hasPrev = true;
|
||||
if (pTsOutput != NULL) {
|
||||
pDiffInfo->prevTs = tsList[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -3100,7 +3194,10 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
|
|||
if (pCtx->subsidiaries.num > 0) {
|
||||
saveTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
|
||||
}
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId,
|
||||
pItem->tuplePos.offset);
|
||||
#endif
|
||||
// allocate the buffer and keep the data of this row into the new allocated buffer
|
||||
pEntryInfo->numOfRes++;
|
||||
taosheapsort((void*)pItems, sizeof(STopBotResItem), pEntryInfo->numOfRes, (const void*)&type, topBotResComparFn,
|
||||
|
@ -3121,7 +3218,9 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
|
|||
if (pCtx->subsidiaries.num > 0) {
|
||||
copyTupleData(pCtx, rowIndex, pSrcBlock, &pItem->tuplePos);
|
||||
}
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_copyTuple pageId:%d, offset:%d", pItem->tuplePos.pageId, pItem->tuplePos.offset);
|
||||
#endif
|
||||
taosheapadjust((void*)pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void*)&type,
|
||||
topBotResComparFn, NULL, !isTopQuery);
|
||||
}
|
||||
|
@ -3131,7 +3230,11 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData
|
|||
void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
|
||||
SFilePage* pPage = NULL;
|
||||
|
||||
int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool);
|
||||
int32_t completeRowSize = pCtx->subsidiaries.num * sizeof(bool);
|
||||
for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
|
||||
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
|
||||
completeRowSize += pc->pExpr->base.resSchema.bytes;
|
||||
}
|
||||
|
||||
if (pCtx->curBufPage == -1) {
|
||||
pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
|
||||
|
@ -3149,19 +3252,22 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
|
|||
// keep the current row data, extract method
|
||||
int32_t offset = 0;
|
||||
bool* nullList = (bool*)((char*)pPage + pPage->num);
|
||||
char* pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock));
|
||||
for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) {
|
||||
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
|
||||
bool isNull = colDataIsNull_s(pCol, rowIndex);
|
||||
if (isNull) {
|
||||
nullList[i] = true;
|
||||
char* pStart = (char*)(nullList + sizeof(bool) * pCtx->subsidiaries.num);
|
||||
for (int32_t i = 0; i < pCtx->subsidiaries.num; ++i) {
|
||||
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[i];
|
||||
|
||||
SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0];
|
||||
int32_t srcSlotId = pFuncParam->pCol->slotId;
|
||||
|
||||
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId);
|
||||
if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) {
|
||||
offset += pCol->info.bytes;
|
||||
continue;
|
||||
}
|
||||
|
||||
char* p = colDataGetData(pCol, rowIndex);
|
||||
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
|
||||
memcpy(pStart + offset, p, varDataTLen(p));
|
||||
memcpy(pStart + offset, p, (pCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(p) : varDataTLen(p));
|
||||
} else {
|
||||
memcpy(pStart + offset, p, pCol->info.bytes);
|
||||
}
|
||||
|
@ -3179,14 +3285,18 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
|
|||
void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) {
|
||||
SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId);
|
||||
|
||||
int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock);
|
||||
int32_t numOfCols = pCtx->subsidiaries.num;
|
||||
|
||||
bool* nullList = (bool*)((char*)pPage + pPos->offset);
|
||||
char* pStart = (char*)(nullList + numOfCols * sizeof(bool));
|
||||
|
||||
int32_t offset = 0;
|
||||
for (int32_t i = 0; i < numOfCols; ++i) {
|
||||
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
|
||||
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[i];
|
||||
SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0];
|
||||
int32_t srcSlotId = pFuncParam->pCol->slotId;
|
||||
|
||||
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, srcSlotId);
|
||||
if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) {
|
||||
offset += pCol->info.bytes;
|
||||
continue;
|
||||
|
@ -3194,7 +3304,7 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
|
|||
|
||||
char* p = colDataGetData(pCol, rowIndex);
|
||||
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
|
||||
memcpy(pStart + offset, p, varDataTLen(p));
|
||||
memcpy(pStart + offset, p, (pCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(p) : varDataTLen(p));
|
||||
} else {
|
||||
memcpy(pStart + offset, p, pCol->info.bytes);
|
||||
}
|
||||
|
@ -3208,7 +3318,7 @@ void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS
|
|||
|
||||
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
||||
SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx);
|
||||
STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo);
|
||||
STopBotRes* pRes = getTopBotOutputInfo(pCtx);
|
||||
|
||||
int16_t type = pCtx->input.pData[0]->info.type;
|
||||
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
|
||||
|
@ -3225,7 +3335,10 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
} else {
|
||||
colDataAppend(pCol, currentRow, (const char*)&pItem->v.i, false);
|
||||
}
|
||||
|
||||
#ifdef BUF_PAGE_DEBUG
|
||||
qDebug("page_finalize i:%d,item:%p,pageId:%d, offset:%d\n", i, pItem, pItem->tuplePos.pageId,
|
||||
pItem->tuplePos.offset);
|
||||
#endif
|
||||
setSelectivityValue(pCtx, pBlock, &pRes->pItems[i].tuplePos, currentRow);
|
||||
currentRow += 1;
|
||||
}
|
||||
|
@ -3464,7 +3577,7 @@ bool elapsedFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo
|
|||
pInfo->min = MAX_TS_KEY;
|
||||
pInfo->max = 0;
|
||||
|
||||
if (pCtx->numOfParams > 2) {
|
||||
if (pCtx->numOfParams > 1) {
|
||||
pInfo->timeUnit = pCtx->param[1].param.i;
|
||||
} else {
|
||||
pInfo->timeUnit = 1;
|
||||
|
@ -4405,7 +4518,6 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) {
|
|||
SMavgInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
|
||||
SInputColumnInfoData* pInput = &pCtx->input;
|
||||
TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
|
||||
|
||||
SColumnInfoData* pInputCol = pInput->pData[0];
|
||||
SColumnInfoData* pTsOutput = pCtx->pTsOutput;
|
||||
|
@ -4445,10 +4557,6 @@ int32_t mavgFunction(SqlFunctionCtx* pCtx) {
|
|||
colDataAppend(pOutput, pos, (char*)&result, false);
|
||||
}
|
||||
|
||||
// TODO: remove this after pTsOutput is handled
|
||||
if (pTsOutput != NULL) {
|
||||
colDataAppendInt64(pTsOutput, pos, &tsList[i]);
|
||||
}
|
||||
numOfElems++;
|
||||
}
|
||||
|
||||
|
@ -4795,11 +4903,11 @@ static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) {
|
|||
return;
|
||||
}
|
||||
|
||||
int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes;
|
||||
SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes);
|
||||
int32_t hashKeyBytes = IS_VAR_DATA_TYPE(pInfo->colType) ? varDataTLen(data) : pInfo->colBytes;
|
||||
SModeItem** pHashItem = taosHashGet(pInfo->pHash, data, hashKeyBytes);
|
||||
if (pHashItem == NULL) {
|
||||
int32_t size = sizeof(SModeItem) + pInfo->colBytes;
|
||||
SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size);
|
||||
int32_t size = sizeof(SModeItem) + pInfo->colBytes;
|
||||
SModeItem* pItem = (SModeItem*)(pInfo->pItems + pInfo->numOfPoints * size);
|
||||
memcpy(pItem->data, data, pInfo->colBytes);
|
||||
pItem->count += 1;
|
||||
|
||||
|
@ -4812,7 +4920,7 @@ static void doModeAdd(SModeInfo* pInfo, char* data, bool isNull) {
|
|||
|
||||
int32_t modeFunction(SqlFunctionCtx* pCtx) {
|
||||
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
|
||||
SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
SModeInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
|
||||
SInputColumnInfoData* pInput = &pCtx->input;
|
||||
|
||||
|
@ -4860,7 +4968,6 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
return pResInfo->numOfRes;
|
||||
}
|
||||
|
||||
|
||||
bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
|
||||
pEnv->calcMemSize = sizeof(STwaInfo);
|
||||
return true;
|
||||
|
@ -5112,7 +5219,7 @@ int32_t twaFinalize(struct SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
return functionFinalize(pCtx, pBlock);
|
||||
}
|
||||
|
||||
bool blockDistSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
|
||||
bool blockDistSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
|
||||
if (!functionSetup(pCtx, pResultInfo)) {
|
||||
return false;
|
||||
}
|
||||
|
@ -5144,7 +5251,7 @@ int32_t blockDistFunction(SqlFunctionCtx* pCtx) {
|
|||
|
||||
pDistInfo->defMinRows = p1.defMinRows;
|
||||
pDistInfo->defMaxRows = p1.defMaxRows;
|
||||
pDistInfo->rowSize = p1.rowSize;
|
||||
pDistInfo->rowSize = p1.rowSize;
|
||||
pDistInfo->numOfSmallBlocks = p1.numOfSmallBlocks;
|
||||
|
||||
if (pDistInfo->minRows > p1.minRows) {
|
||||
|
@ -5230,16 +5337,18 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
int32_t row = 0;
|
||||
char st[256] = {0};
|
||||
double totalRawSize = pData->totalRows * pData->rowSize;
|
||||
int32_t len =
|
||||
sprintf(st + VARSTR_HEADER_SIZE, "Total_Blocks=[%d] Total_Size=[%.2f Kb] Average_size=[%.2f Kb] Compression_Ratio=[%.2f %c]",
|
||||
pData->numOfBlocks, pData->totalSize / 1024.0, ((double)pData->totalSize) / pData->numOfBlocks,
|
||||
pData->totalSize * 100 / totalRawSize, '%');
|
||||
int32_t len = sprintf(st + VARSTR_HEADER_SIZE,
|
||||
"Total_Blocks=[%d] Total_Size=[%.2f Kb] Average_size=[%.2f Kb] Compression_Ratio=[%.2f %c]",
|
||||
pData->numOfBlocks, pData->totalSize / 1024.0, ((double)pData->totalSize) / pData->numOfBlocks,
|
||||
pData->totalSize * 100 / totalRawSize, '%');
|
||||
|
||||
varDataSetLen(st, len);
|
||||
colDataAppend(pColInfo, row++, st, false);
|
||||
|
||||
len = sprintf(st + VARSTR_HEADER_SIZE, "Total_Rows=[%"PRId64"] Inmem_Rows=[%d] MinRows=[%d] MaxRows=[%d] Average_Rows=[%"PRId64"]",
|
||||
pData->totalRows, pData->numOfInmemRows, pData->minRows, pData->maxRows, pData->totalRows / pData->numOfBlocks);
|
||||
len = sprintf(st + VARSTR_HEADER_SIZE,
|
||||
"Total_Rows=[%" PRId64 "] Inmem_Rows=[%d] MinRows=[%d] MaxRows=[%d] Average_Rows=[%" PRId64 "]",
|
||||
pData->totalRows, pData->numOfInmemRows, pData->minRows, pData->maxRows,
|
||||
pData->totalRows / pData->numOfBlocks);
|
||||
|
||||
varDataSetLen(st, len);
|
||||
colDataAppend(pColInfo, row++, st, false);
|
||||
|
@ -5415,25 +5524,25 @@ bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) {
|
|||
|
||||
SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
|
||||
pInfo->firstKey = INT64_MIN;
|
||||
pInfo->lastKey = INT64_MIN;
|
||||
pInfo->firstKey = INT64_MIN;
|
||||
pInfo->lastKey = INT64_MIN;
|
||||
pInfo->firstValue = (double)INT64_MIN;
|
||||
pInfo->lastValue = (double)INT64_MIN;
|
||||
pInfo->lastValue = (double)INT64_MIN;
|
||||
|
||||
pInfo->hasResult = 0;
|
||||
pInfo->hasResult = 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
int32_t irateFunction(SqlFunctionCtx* pCtx) {
|
||||
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
|
||||
SRateInfo* pRateInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
SRateInfo* pRateInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
|
||||
SInputColumnInfoData* pInput = &pCtx->input;
|
||||
SColumnInfoData* pInputCol = pInput->pData[0];
|
||||
|
||||
SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
|
||||
|
||||
TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
|
||||
TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
|
||||
|
||||
int32_t numOfElems = 0;
|
||||
int32_t type = pInputCol->info.type;
|
||||
|
@ -5445,13 +5554,13 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) {
|
|||
|
||||
numOfElems++;
|
||||
|
||||
char* data = colDataGetData(pInputCol, i);
|
||||
double v = 0;
|
||||
char* data = colDataGetData(pInputCol, i);
|
||||
double v = 0;
|
||||
GET_TYPED_DATA(v, double, type, data);
|
||||
|
||||
if (INT64_MIN == pRateInfo->lastKey) {
|
||||
pRateInfo->lastValue = v;
|
||||
pRateInfo->lastKey = tsList[i];
|
||||
pRateInfo->lastKey = tsList[i];
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -5462,7 +5571,7 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
|
||||
pRateInfo->lastValue = v;
|
||||
pRateInfo->lastKey = tsList[i];
|
||||
pRateInfo->lastKey = tsList[i];
|
||||
|
||||
continue;
|
||||
}
|
||||
|
@ -5471,7 +5580,6 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) {
|
|||
pRateInfo->firstValue = v;
|
||||
pRateInfo->firstKey = tsList[i];
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
SET_VAL(pResInfo, numOfElems, 1);
|
||||
|
@ -5497,7 +5605,7 @@ static double doCalcRate(const SRateInfo* pRateInfo, double tickPerSec) {
|
|||
return 0;
|
||||
}
|
||||
|
||||
return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0;
|
||||
return (duration > 0) ? ((double)diff) / (duration / tickPerSec) : 0.0;
|
||||
}
|
||||
|
||||
int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
||||
|
@ -5508,7 +5616,7 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0;
|
||||
|
||||
SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
double result = doCalcRate(pInfo, (double)TSDB_TICK_PER_SECOND(pCtx->param[1].param.i));
|
||||
double result = doCalcRate(pInfo, (double)TSDB_TICK_PER_SECOND(pCtx->param[1].param.i));
|
||||
colDataAppend(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes);
|
||||
|
||||
return pResInfo->numOfRes;
|
||||
|
@ -5516,16 +5624,14 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
|
|||
|
||||
int32_t groupKeyFunction(SqlFunctionCtx* pCtx) {
|
||||
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
|
||||
SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo);
|
||||
|
||||
SInputColumnInfoData* pInput = &pCtx->input;
|
||||
SColumnInfoData* pInputCol = pInput->pData[0];
|
||||
|
||||
int32_t bytes = pInputCol->info.bytes;
|
||||
SColumnInfoData* pInputCol = pInput->pData[0];
|
||||
|
||||
int32_t startIndex = pInput->startRowIndex;
|
||||
|
||||
//escape rest of data blocks to avoid first entry to be overwritten.
|
||||
// escape rest of data blocks to avoid first entry to be overwritten.
|
||||
if (pInfo->hasResult) {
|
||||
goto _group_key_over;
|
||||
}
|
||||
|
@ -5537,7 +5643,12 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) {
|
|||
}
|
||||
|
||||
char* data = colDataGetData(pInputCol, startIndex);
|
||||
memcpy(pInfo->data, data, bytes);
|
||||
if (IS_VAR_DATA_TYPE(pInputCol->info.type)) {
|
||||
memcpy(pInfo->data, data,
|
||||
(pInputCol->info.type == TSDB_DATA_TYPE_JSON) ? getJsonValueLen(data) : varDataTLen(data));
|
||||
} else {
|
||||
memcpy(pInfo->data, data, pInputCol->info.bytes);
|
||||
}
|
||||
pInfo->hasResult = true;
|
||||
|
||||
_group_key_over:
|
||||
|
|
|
@ -181,6 +181,8 @@ bool fmIsForbidGroupByFunc(int32_t funcId) { return isSpecificClassifyFunc(funcI
|
|||
|
||||
bool fmIsSystemInfoFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SYSTEM_INFO_FUNC); }
|
||||
|
||||
bool fmIsImplicitTsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IMPLICIT_TS_FUNC); }
|
||||
|
||||
bool fmIsInterpFunc(int32_t funcId) {
|
||||
if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
|
||||
return false;
|
||||
|
|
|
@ -1887,10 +1887,10 @@ static void top_bottom_func_finalizer(SqlFunctionCtx *pCtx) {
|
|||
// user specify the order of output by sort the result according to timestamp
|
||||
if (pCtx->param[1].param.i == PRIMARYKEY_TIMESTAMP_COL_ID) {
|
||||
__compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn;
|
||||
qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
|
||||
taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
|
||||
} else /*if (pCtx->param[1].param.i > PRIMARYKEY_TIMESTAMP_COL_ID)*/ {
|
||||
__compar_fn_t comparator = (pCtx->param[2].param.i == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn;
|
||||
qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
|
||||
taosSort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator);
|
||||
}
|
||||
|
||||
GET_TRUE_DATA_TYPE();
|
||||
|
|
|
@ -44,7 +44,7 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx)
|
|||
offset += (int32_t)(pg->num * pMemBucket->bytes);
|
||||
}
|
||||
|
||||
qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
|
||||
taosSort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
|
|
|
@ -35,7 +35,7 @@ if (${BUILD_WITH_INVERTEDINDEX})
|
|||
endif(${BUILD_WITH_INVERTEDINDEX})
|
||||
|
||||
|
||||
if (${BUILD_TEST})
|
||||
add_subdirectory(test)
|
||||
endif(${BUILD_TEST})
|
||||
# if (${BUILD_TEST})
|
||||
# add_subdirectory(test)
|
||||
# endif(${BUILD_TEST})
|
||||
|
||||
|
|
|
@ -21,7 +21,7 @@ extern "C" {
|
|||
#endif
|
||||
|
||||
#include "indexFstAutomation.h"
|
||||
#include "indexFstCountingWriter.h"
|
||||
#include "indexFstFile.h"
|
||||
#include "indexFstNode.h"
|
||||
#include "indexFstRegistry.h"
|
||||
#include "indexFstUtil.h"
|
||||
|
@ -90,8 +90,8 @@ FstBuilderNode* fstUnFinishedNodesPopEmpty(FstUnFinishedNodes* nodes);
|
|||
uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node, FstSlice bs, Output in, Output* out);
|
||||
|
||||
typedef struct FstBuilder {
|
||||
FstCountingWriter* wrt; // The FST raw data is written directly to `wtr`.
|
||||
FstUnFinishedNodes* unfinished; // The stack of unfinished nodes
|
||||
IdxFstFile* wrt; // The FST raw data is written directly to `wtr`.
|
||||
FstUnFinishedNodes* unfinished; // The stack of unfinished nodes
|
||||
FstRegistry* registry; // A map of finished nodes.
|
||||
FstSlice last; // The last word added
|
||||
CompiledAddr lastAddr; // The address of the last compiled node
|
||||
|
@ -125,9 +125,9 @@ FstState fstStateCreateFrom(FstSlice* data, CompiledAddr addr);
|
|||
FstState fstStateCreate(State state);
|
||||
|
||||
// compile
|
||||
void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp);
|
||||
void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn);
|
||||
void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node);
|
||||
void fstStateCompileForOneTransNext(IdxFstFile* w, CompiledAddr addr, uint8_t inp);
|
||||
void fstStateCompileForOneTrans(IdxFstFile* w, CompiledAddr addr, FstTransition* trn);
|
||||
void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode* node);
|
||||
|
||||
// set_comm_input
|
||||
void fstStateSetCommInput(FstState* state, uint8_t inp);
|
||||
|
@ -282,7 +282,7 @@ FStmSt* stmBuilderIntoStm(FStmBuilder* sb);
|
|||
bool fstVerify(Fst* fst);
|
||||
|
||||
// refactor this function
|
||||
bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
|
||||
bool fstBuilderNodeCompileTo(FstBuilderNode* b, IdxFstFile* wrt, CompiledAddr lastAddr, CompiledAddr startAddr);
|
||||
|
||||
typedef struct StreamState {
|
||||
FstNode* node;
|
||||
|
|
|
@ -1,96 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef __INDEX_FST_COUNTING_WRITER_H__
|
||||
#define __INDEX_FST_COUNTING_WRITER_H__
|
||||
|
||||
#include "indexInt.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//#define USE_MMAP 1
|
||||
|
||||
#define DefaultMem 1024 * 1024
|
||||
|
||||
static char tmpFile[] = "./index";
|
||||
typedef enum WriterType { TMemory, TFile } WriterType;
|
||||
|
||||
typedef struct WriterCtx {
|
||||
int (*write)(struct WriterCtx* ctx, uint8_t* buf, int len);
|
||||
int (*read)(struct WriterCtx* ctx, uint8_t* buf, int len);
|
||||
int (*flush)(struct WriterCtx* ctx);
|
||||
int (*readFrom)(struct WriterCtx* ctx, uint8_t* buf, int len, int32_t offset);
|
||||
int (*size)(struct WriterCtx* ctx);
|
||||
WriterType type;
|
||||
union {
|
||||
struct {
|
||||
TdFilePtr pFile;
|
||||
bool readOnly;
|
||||
char buf[256];
|
||||
int size;
|
||||
#ifdef USE_MMAP
|
||||
char* ptr;
|
||||
#endif
|
||||
} file;
|
||||
struct {
|
||||
int32_t capa;
|
||||
char* buf;
|
||||
} mem;
|
||||
};
|
||||
int32_t offset;
|
||||
int32_t limit;
|
||||
} WriterCtx;
|
||||
|
||||
static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len);
|
||||
static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len);
|
||||
static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset);
|
||||
static int writeCtxDoFlush(WriterCtx* ctx);
|
||||
|
||||
WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity);
|
||||
void writerCtxDestroy(WriterCtx* w, bool remove);
|
||||
|
||||
typedef uint32_t CheckSummer;
|
||||
|
||||
typedef struct FstCountingWriter {
|
||||
void* wrt; // wrap any writer that counts and checksum bytes written
|
||||
uint64_t count;
|
||||
CheckSummer summer;
|
||||
} FstCountingWriter;
|
||||
|
||||
int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len);
|
||||
|
||||
int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len);
|
||||
|
||||
int fstCountingWriterFlush(FstCountingWriter* write);
|
||||
|
||||
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write);
|
||||
|
||||
FstCountingWriter* fstCountingWriterCreate(void* wtr);
|
||||
void fstCountingWriterDestroy(FstCountingWriter* w);
|
||||
|
||||
void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes);
|
||||
uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n);
|
||||
|
||||
#define FST_WRITER_COUNT(writer) (writer->count)
|
||||
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
|
||||
#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
|
@ -0,0 +1,96 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#ifndef __INDEX_FST_FILE_H__
|
||||
#define __INDEX_FST_FILE_H__
|
||||
|
||||
#include "indexInt.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
//#define USE_MMAP 1
|
||||
|
||||
#define DefaultMem 1024 * 1024
|
||||
|
||||
static char tmpFile[] = "./index";
|
||||
typedef enum WriterType { TMemory, TFile } WriterType;
|
||||
|
||||
typedef struct IFileCtx {
|
||||
int (*write)(struct IFileCtx* ctx, uint8_t* buf, int len);
|
||||
int (*read)(struct IFileCtx* ctx, uint8_t* buf, int len);
|
||||
int (*flush)(struct IFileCtx* ctx);
|
||||
int (*readFrom)(struct IFileCtx* ctx, uint8_t* buf, int len, int32_t offset);
|
||||
int (*size)(struct IFileCtx* ctx);
|
||||
WriterType type;
|
||||
union {
|
||||
struct {
|
||||
TdFilePtr pFile;
|
||||
bool readOnly;
|
||||
char buf[256];
|
||||
int64_t size;
|
||||
#ifdef USE_MMAP
|
||||
char* ptr;
|
||||
#endif
|
||||
} file;
|
||||
struct {
|
||||
int32_t cap;
|
||||
char* buf;
|
||||
} mem;
|
||||
};
|
||||
int32_t offset;
|
||||
int32_t limit;
|
||||
} IFileCtx;
|
||||
|
||||
static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len);
|
||||
static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len);
|
||||
static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset);
|
||||
static int idxFileCtxDoFlush(IFileCtx* ctx);
|
||||
|
||||
IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity);
|
||||
void idxFileCtxDestroy(IFileCtx* w, bool remove);
|
||||
|
||||
typedef uint32_t CheckSummer;
|
||||
|
||||
typedef struct IdxFstFile {
|
||||
void* wrt; // wrap any writer that counts and checksum bytes written
|
||||
uint64_t count;
|
||||
CheckSummer summer;
|
||||
} IdxFstFile;
|
||||
|
||||
int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len);
|
||||
|
||||
int idxFileRead(IdxFstFile* write, uint8_t* buf, uint32_t len);
|
||||
|
||||
int idxFileFlush(IdxFstFile* write);
|
||||
|
||||
uint32_t idxFileMaskedCheckSum(IdxFstFile* write);
|
||||
|
||||
IdxFstFile* idxFileCreate(void* wtr);
|
||||
void idxFileDestroy(IdxFstFile* w);
|
||||
|
||||
void idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes);
|
||||
uint8_t idxFilePackUint(IdxFstFile* writer, uint64_t n);
|
||||
|
||||
#define FST_WRITER_COUNT(writer) (writer->count)
|
||||
#define FST_WRITER_INTER_WRITER(writer) (writer->wtr)
|
||||
#define FST_WRITE_CHECK_SUMMER(writer) (writer->summer)
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
|
@ -20,12 +20,12 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
#include "indexFstCountingWriter.h"
|
||||
#include "indexFstFile.h"
|
||||
#include "indexFstUtil.h"
|
||||
#include "indexInt.h"
|
||||
|
||||
#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal)
|
||||
#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0)
|
||||
#define FST_BUILDER_NODE_IS_FINAL(bn) (bn->isFinal)
|
||||
#define FST_BUILDER_NODE_TRANS_ISEMPTY(bn) (taosArrayGetSize(bn->trans) == 0)
|
||||
#define FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn) (bn->finalOutput == 0)
|
||||
|
||||
typedef struct FstTransition {
|
||||
|
@ -46,7 +46,7 @@ FstBuilderNode* fstBuilderNodeClone(FstBuilderNode* src);
|
|||
|
||||
void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src);
|
||||
|
||||
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt,
|
||||
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile' *wrt,
|
||||
// CompiledAddr lastAddr, CompiledAddr startAddr);
|
||||
bool fstBuilderNodeEqual(FstBuilderNode* n1, FstBuilderNode* n2);
|
||||
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
#define __INDEX_TFILE_H__
|
||||
|
||||
#include "indexFst.h"
|
||||
#include "indexFstCountingWriter.h"
|
||||
#include "indexFstFile.h"
|
||||
#include "indexInt.h"
|
||||
#include "indexTfile.h"
|
||||
#include "indexUtil.h"
|
||||
|
@ -59,7 +59,7 @@ typedef struct TFileCache {
|
|||
|
||||
typedef struct TFileWriter {
|
||||
FstBuilder* fb;
|
||||
WriterCtx* ctx;
|
||||
IFileCtx* ctx;
|
||||
TFileHeader header;
|
||||
uint32_t offset;
|
||||
} TFileWriter;
|
||||
|
@ -68,7 +68,7 @@ typedef struct TFileWriter {
|
|||
typedef struct TFileReader {
|
||||
T_REF_DECLARE()
|
||||
Fst* fst;
|
||||
WriterCtx* ctx;
|
||||
IFileCtx* ctx;
|
||||
TFileHeader header;
|
||||
bool remove;
|
||||
} TFileReader;
|
||||
|
@ -103,7 +103,7 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* read
|
|||
TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName);
|
||||
|
||||
TFileReader* tfileReaderOpen(char* path, uint64_t suid, int64_t version, const char* colName);
|
||||
TFileReader* tfileReaderCreate(WriterCtx* ctx);
|
||||
TFileReader* tfileReaderCreate(IFileCtx* ctx);
|
||||
void tfileReaderDestroy(TFileReader* reader);
|
||||
int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr);
|
||||
void tfileReaderRef(TFileReader* reader);
|
||||
|
@ -111,7 +111,7 @@ void tfileReaderUnRef(TFileReader* reader);
|
|||
|
||||
TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const char* colName, uint8_t type);
|
||||
void tfileWriterClose(TFileWriter* tw);
|
||||
TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header);
|
||||
TFileWriter* tfileWriterCreate(IFileCtx* ctx, TFileHeader* header);
|
||||
void tfileWriterDestroy(TFileWriter* tw);
|
||||
int tfileWriterPut(TFileWriter* tw, void* data, bool order);
|
||||
int tfileWriterFinish(TFileWriter* tw);
|
||||
|
|
|
@ -39,7 +39,7 @@
|
|||
#define INDEX_DATA_BIGINT_NULL 0x8000000000000000LL
|
||||
#define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL
|
||||
|
||||
#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
|
||||
#define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN
|
||||
#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN
|
||||
#define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF
|
||||
#define INDEX_DATA_BINARY_NULL 0xFF
|
||||
|
@ -614,7 +614,7 @@ static int idxGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
|
|||
return ret;
|
||||
END:
|
||||
if (tw != NULL) {
|
||||
writerCtxDestroy(tw->ctx, true);
|
||||
idxFileCtxDestroy(tw->ctx, true);
|
||||
taosMemoryFree(tw);
|
||||
}
|
||||
return -1;
|
||||
|
|
|
@ -19,11 +19,11 @@
|
|||
#include "tchecksum.h"
|
||||
#include "tcoding.h"
|
||||
|
||||
static void fstPackDeltaIn(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) {
|
||||
static void fstPackDeltaIn(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) {
|
||||
CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr;
|
||||
fstCountingWriterPackUintIn(wrt, deltaAddr, nBytes);
|
||||
idxFilePackUintIn(wrt, deltaAddr, nBytes);
|
||||
}
|
||||
static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) {
|
||||
static uint8_t fstPackDetla(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) {
|
||||
uint8_t nBytes = packDeltaSize(nodeAddr, transAddr);
|
||||
fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes);
|
||||
return nBytes;
|
||||
|
@ -208,7 +208,7 @@ FstState fstStateCreate(State state) {
|
|||
return fstStateDict[idx];
|
||||
}
|
||||
// compile
|
||||
void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uint8_t inp) {
|
||||
void fstStateCompileForOneTransNext(IdxFstFile* w, CompiledAddr addr, uint8_t inp) {
|
||||
FstState s = fstStateCreate(OneTransNext);
|
||||
fstStateSetCommInput(&s, inp);
|
||||
|
||||
|
@ -216,21 +216,21 @@ void fstStateCompileForOneTransNext(FstCountingWriter* w, CompiledAddr addr, uin
|
|||
uint8_t v = fstStateCommInput(&s, &null);
|
||||
if (null) {
|
||||
// w->write_all(&[inp])
|
||||
fstCountingWriterWrite(w, &inp, 1);
|
||||
idxFileWrite(w, &inp, 1);
|
||||
}
|
||||
fstCountingWriterWrite(w, &(s.val), 1);
|
||||
idxFileWrite(w, &(s.val), 1);
|
||||
// w->write_all(&[s.val])
|
||||
return;
|
||||
}
|
||||
void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTransition* trn) {
|
||||
void fstStateCompileForOneTrans(IdxFstFile* w, CompiledAddr addr, FstTransition* trn) {
|
||||
Output out = trn->out;
|
||||
uint8_t outPackSize = (out == 0 ? 0 : fstCountingWriterPackUint(w, out));
|
||||
uint8_t outPackSize = (out == 0 ? 0 : idxFilePackUint(w, out));
|
||||
uint8_t transPackSize = fstPackDetla(w, addr, trn->addr);
|
||||
PackSizes packSizes = 0;
|
||||
|
||||
FST_SET_OUTPUT_PACK_SIZE(packSizes, outPackSize);
|
||||
FST_SET_TRANSITION_PACK_SIZE(packSizes, transPackSize);
|
||||
fstCountingWriterWrite(w, (char*)&packSizes, sizeof(packSizes));
|
||||
idxFileWrite(w, (char*)&packSizes, sizeof(packSizes));
|
||||
|
||||
FstState st = fstStateCreate(OneTrans);
|
||||
|
||||
|
@ -239,12 +239,12 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran
|
|||
bool null = false;
|
||||
uint8_t inp = fstStateCommInput(&st, &null);
|
||||
if (null == true) {
|
||||
fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp));
|
||||
idxFileWrite(w, (char*)&trn->inp, sizeof(trn->inp));
|
||||
}
|
||||
fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val));
|
||||
idxFileWrite(w, (char*)(&(st.val)), sizeof(st.val));
|
||||
return;
|
||||
}
|
||||
void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuilderNode* node) {
|
||||
void fstStateCompileForAnyTrans(IdxFstFile* w, CompiledAddr addr, FstBuilderNode* node) {
|
||||
int32_t sz = taosArrayGetSize(node->trans);
|
||||
assert(sz <= 256);
|
||||
|
||||
|
@ -275,11 +275,11 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
|
|||
|
||||
if (anyOuts) {
|
||||
if (FST_BUILDER_NODE_IS_FINAL(node)) {
|
||||
fstCountingWriterPackUintIn(w, node->finalOutput, oSize);
|
||||
idxFilePackUintIn(w, node->finalOutput, oSize);
|
||||
}
|
||||
for (int32_t i = sz - 1; i >= 0; i--) {
|
||||
FstTransition* t = taosArrayGet(node->trans, i);
|
||||
fstCountingWriterPackUintIn(w, t->out, oSize);
|
||||
idxFilePackUintIn(w, t->out, oSize);
|
||||
}
|
||||
}
|
||||
for (int32_t i = sz - 1; i >= 0; i--) {
|
||||
|
@ -288,7 +288,7 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
|
|||
}
|
||||
for (int32_t i = sz - 1; i >= 0; i--) {
|
||||
FstTransition* t = taosArrayGet(node->trans, i);
|
||||
fstCountingWriterWrite(w, (char*)&t->inp, 1);
|
||||
idxFileWrite(w, (char*)&t->inp, 1);
|
||||
// fstPackDeltaIn(w, addr, t->addr, tSize);
|
||||
}
|
||||
if (sz > TRANS_INDEX_THRESHOLD) {
|
||||
|
@ -306,10 +306,10 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
|
|||
index[t->inp] = i;
|
||||
// fstPackDeltaIn(w, addr, t->addr, tSize);
|
||||
}
|
||||
fstCountingWriterWrite(w, (char*)index, 256);
|
||||
idxFileWrite(w, (char*)index, 256);
|
||||
taosMemoryFree(index);
|
||||
}
|
||||
fstCountingWriterWrite(w, (char*)&packSizes, 1);
|
||||
idxFileWrite(w, (char*)&packSizes, 1);
|
||||
bool null = false;
|
||||
fstStateStateNtrans(&st, &null);
|
||||
if (null == true) {
|
||||
|
@ -318,12 +318,12 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
|
|||
// encoded in the state byte.
|
||||
uint8_t v = 1;
|
||||
if (sz == 256) {
|
||||
fstCountingWriterWrite(w, (char*)&v, 1);
|
||||
idxFileWrite(w, (char*)&v, 1);
|
||||
} else {
|
||||
fstCountingWriterWrite(w, (char*)&sz, 1);
|
||||
idxFileWrite(w, (char*)&sz, 1);
|
||||
}
|
||||
}
|
||||
fstCountingWriterWrite(w, (char*)(&(st.val)), 1);
|
||||
idxFileWrite(w, (char*)(&(st.val)), 1);
|
||||
return;
|
||||
}
|
||||
|
||||
|
@ -753,7 +753,7 @@ bool fstNodeCompile(FstNode* node, void* w, CompiledAddr lastAddr, CompiledAddr
|
|||
return true;
|
||||
}
|
||||
|
||||
bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) {
|
||||
bool fstBuilderNodeCompileTo(FstBuilderNode* b, IdxFstFile* wrt, CompiledAddr lastAddr, CompiledAddr startAddr) {
|
||||
return fstNodeCompile(NULL, wrt, lastAddr, startAddr, b);
|
||||
}
|
||||
|
||||
|
@ -763,7 +763,7 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) {
|
|||
return b;
|
||||
}
|
||||
|
||||
b->wrt = fstCountingWriterCreate(w);
|
||||
b->wrt = idxFileCreate(w);
|
||||
b->unfinished = fstUnFinishedNodesCreate();
|
||||
b->registry = fstRegistryCreate(10000, 2);
|
||||
b->last = fstSliceCreate(NULL, 0);
|
||||
|
@ -773,12 +773,12 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) {
|
|||
char buf64[8] = {0};
|
||||
void* pBuf64 = buf64;
|
||||
taosEncodeFixedU64(&pBuf64, VERSION);
|
||||
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
|
||||
idxFileWrite(b->wrt, buf64, sizeof(buf64));
|
||||
|
||||
pBuf64 = buf64;
|
||||
memset(buf64, 0, sizeof(buf64));
|
||||
taosEncodeFixedU64(&pBuf64, ty);
|
||||
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
|
||||
idxFileWrite(b->wrt, buf64, sizeof(buf64));
|
||||
|
||||
return b;
|
||||
}
|
||||
|
@ -787,7 +787,7 @@ void fstBuilderDestroy(FstBuilder* b) {
|
|||
return;
|
||||
}
|
||||
|
||||
fstCountingWriterDestroy(b->wrt);
|
||||
idxFileDestroy(b->wrt);
|
||||
fstUnFinishedNodesDestroy(b->unfinished);
|
||||
fstRegistryDestroy(b->registry);
|
||||
fstSliceDestroy(&b->last);
|
||||
|
@ -905,21 +905,19 @@ void* fstBuilderInsertInner(FstBuilder* b) {
|
|||
|
||||
void* pBuf64 = buf64;
|
||||
taosEncodeFixedU64(&pBuf64, b->len);
|
||||
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
|
||||
idxFileWrite(b->wrt, buf64, sizeof(buf64));
|
||||
|
||||
pBuf64 = buf64;
|
||||
taosEncodeFixedU64(&pBuf64, rootAddr);
|
||||
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
|
||||
idxFileWrite(b->wrt, buf64, sizeof(buf64));
|
||||
|
||||
char buf32[4] = {0};
|
||||
void* pBuf32 = buf32;
|
||||
uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt);
|
||||
uint32_t sum = idxFileMaskedCheckSum(b->wrt);
|
||||
taosEncodeFixedU32(&pBuf32, sum);
|
||||
fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32));
|
||||
idxFileWrite(b->wrt, buf32, sizeof(buf32));
|
||||
|
||||
fstCountingWriterFlush(b->wrt);
|
||||
// fstCountingWriterDestroy(b->wrt);
|
||||
// b->wrt = NULL;
|
||||
idxFileFlush(b->wrt);
|
||||
return b->wrt;
|
||||
}
|
||||
void fstBuilderFinish(FstBuilder* b) { fstBuilderInsertInner(b); }
|
||||
|
|
|
@ -61,9 +61,10 @@ void dfaBuilderDestroy(FstDfaBuilder *builder) {
|
|||
pIter = taosHashIterate(builder->cache, pIter);
|
||||
}
|
||||
taosHashCleanup(builder->cache);
|
||||
taosMemoryFree(builder);
|
||||
}
|
||||
|
||||
FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) {
|
||||
FstDfa *dfaBuilder(FstDfaBuilder *builder) {
|
||||
uint32_t sz = taosArrayGetSize(builder->dfa->insts);
|
||||
FstSparseSet *cur = sparSetCreate(sz);
|
||||
FstSparseSet *nxt = sparSetCreate(sz);
|
||||
|
|
|
@ -13,13 +13,13 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "indexFstCountingWriter.h"
|
||||
#include "indexFstFile.h"
|
||||
#include "indexFstUtil.h"
|
||||
#include "indexInt.h"
|
||||
#include "os.h"
|
||||
#include "tutil.h"
|
||||
|
||||
static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
|
||||
static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) {
|
||||
if (ctx->type == TFile) {
|
||||
assert(len == taosWriteFile(ctx->file.pFile, buf, len));
|
||||
} else {
|
||||
|
@ -28,7 +28,7 @@ static int writeCtxDoWrite(WriterCtx* ctx, uint8_t* buf, int len) {
|
|||
ctx->offset += len;
|
||||
return len;
|
||||
}
|
||||
static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
|
||||
static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) {
|
||||
int nRead = 0;
|
||||
if (ctx->type == TFile) {
|
||||
#ifdef USE_MMAP
|
||||
|
@ -44,7 +44,7 @@ static int writeCtxDoRead(WriterCtx* ctx, uint8_t* buf, int len) {
|
|||
|
||||
return nRead;
|
||||
}
|
||||
static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t offset) {
|
||||
static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t offset) {
|
||||
int nRead = 0;
|
||||
if (ctx->type == TFile) {
|
||||
// tfLseek(ctx->file.pFile, offset, 0);
|
||||
|
@ -61,7 +61,7 @@ static int writeCtxDoReadFrom(WriterCtx* ctx, uint8_t* buf, int len, int32_t off
|
|||
}
|
||||
return nRead;
|
||||
}
|
||||
static int writeCtxGetSize(WriterCtx* ctx) {
|
||||
static int idxFileCtxGetSize(IFileCtx* ctx) {
|
||||
if (ctx->type == TFile) {
|
||||
int64_t file_size = 0;
|
||||
taosStatFile(ctx->file.buf, &file_size, NULL);
|
||||
|
@ -69,7 +69,7 @@ static int writeCtxGetSize(WriterCtx* ctx) {
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
static int writeCtxDoFlush(WriterCtx* ctx) {
|
||||
static int idxFileCtxDoFlush(IFileCtx* ctx) {
|
||||
if (ctx->type == TFile) {
|
||||
// taosFsyncFile(ctx->file.pFile);
|
||||
taosFsyncFile(ctx->file.pFile);
|
||||
|
@ -80,8 +80,8 @@ static int writeCtxDoFlush(WriterCtx* ctx) {
|
|||
return 1;
|
||||
}
|
||||
|
||||
WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) {
|
||||
WriterCtx* ctx = taosMemoryCalloc(1, sizeof(WriterCtx));
|
||||
IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int32_t capacity) {
|
||||
IFileCtx* ctx = taosMemoryCalloc(1, sizeof(IFileCtx));
|
||||
if (ctx == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
@ -90,39 +90,36 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
|
|||
if (ctx->type == TFile) {
|
||||
// ugly code, refactor later
|
||||
ctx->file.readOnly = readOnly;
|
||||
memcpy(ctx->file.buf, path, strlen(path));
|
||||
if (readOnly == false) {
|
||||
// ctx->file.pFile = open(path, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO);
|
||||
ctx->file.pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND);
|
||||
taosFtruncateFile(ctx->file.pFile, 0);
|
||||
int64_t file_size;
|
||||
taosStatFile(path, &file_size, NULL);
|
||||
ctx->file.size = (int)file_size;
|
||||
taosStatFile(path, &ctx->file.size, NULL);
|
||||
// ctx->file.size = (int)size;
|
||||
|
||||
} else {
|
||||
// ctx->file.pFile = open(path, O_RDONLY, S_IRWXU | S_IRWXG | S_IRWXO);
|
||||
ctx->file.pFile = taosOpenFile(path, TD_FILE_READ);
|
||||
|
||||
int64_t file_size = 0;
|
||||
taosFStatFile(ctx->file.pFile, &file_size, NULL);
|
||||
ctx->file.size = (int)file_size;
|
||||
int64_t size = 0;
|
||||
taosFStatFile(ctx->file.pFile, &ctx->file.size, NULL);
|
||||
ctx->file.size = (int)size;
|
||||
#ifdef USE_MMAP
|
||||
ctx->file.ptr = (char*)tfMmapReadOnly(ctx->file.pFile, ctx->file.size);
|
||||
#endif
|
||||
}
|
||||
memcpy(ctx->file.buf, path, strlen(path));
|
||||
if (ctx->file.pFile == NULL) {
|
||||
indexError("failed to open file, error %d", errno);
|
||||
goto END;
|
||||
}
|
||||
} else if (ctx->type == TMemory) {
|
||||
ctx->mem.buf = taosMemoryCalloc(1, sizeof(char) * capacity);
|
||||
ctx->mem.capa = capacity;
|
||||
ctx->mem.cap = capacity;
|
||||
}
|
||||
ctx->write = writeCtxDoWrite;
|
||||
ctx->read = writeCtxDoRead;
|
||||
ctx->flush = writeCtxDoFlush;
|
||||
ctx->readFrom = writeCtxDoReadFrom;
|
||||
ctx->size = writeCtxGetSize;
|
||||
ctx->write = idxFileCtxDoWrite;
|
||||
ctx->read = idxFileCtxDoRead;
|
||||
ctx->flush = idxFileCtxDoFlush;
|
||||
ctx->readFrom = idxFileCtxDoReadFrom;
|
||||
ctx->size = idxFileCtxGetSize;
|
||||
|
||||
ctx->offset = 0;
|
||||
ctx->limit = capacity;
|
||||
|
@ -135,7 +132,7 @@ END:
|
|||
taosMemoryFree(ctx);
|
||||
return NULL;
|
||||
}
|
||||
void writerCtxDestroy(WriterCtx* ctx, bool remove) {
|
||||
void idxFileCtxDestroy(IFileCtx* ctx, bool remove) {
|
||||
if (ctx->type == TMemory) {
|
||||
taosMemoryFree(ctx->mem.buf);
|
||||
} else {
|
||||
|
@ -149,9 +146,6 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
|
|||
if (ctx->file.readOnly == false) {
|
||||
int64_t file_size = 0;
|
||||
taosStatFile(ctx->file.buf, &file_size, NULL);
|
||||
// struct stat fstat;
|
||||
// stat(ctx->file.buf, &fstat);
|
||||
// indexError("write file size: %d", (int)(fstat.st_size));
|
||||
}
|
||||
if (remove) {
|
||||
unlink(ctx->file.buf);
|
||||
|
@ -160,30 +154,29 @@ void writerCtxDestroy(WriterCtx* ctx, bool remove) {
|
|||
taosMemoryFree(ctx);
|
||||
}
|
||||
|
||||
FstCountingWriter* fstCountingWriterCreate(void* wrt) {
|
||||
FstCountingWriter* cw = taosMemoryCalloc(1, sizeof(FstCountingWriter));
|
||||
IdxFstFile* idxFileCreate(void* wrt) {
|
||||
IdxFstFile* cw = taosMemoryCalloc(1, sizeof(IdxFstFile));
|
||||
if (cw == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
cw->wrt = wrt;
|
||||
//(void *)(writerCtxCreate(TFile, readOnly));
|
||||
return cw;
|
||||
}
|
||||
void fstCountingWriterDestroy(FstCountingWriter* cw) {
|
||||
void idxFileDestroy(IdxFstFile* cw) {
|
||||
// free wrt object: close fd or free mem
|
||||
fstCountingWriterFlush(cw);
|
||||
// writerCtxDestroy((WriterCtx *)(cw->wrt));
|
||||
idxFileFlush(cw);
|
||||
// idxFileCtxDestroy((IFileCtx *)(cw->wrt));
|
||||
taosMemoryFree(cw);
|
||||
}
|
||||
|
||||
int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
|
||||
int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) {
|
||||
if (write == NULL) {
|
||||
return 0;
|
||||
}
|
||||
// update checksum
|
||||
// write data to file/socket or mem
|
||||
WriterCtx* ctx = write->wrt;
|
||||
IFileCtx* ctx = write->wrt;
|
||||
|
||||
int nWrite = ctx->write(ctx, buf, len);
|
||||
assert(nWrite == len);
|
||||
|
@ -192,42 +185,41 @@ int fstCountingWriterWrite(FstCountingWriter* write, uint8_t* buf, uint32_t len)
|
|||
write->summer = taosCalcChecksum(write->summer, buf, len);
|
||||
return len;
|
||||
}
|
||||
int fstCountingWriterRead(FstCountingWriter* write, uint8_t* buf, uint32_t len) {
|
||||
int idxFileRead(IdxFstFile* write, uint8_t* buf, uint32_t len) {
|
||||
if (write == NULL) {
|
||||
return 0;
|
||||
}
|
||||
WriterCtx* ctx = write->wrt;
|
||||
int nRead = ctx->read(ctx, buf, len);
|
||||
IFileCtx* ctx = write->wrt;
|
||||
int nRead = ctx->read(ctx, buf, len);
|
||||
// assert(nRead == len);
|
||||
return nRead;
|
||||
}
|
||||
|
||||
uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter* write) {
|
||||
uint32_t idxFileMaskedCheckSum(IdxFstFile* write) {
|
||||
// opt
|
||||
return write->summer;
|
||||
}
|
||||
|
||||
int fstCountingWriterFlush(FstCountingWriter* write) {
|
||||
WriterCtx* ctx = write->wrt;
|
||||
int idxFileFlush(IdxFstFile* write) {
|
||||
IFileCtx* ctx = write->wrt;
|
||||
ctx->flush(ctx);
|
||||
// write->wtr->flush
|
||||
return 1;
|
||||
}
|
||||
|
||||
void fstCountingWriterPackUintIn(FstCountingWriter* writer, uint64_t n, uint8_t nBytes) {
|
||||
void idxFilePackUintIn(IdxFstFile* writer, uint64_t n, uint8_t nBytes) {
|
||||
assert(1 <= nBytes && nBytes <= 8);
|
||||
uint8_t* buf = taosMemoryCalloc(8, sizeof(uint8_t));
|
||||
for (uint8_t i = 0; i < nBytes; i++) {
|
||||
buf[i] = (uint8_t)n;
|
||||
n = n >> 8;
|
||||
}
|
||||
fstCountingWriterWrite(writer, buf, nBytes);
|
||||
idxFileWrite(writer, buf, nBytes);
|
||||
taosMemoryFree(buf);
|
||||
return;
|
||||
}
|
||||
|
||||
uint8_t fstCountingWriterPackUint(FstCountingWriter* writer, uint64_t n) {
|
||||
uint8_t idxFilePackUint(IdxFstFile* writer, uint64_t n) {
|
||||
uint8_t nBytes = packSize(n);
|
||||
fstCountingWriterPackUintIn(writer, n, nBytes);
|
||||
idxFilePackUintIn(writer, n, nBytes);
|
||||
return nBytes;
|
||||
}
|
|
@ -95,7 +95,7 @@ void fstBuilderNodeCloneFrom(FstBuilderNode* dst, FstBuilderNode* src) {
|
|||
}
|
||||
}
|
||||
|
||||
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr
|
||||
// bool fstBuilderNodeCompileTo(FstBuilderNode *b, IdxFile *wrt, CompiledAddr lastAddr, CompiledAddr
|
||||
// startAddr) {
|
||||
|
||||
// size_t sz = taosArrayGetSize(b->trans);
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
FstSparseSet *sparSetCreate(int32_t sz) {
|
||||
FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet));
|
||||
if (ss = NULL) {
|
||||
if (ss == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue