Merge branch '3.0' of github.com:taosdata/TDengine into TEST/3.0/TS-4243

This commit is contained in:
Chris Zhai 2024-04-19 13:33:57 +08:00
commit 8c422fac52
296 changed files with 49474 additions and 10722 deletions

View File

@ -454,7 +454,7 @@ pipeline {
cd ${WKC}/tests/parallel_test cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2 export DEFAULT_RETRY_TIME=2
date date
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 600 ''' + extra_param + ''' ''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + '''
''' '''
} }
} }

View File

@ -0,0 +1,15 @@
# xz
if (${TD_LINUX})
ExternalProject_Add(lzma2
GIT_REPOSITORY https://github.com/conor42/fast-lzma2.git
SOURCE_DIR "${TD_CONTRIB_DIR}/lzma2"
#BINARY_DIR ""
BUILD_IN_SOURCE TRUE
CONFIGURE_COMMAND ""
BUILD_COMMAND make
INSTALL_COMMAND ""
TEST_COMMAND ""
)
endif()

View File

@ -0,0 +1,17 @@
# xz
if (${TD_LINUX})
ExternalProject_Add(xz
GIT_REPOSITORY https://github.com/xz-mirror/xz.git
GIT_TAG v5.4.4
SOURCE_DIR "${TD_CONTRIB_DIR}/xz"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
CMAKE_ARGS
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
TEST_COMMAND ""
)
endif()

View File

@ -1,5 +1,8 @@
# zlib # zlib
if (${TD_LINUX})
ExternalProject_Add(zlib ExternalProject_Add(zlib
GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git GIT_REPOSITORY https://github.com/taosdata-contrib/zlib.git
GIT_TAG v1.2.11 GIT_TAG v1.2.11
@ -11,3 +14,4 @@ ExternalProject_Add(zlib
INSTALL_COMMAND "" INSTALL_COMMAND ""
TEST_COMMAND "" TEST_COMMAND ""
) )
endif()

View File

@ -0,0 +1,15 @@
# zstb
#ExternalProject_Add(zstd
#GIT_REPOSITORY https://github.com/facebook/zstd.git
#GIT_TAG v1.5.5
#SOURCE_DIR "${TD_CONTRIB_DIR}/zstd"
#DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#BINARY_DIR ""
#CMAKE_COMMAND
#CMAKE_ARGS ${TD_CONTRIB_DIR}/zstd/build/cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_ARCHIVE_OUTPUT_DIRECTORY=${CMAKE_BINARY_DIR}/build/lib
#BUILD_COMMAND make -j4
#INSTALL_COMMAND ""
#TEST_COMMAND ""
#)

View File

@ -952,7 +952,7 @@ FIRST(expr)
**More explanation**: **More explanation**:
- FIRST(\*) can be used to get the first non-null value of all columns - FIRST(\*) can be used to get the first non-null value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), FIRST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
- NULL will be returned if all the values of the specified column are all NULL - NULL will be returned if all the values of the specified column are all NULL
- A result will NOT be returned if all the columns in the result set are all NULL - A result will NOT be returned if all the columns in the result set are all NULL
@ -1014,7 +1014,7 @@ LAST(expr)
**More explanation**: **More explanation**:
- LAST(\*) can be used to get the last non-NULL value of all columns - LAST(\*) can be used to get the last non-NULL value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
- If the values of a column in the result set are all NULL, NULL is returned for that column; if all columns in the result are all NULL, no result will be returned. - If the values of a column in the result set are all NULL, NULL is returned for that column; if all columns in the result are all NULL, no result will be returned.
- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times. - When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times.
@ -1035,6 +1035,7 @@ LAST_ROW(expr)
**More explanations**: **More explanations**:
- LAST_ROW(\*) can be used to get the last value of all columns; When querying a super table and multiResultFunctionStarReturnTags is set to 0 (default), LAST_ROW(\*) only returns columns of super table; When set to 1, returns columns and tags of the super table.
- When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times. - When it's used on a STable, if there are multiple values with the timestamp in the result set, one of them will be returned randomly and it's not guaranteed that the same value is returned if the same query is run multiple times.
- Can't be used with `INTERVAL`. - Can't be used with `INTERVAL`.

View File

@ -231,6 +231,16 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo
| Default Value | 0 | | Default Value | 0 |
| Notes | When multiple of the above functions act on the same column at the same time and no alias is specified, if the order by clause refers to the column name, column selection ambiguous will occur because the aliases of multiple columns are the same. | | Notes | When multiple of the above functions act on the same column at the same time and no alias is specified, if the order by clause refers to the column name, column selection ambiguous will occur because the aliases of multiple columns are the same. |
### multiResultFunctionStarReturnTags
| Attribute | Description |
| ------------- | --------------------------------------------------------------------------------------------------------------- |
| Applicable | Client only |
| Meaning | When querying a super table, whether last(\*)/last_row(\*)/first(\*) returns tags is affected by this parameter. When querying a normal table or subtable, this parameter has no effect. |
| Value Range | 0: do not return tags, 1: return tags |
| Default Value | 0 |
| Notes | When this parameter is set to 0, last(\*)/last_row(\*)/first(\*) only returns the columns of the super table; When it is 1, return the columns and tags of the super table. |
## Locale Parameters ## Locale Parameters
### timezone ### timezone

View File

@ -954,7 +954,7 @@ FIRST(expr)
**使用说明**: **使用说明**:
- 如果要返回各个列的首个(时间戳最小)非 NULL 值,可以使用 FIRST(\*) - 如果要返回各个列的首个(时间戳最小)非 NULL 值,可以使用 FIRST(\*)查询超级表且multiResultFunctionStarReturnTags设置为 0 (默认值) 时FIRST(\*)只返回超级表的普通列;设置为 1 时,返回超级表的普通列和标签列。
- 如果结果集中的某列全部为 NULL 值,则该列的返回结果也是 NULL - 如果结果集中的某列全部为 NULL 值,则该列的返回结果也是 NULL
- 如果结果集中所有列全部为 NULL 值,则不返回结果。 - 如果结果集中所有列全部为 NULL 值,则不返回结果。
@ -1006,7 +1006,7 @@ LAST(expr)
**使用说明**: **使用说明**:
- 如果要返回各个列的最后(时间戳最大)一个非 NULL 值,可以使用 LAST(\*) - 如果要返回各个列的最后(时间戳最大)一个非 NULL 值,可以使用 LAST(\*)查询超级表且multiResultFunctionStarReturnTags设置为 0 (默认值) 时LAST(\*)只返回超级表的普通列;设置为 1 时,返回超级表的普通列和标签列。
- 如果结果集中的某列全部为 NULL 值,则该列的返回结果也是 NULL如果结果集中所有列全部为 NULL 值,则不返回结果。 - 如果结果集中的某列全部为 NULL 值,则该列的返回结果也是 NULL如果结果集中所有列全部为 NULL 值,则不返回结果。
- 在用于超级表时,时间戳完全一样且同为最大的数据行可能有多个,那么会从中随机返回一条,而并不保证多次运行所挑选的数据行必然一致。 - 在用于超级表时,时间戳完全一样且同为最大的数据行可能有多个,那么会从中随机返回一条,而并不保证多次运行所挑选的数据行必然一致。
@ -1026,7 +1026,7 @@ LAST_ROW(expr)
**适用于**:表和超级表。 **适用于**:表和超级表。
**使用说明** **使用说明**
- 如果要返回各个列的最后一条记录(时间戳最大),可以使用 LAST_ROW(\*)查询超级表且multiResultFunctionStarReturnTags设置为 0 (默认值) 时LAST_ROW(\*)只返回超级表的普通列;设置为 1 时,返回超级表的普通列和标签列。
- 在用于超级表时,时间戳完全一样且同为最大的数据行可能有多个,那么会从中随机返回一条,而并不保证多次运行所挑选的数据行必然一致。 - 在用于超级表时,时间戳完全一样且同为最大的数据行可能有多个,那么会从中随机返回一条,而并不保证多次运行所挑选的数据行必然一致。
- 不能与 INTERVAL 一起使用。 - 不能与 INTERVAL 一起使用。

View File

@ -230,6 +230,16 @@ taos -C
| 缺省值 | 1 | | 缺省值 | 1 |
| 补充说明 | 该参数设置为 1 时,如果查询中含有 GROUP BYPARTITION BY 以及 INTERVAL 子句且相应的组或窗口内数据为空或者NULL 对应的组或窗口将不返回查询结果 | | 补充说明 | 该参数设置为 1 时,如果查询中含有 GROUP BYPARTITION BY 以及 INTERVAL 子句且相应的组或窗口内数据为空或者NULL 对应的组或窗口将不返回查询结果 |
### multiResultFunctionStarReturnTags
| 属性 | 说明 |
| -------- | ---------------------------------------------------------------------------------------------------------------------------------------------- |
| 适用范围 | 仅客户端适用 |
| 含义 | 查询超级表时last(\*)/last_row(\*)/first(\*) 是否返回标签列;查询普通表、子表时,不受该参数影响。 |
| 取值范围 | 0不返回标签列1返回标签列 |
| 缺省值 | 0 |
| 补充说明 | 该参数设置为 0 时last(\*)/last_row(\*)/first(\*) 只返回超级表的普通列;为 1 时,返回超级表的普通列和标签列 |
## 区域相关 ## 区域相关
### timezone ### timezone

View File

@ -33,10 +33,12 @@ extern int32_t tsS3UploadDelaySec;
int32_t s3Init(); int32_t s3Init();
void s3CleanUp(); void s3CleanUp();
int32_t s3CheckCfg();
int32_t s3PutObjectFromFile(const char *file, const char *object); int32_t s3PutObjectFromFile(const char *file, const char *object);
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp); int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp);
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size);
void s3DeleteObjectsByPrefix(const char *prefix); void s3DeleteObjectsByPrefix(const char *prefix);
void s3DeleteObjects(const char *object_name[], int nobject); int32_t s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name); bool s3Exists(const char *object_name);
bool s3Get(const char *object_name, const char *path); bool s3Get(const char *object_name, const char *path);
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock); int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock);
@ -45,6 +47,8 @@ void s3EvictCache(const char *path, long object_size);
long s3Size(const char *object_name); long s3Size(const char *object_name);
int32_t s3GetObjectToFile(const char *object_name, char *fileName); int32_t s3GetObjectToFile(const char *object_name, char *fileName);
#define S3_DATA_CHUNK_PAGES (256 * 1024 * 1024)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -56,6 +56,7 @@ extern "C" {
#define TSDB_INS_TABLE_GRANTS_FULL "ins_grants_full" #define TSDB_INS_TABLE_GRANTS_FULL "ins_grants_full"
#define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs" #define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs"
#define TSDB_INS_TABLE_MACHINES "ins_machines" #define TSDB_INS_TABLE_MACHINES "ins_machines"
#define TSDB_INS_TABLE_TSMAS "ins_tsmas"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas" #define TSDB_PERFS_TABLE_SMAS "perf_smas"

101
include/common/tcol.h Normal file
View File

@ -0,0 +1,101 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "taosdef.h"
#ifndef _TD_TCOL_H_
#define _TD_TCOL_H_
#define TSDB_COLUMN_ENCODE_UNKNOWN "unknown"
#define TSDB_COLUMN_ENCODE_SIMPLE8B "simple8b"
#define TSDB_COLUMN_ENCODE_XOR "delta-i"
#define TSDB_COLUMN_ENCODE_RLE "bit-packing"
#define TSDB_COLUMN_ENCODE_DELTAD "delta-d"
#define TSDB_COLUMN_ENCODE_DISABLED "disabled"
#define TSDB_COLUMN_COMPRESS_UNKNOWN "unknown"
#define TSDB_COLUMN_COMPRESS_LZ4 "lz4"
#define TSDB_COLUMN_COMPRESS_ZLIB "zlib"
#define TSDB_COLUMN_COMPRESS_ZSTD "zstd"
#define TSDB_COLUMN_COMPRESS_TSZ "tsz"
#define TSDB_COLUMN_COMPRESS_XZ "xz"
#define TSDB_COLUMN_COMPRESS_DISABLED "disabled"
#define TSDB_COLUMN_LEVEL_UNKNOWN "unknown"
#define TSDB_COLUMN_LEVEL_HIGH "high"
#define TSDB_COLUMN_LEVEL_MEDIUM "medium"
#define TSDB_COLUMN_LEVEL_LOW "low"
#define TSDB_COLVAL_ENCODE_NOCHANGE 0
#define TSDB_COLVAL_ENCODE_SIMPLE8B 1
#define TSDB_COLVAL_ENCODE_XOR 2
#define TSDB_COLVAL_ENCODE_RLE 3
#define TSDB_COLVAL_ENCODE_DELTAD 4
#define TSDB_COLVAL_ENCODE_DISABLED 0xff
#define TSDB_COLVAL_COMPRESS_NOCHANGE 0
#define TSDB_COLVAL_COMPRESS_LZ4 1
#define TSDB_COLVAL_COMPRESS_ZLIB 2
#define TSDB_COLVAL_COMPRESS_ZSTD 3
#define TSDB_COLVAL_COMPRESS_TSZ 4
#define TSDB_COLVAL_COMPRESS_XZ 5
#define TSDB_COLVAL_COMPRESS_DISABLED 0xff
#define TSDB_COLVAL_LEVEL_NOCHANGE 0
#define TSDB_COLVAL_LEVEL_LOW 1
#define TSDB_COLVAL_LEVEL_MEDIUM 2
#define TSDB_COLVAL_LEVEL_HIGH 3
#define TSDB_COLVAL_LEVEL_DISABLED 0xff
#define TSDB_CL_COMMENT_LEN 1025
#define TSDB_CL_COMPRESS_OPTION_LEN 12
extern const char* supportedEncode[5];
extern const char* supportedCompress[6];
extern const char* supportedLevel[3];
uint8_t getDefaultEncode(uint8_t type);
uint16_t getDefaultCompress(uint8_t type);
uint8_t getDefaultLevel(uint8_t type);
const char* getDefaultEncodeStr(uint8_t type);
const char* getDefaultCompressStr(uint8_t type);
const char* getDefaultLevelStr(uint8_t type);
const char* columnEncodeStr(uint8_t type);
const char* columnCompressStr(uint16_t type);
const char* columnLevelStr(uint8_t type);
uint8_t columnLevelVal(const char* level);
uint8_t columnEncodeVal(const char* encode);
uint16_t columnCompressVal(const char* compress);
bool useCompress(uint8_t tableType);
bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnCompressOrSetDefault(uint8_t type, char compress[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnLevel(char level[TSDB_CL_COMPRESS_OPTION_LEN]);
bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTION_LEN]);
void setColEncode(uint32_t* compress, uint8_t encode);
void setColCompress(uint32_t* compress, uint16_t compressType);
void setColLevel(uint32_t* compress, uint8_t level);
int8_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressType, uint8_t level, bool check,
uint32_t* compress);
int8_t validColCompressLevel(uint8_t type, uint8_t level);
int8_t validColCompress(uint8_t type, uint8_t l2);
int8_t validColEncode(uint8_t type, uint8_t l1);
uint32_t createDefaultColCmprByType(uint8_t type);
bool validColCmprByType(uint8_t type, uint32_t cmpr);
#endif /*_TD_TCOL_H_*/

View File

@ -15,8 +15,7 @@
#ifndef _TD_COMMON_DEF_H_ #ifndef _TD_COMMON_DEF_H_
#define _TD_COMMON_DEF_H_ #define _TD_COMMON_DEF_H_
// #include "taosdef.h"
#include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tmsg.h" #include "tmsg.h"
#include "tvariant.h" #include "tvariant.h"
@ -258,9 +257,9 @@ typedef struct SColumnInfoData {
char* nullbitmap; // bitmap, one bit for each item in the list char* nullbitmap; // bitmap, one bit for each item in the list
SVarColAttr varmeta; SVarColAttr varmeta;
}; };
SColumnInfo info; // column info SColumnInfo info; // column info
bool hasNull; // if current column data has null value. bool hasNull; // if current column data has null value.
bool reassigned; // if current column data is reassigned. bool reassigned; // if current column data is reassigned.
} SColumnInfoData; } SColumnInfoData;
typedef struct SQueryTableDataCond { typedef struct SQueryTableDataCond {
@ -274,13 +273,13 @@ typedef struct SQueryTableDataCond {
STimeWindow twindows; STimeWindow twindows;
int64_t startVersion; int64_t startVersion;
int64_t endVersion; int64_t endVersion;
bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock
} SQueryTableDataCond; } SQueryTableDataCond;
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
void colDataDestroy(SColumnInfoData* pColData); void colDataDestroy(SColumnInfoData* pColData);
//====================================================================================================================== //======================================================================================================================
// the following structure shared by parser and executor // the following structure shared by parser and executor
@ -366,8 +365,8 @@ typedef struct {
float f; float f;
}; };
size_t length; size_t length;
bool keyEscaped; bool keyEscaped;
bool valueEscaped; bool valueEscaped;
} SSmlKv; } SSmlKv;
#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_ASC_FORWARD_STEP 1
@ -389,7 +388,6 @@ typedef struct SNonSortExecInfo {
int32_t blkNums; int32_t blkNums;
} SNonSortExecInfo; } SNonSortExecInfo;
typedef struct STUidTagInfo { typedef struct STUidTagInfo {
char* name; char* name;
uint64_t uid; uint64_t uid;
@ -412,7 +410,19 @@ typedef struct STUidTagInfo {
#define UD_GROUPID_COLUMN_INDEX 1 #define UD_GROUPID_COLUMN_INDEX 1
#define UD_TAG_COLUMN_INDEX 2 #define UD_TAG_COLUMN_INDEX 2
int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime); int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime);
#define TSMA_RES_STB_POSTFIX "_tsma_res_stb_"
#define MD5_OUTPUT_LEN 32
#define TSMA_RES_STB_EXTRA_COLUMN_NUM 4 // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname
static inline bool isTsmaResSTb(const char* stbName) {
const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX);
if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) {
return true;
}
return false;
}
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -37,6 +37,7 @@ typedef struct SBlockOrderInfo {
#define NBIT (3u) #define NBIT (3u)
#define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) #define BitPos(_n) ((_n) & ((1 << NBIT) - 1))
#define CharPos(r_) ((r_) >> NBIT)
#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) #define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT])
#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_)))) #define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
@ -192,12 +193,14 @@ int32_t getJsonValueLen(const char* data);
int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull); int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData); int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue); int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue);
void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows);
int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
uint32_t numOfRows, bool isNull); uint32_t numOfRows, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
const SColumnInfoData* pSource, int32_t numOfRow2); const SColumnInfoData* pSource, int32_t numOfRow2);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
const SDataBlockInfo* pBlockInfo); const SDataBlockInfo* pBlockInfo);
int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, int32_t numOfRows);
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);
int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, bool asc); int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, bool asc);
@ -210,6 +213,8 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock); size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t srcIdx, int32_t numOfRows);
void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows);
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,
int32_t pageSize); int32_t pageSize);
int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
@ -234,7 +239,10 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
void blockDataCleanup(SSDataBlock* pDataBlock); void blockDataCleanup(SSDataBlock* pDataBlock);
void blockDataReset(SSDataBlock* pDataBlock);
void blockDataEmpty(SSDataBlock* pDataBlock); void blockDataEmpty(SSDataBlock* pDataBlock);
int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
bool clearPayload);
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize); size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize);

View File

@ -28,17 +28,18 @@
extern "C" { extern "C" {
#endif #endif
typedef struct SSchema SSchema; typedef struct SSchema SSchema;
typedef struct SSchema2 SSchema2; typedef struct SSchema2 SSchema2;
typedef struct STColumn STColumn; typedef struct SSchemaExt SSchemaExt;
typedef struct STSchema STSchema; typedef struct STColumn STColumn;
typedef struct SValue SValue; typedef struct STSchema STSchema;
typedef struct SColVal SColVal; typedef struct SValue SValue;
typedef struct SRow SRow; typedef struct SColVal SColVal;
typedef struct SRowIter SRowIter; typedef struct SRow SRow;
typedef struct STagVal STagVal; typedef struct SRowIter SRowIter;
typedef struct STag STag; typedef struct STagVal STagVal;
typedef struct SColData SColData; typedef struct STag STag;
typedef struct SColData SColData;
typedef struct SRowKey SRowKey; typedef struct SRowKey SRowKey;
typedef struct SValueColumn SValueColumn; typedef struct SValueColumn SValueColumn;
@ -56,9 +57,9 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111
#define ONE ((uint8_t)1) #define ONE ((uint8_t)1)
#define THREE ((uint8_t)3) #define THREE ((uint8_t)3)
#define DIV_8(i) ((i) >> 3) #define DIV_8(i) ((i) >> 3)
#define MOD_8(i) ((i) & 7) #define MOD_8(i) ((i)&7)
#define DIV_4(i) ((i) >> 2) #define DIV_4(i) ((i) >> 2)
#define MOD_4(i) ((i) & 3) #define MOD_4(i) ((i)&3)
#define MOD_4_TIME_2(i) (MOD_4(i) << 1) #define MOD_4_TIME_2(i) (MOD_4(i) << 1)
#define BIT1_SIZE(n) (DIV_8((n)-1) + 1) #define BIT1_SIZE(n) (DIV_8((n)-1) + 1)
#define BIT2_SIZE(n) (DIV_4((n)-1) + 1) #define BIT2_SIZE(n) (DIV_4((n)-1) + 1)
@ -149,18 +150,18 @@ int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, voi
// SColData ================================ // SColData ================================
typedef struct { typedef struct {
int8_t cmprAlg; // filled by caller uint32_t cmprAlg; // filled by caller
int8_t columnFlag; int8_t columnFlag;
int8_t flag; int8_t flag;
int8_t dataType; int8_t dataType;
int16_t columnId; int16_t columnId;
int32_t numOfData; int32_t numOfData;
int32_t bitmapOriginalSize; int32_t bitmapOriginalSize;
int32_t bitmapCompressedSize; int32_t bitmapCompressedSize;
int32_t offsetOriginalSize; int32_t offsetOriginalSize;
int32_t offsetCompressedSize; int32_t offsetCompressedSize;
int32_t dataOriginalSize; int32_t dataOriginalSize;
int32_t dataCompressedSize; int32_t dataCompressedSize;
} SColDataCompressInfo; } SColDataCompressInfo;
typedef void *(*xMallocFn)(void *, int32_t); typedef void *(*xMallocFn)(void *, int32_t);
@ -339,10 +340,10 @@ struct SValueColumn {
}; };
typedef struct { typedef struct {
int8_t dataType; // filled by caller int32_t dataType; // filled by caller
int8_t cmprAlg; // filled by caller uint32_t cmprAlg; // filled by caller
int32_t originalSize; // filled by caller int32_t originalSize; // filled by caller
int32_t compressedSize; int32_t compressedSize;
} SCompressInfo; } SCompressInfo;
int32_t tCompressData(void *input, // input int32_t tCompressData(void *input, // input

View File

@ -116,9 +116,9 @@ extern int32_t tsMonitorIntervalForBasic;
extern bool tsMonitorForceV2; extern bool tsMonitorForceV2;
// audit // audit
extern bool tsEnableAudit; extern bool tsEnableAudit;
extern bool tsEnableAuditCreateTable; extern bool tsEnableAuditCreateTable;
extern int32_t tsAuditInterval; extern int32_t tsAuditInterval;
// telem // telem
extern bool tsEnableTelem; extern bool tsEnableTelem;
@ -126,9 +126,9 @@ extern int32_t tsTelemInterval;
extern char tsTelemServer[]; extern char tsTelemServer[];
extern uint16_t tsTelemPort; extern uint16_t tsTelemPort;
extern bool tsEnableCrashReport; extern bool tsEnableCrashReport;
extern char * tsTelemUri; extern char *tsTelemUri;
extern char * tsClientCrashReportUri; extern char *tsClientCrashReportUri;
extern char * tsSvrCrashReportUri; extern char *tsSvrCrashReportUri;
// query buffer management // query buffer management
extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing extern int32_t tsQueryBufferSize; // maximum allowed usage buffer size in MB for each data node during query processing
@ -158,6 +158,7 @@ extern int32_t tsMetaCacheMaxSize;
extern int32_t tsSlowLogThreshold; extern int32_t tsSlowLogThreshold;
extern int32_t tsSlowLogScope; extern int32_t tsSlowLogScope;
extern int32_t tsTimeSeriesThreshold; extern int32_t tsTimeSeriesThreshold;
extern bool tsMultiResultFunctionStarReturnTags;
// client // client
extern int32_t tsMinSlidingTime; extern int32_t tsMinSlidingTime;
@ -200,6 +201,8 @@ extern char tsSmlTsDefaultName[];
extern int32_t tmqMaxTopicNum; extern int32_t tmqMaxTopicNum;
extern int32_t tmqRowSize; extern int32_t tmqRowSize;
extern int32_t tsMaxTsmaNum;
extern int32_t tsMaxTsmaCalcDelay;
// wal // wal
extern int64_t tsWalFsyncDataSizeLimit; extern int64_t tsWalFsyncDataSizeLimit;
@ -215,6 +218,8 @@ extern int32_t tsTtlUnit;
extern int32_t tsTtlPushIntervalSec; extern int32_t tsTtlPushIntervalSec;
extern int32_t tsTtlBatchDropNum; extern int32_t tsTtlBatchDropNum;
extern int32_t tsTrimVDbIntervalSec; extern int32_t tsTrimVDbIntervalSec;
extern int32_t tsS3MigrateIntervalSec;
extern bool tsS3MigrateEnabled;
extern int32_t tsGrantHBInterval; extern int32_t tsGrantHBInterval;
extern int32_t tsUptimeInterval; extern int32_t tsUptimeInterval;

View File

@ -20,6 +20,7 @@
#include "taoserror.h" #include "taoserror.h"
#include "tarray.h" #include "tarray.h"
#include "tcoding.h" #include "tcoding.h"
#include "tcol.h"
#include "tencode.h" #include "tencode.h"
#include "thash.h" #include "thash.h"
#include "tlist.h" #include "tlist.h"
@ -110,6 +111,7 @@ enum {
HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_TMQ,
HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_DYN_VIEW,
HEARTBEAT_KEY_VIEWINFO, HEARTBEAT_KEY_VIEWINFO,
HEARTBEAT_KEY_TSMA,
}; };
typedef enum _mgmt_table { typedef enum _mgmt_table {
@ -145,6 +147,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_STREAM_TASKS, TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES, TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_VIEWS, TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_TSMAS,
TSDB_MGMT_TABLE_COMPACT, TSDB_MGMT_TABLE_COMPACT,
TSDB_MGMT_TABLE_COMPACT_DETAIL, TSDB_MGMT_TABLE_COMPACT_DETAIL,
TSDB_MGMT_TABLE_GRANTS_FULL, TSDB_MGMT_TABLE_GRANTS_FULL,
@ -154,18 +157,19 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
} EShowType; } EShowType;
#define TSDB_ALTER_TABLE_ADD_TAG 1 #define TSDB_ALTER_TABLE_ADD_TAG 1
#define TSDB_ALTER_TABLE_DROP_TAG 2 #define TSDB_ALTER_TABLE_DROP_TAG 2
#define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3 #define TSDB_ALTER_TABLE_UPDATE_TAG_NAME 3
#define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4 #define TSDB_ALTER_TABLE_UPDATE_TAG_VAL 4
#define TSDB_ALTER_TABLE_ADD_COLUMN 5 #define TSDB_ALTER_TABLE_ADD_COLUMN 5
#define TSDB_ALTER_TABLE_DROP_COLUMN 6 #define TSDB_ALTER_TABLE_DROP_COLUMN 6
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES 7
#define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8
#define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9 #define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10
#define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11 #define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11
#define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12 #define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12
#define TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS 13
#define TSDB_FILL_NONE 0 #define TSDB_FILL_NONE 0
#define TSDB_FILL_NULL 1 #define TSDB_FILL_NULL 1
@ -245,7 +249,10 @@ typedef enum ENodeType {
QUERY_NODE_EVENT_WINDOW, QUERY_NODE_EVENT_WINDOW,
QUERY_NODE_HINT, QUERY_NODE_HINT,
QUERY_NODE_VIEW, QUERY_NODE_VIEW,
QUERY_NODE_WINDOW_OFFSET,
QUERY_NODE_COUNT_WINDOW, QUERY_NODE_COUNT_WINDOW,
QUERY_NODE_COLUMN_OPTIONS,
QUERY_NODE_TSMA_OPTIONS,
// Statement nodes are used in parser and planner module. // Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100, QUERY_NODE_SET_OPERATOR = 100,
@ -301,7 +308,8 @@ typedef enum ENodeType {
QUERY_NODE_GRANT_STMT, QUERY_NODE_GRANT_STMT,
QUERY_NODE_REVOKE_STMT, QUERY_NODE_REVOKE_STMT,
QUERY_NODE_ALTER_CLUSTER_STMT, QUERY_NODE_ALTER_CLUSTER_STMT,
// placeholder for [153, 180] QUERY_NODE_S3MIGRATE_DATABASE_STMT,
// placeholder for [154, 180]
QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181, QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT, QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT, QUERY_NODE_SHOW_CREATE_TABLE_STMT,
@ -366,6 +374,10 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_GRANTS_FULL_STMT, QUERY_NODE_SHOW_GRANTS_FULL_STMT,
QUERY_NODE_SHOW_GRANTS_LOGS_STMT, QUERY_NODE_SHOW_GRANTS_LOGS_STMT,
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
QUERY_NODE_DROP_TSMA_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000, QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -438,9 +450,9 @@ typedef enum ENodeType {
} ENodeType; } ENodeType;
typedef struct { typedef struct {
int32_t vgId; int32_t vgId;
char* dbFName; const char* dbFName;
char* tbName; const char* tbName;
} SBuildTableInput; } SBuildTableInput;
typedef struct { typedef struct {
@ -457,6 +469,13 @@ typedef struct SField {
int8_t flags; int8_t flags;
int32_t bytes; int32_t bytes;
} SField; } SField;
typedef struct SFieldWithOptions {
char name[TSDB_COL_NAME_LEN];
uint8_t type;
int8_t flags;
int32_t bytes;
uint32_t compress;
} SFieldWithOptions;
typedef struct SRetention { typedef struct SRetention {
int64_t freq; int64_t freq;
@ -535,6 +554,12 @@ struct SSchema {
int32_t bytes; int32_t bytes;
char name[TSDB_COL_NAME_LEN]; char name[TSDB_COL_NAME_LEN];
}; };
struct SSchemaExt {
col_id_t colId;
uint32_t compress;
};
//
struct SSchema2 { struct SSchema2 {
int8_t type; int8_t type;
@ -542,25 +567,26 @@ struct SSchema2 {
col_id_t colId; col_id_t colId;
int32_t bytes; int32_t bytes;
char name[TSDB_COL_NAME_LEN]; char name[TSDB_COL_NAME_LEN];
char alias[TSDB_COL_NAME_LEN]; uint32_t compress;
}; };
typedef struct { typedef struct {
char tbName[TSDB_TABLE_NAME_LEN]; char tbName[TSDB_TABLE_NAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
int64_t dbId; int64_t dbId;
int32_t numOfTags; int32_t numOfTags;
int32_t numOfColumns; int32_t numOfColumns;
int8_t precision; int8_t precision;
int8_t tableType; int8_t tableType;
int32_t sversion; int32_t sversion;
int32_t tversion; int32_t tversion;
uint64_t suid; uint64_t suid;
uint64_t tuid; uint64_t tuid;
int32_t vgId; int32_t vgId;
int8_t sysInfo; int8_t sysInfo;
SSchema* pSchemas; SSchema* pSchemas;
SSchemaExt* pSchemaExt;
} STableMetaRsp; } STableMetaRsp;
typedef struct { typedef struct {
@ -586,7 +612,7 @@ typedef struct {
// int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); // int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
// int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); // int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
// void tFreeSSubmitBlkRsp(void* param); // void tFreeSSubmitBlkRsp(void* param);
void tFreeSSubmitRsp(SSubmitRsp* pRsp); void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define COL_SMA_ON ((int8_t)0x1) #define COL_SMA_ON ((int8_t)0x1)
#define COL_IDX_ON ((int8_t)0x2) #define COL_IDX_ON ((int8_t)0x2)
@ -624,6 +650,54 @@ typedef struct {
SSchema* pSchema; SSchema* pSchema;
} SSchemaWrapper; } SSchemaWrapper;
typedef struct {
col_id_t id;
uint32_t alg;
} SColCmpr;
typedef struct {
int32_t nCols;
int32_t version;
SColCmpr* pColCmpr;
} SColCmprWrapper;
static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper* pSrcWrapper) {
if (pSrcWrapper->pColCmpr == NULL || pSrcWrapper->nCols == 0) return NULL;
SColCmprWrapper* pDstWrapper = (SColCmprWrapper*)taosMemoryMalloc(sizeof(SColCmprWrapper));
pDstWrapper->nCols = pSrcWrapper->nCols;
pDstWrapper->version = pSrcWrapper->version;
int32_t size = sizeof(SColCmpr) * pDstWrapper->nCols;
pDstWrapper->pColCmpr = (SColCmpr*)taosMemoryCalloc(1, size);
memcpy(pDstWrapper->pColCmpr, pSrcWrapper->pColCmpr, size);
return pDstWrapper;
}
static FORCE_INLINE void tInitDefaultSColCmprWrapperByCols(SColCmprWrapper* pCmpr, int32_t nCols) {
assert(!pCmpr->pColCmpr);
pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(nCols, sizeof(SColCmpr));
pCmpr->nCols = nCols;
}
static FORCE_INLINE void tInitDefaultSColCmprWrapper(SColCmprWrapper* pCmpr, SSchemaWrapper* pSchema) {
pCmpr->nCols = pSchema->nCols;
assert(!pCmpr->pColCmpr);
pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(pCmpr->nCols, sizeof(SColCmpr));
for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr* pColCmpr = &pCmpr->pColCmpr[i];
SSchema* pColSchema = &pSchema->pSchema[i];
pColCmpr->id = pColSchema->colId;
pColCmpr->alg = 0;
}
}
static FORCE_INLINE void tDeleteSColCmprWrapper(SColCmprWrapper* pWrapper) {
if (pWrapper == NULL) return;
taosMemoryFreeClear(pWrapper->pColCmpr);
taosMemoryFreeClear(pWrapper);
}
static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) { static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* pSchemaWrapper) {
if (pSchemaWrapper->pSchema == NULL) return NULL; if (pSchemaWrapper->pSchema == NULL) return NULL;
@ -692,6 +766,18 @@ static FORCE_INLINE int32_t tDecodeSSchema(SDecoder* pDecoder, SSchema* pSchema)
return 0; return 0;
} }
static FORCE_INLINE int32_t tEncodeSSchemaExt(SEncoder* pEncoder, const SSchemaExt* pSchemaExt) {
if (tEncodeI16v(pEncoder, pSchemaExt->colId) < 0) return -1;
if (tEncodeU32(pEncoder, pSchemaExt->compress) < 0) return -1;
return 0;
}
static FORCE_INLINE int32_t tDecodeSSchemaExt(SDecoder* pDecoder, SSchemaExt* pSchemaExt) {
if (tDecodeI16v(pDecoder, &pSchemaExt->colId) < 0) return -1;
if (tDecodeU32(pDecoder, &pSchemaExt->compress) < 0) return -1;
return 0;
}
static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { static FORCE_INLINE int32_t taosEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) {
int32_t tlen = 0; int32_t tlen = 0;
tlen += taosEncodeVariantI32(buf, pSW->nCols); tlen += taosEncodeVariantI32(buf, pSW->nCols);
@ -775,7 +861,7 @@ typedef struct {
int32_t commentLen; int32_t commentLen;
int32_t ast1Len; int32_t ast1Len;
int32_t ast2Len; int32_t ast2Len;
SArray* pColumns; // array of SField SArray* pColumns; // array of SFieldWithOptions
SArray* pTags; // array of SField SArray* pTags; // array of SField
SArray* pFuncs; SArray* pFuncs;
char* pComment; char* pComment;
@ -1092,23 +1178,24 @@ typedef struct {
} STableCfgReq; } STableCfgReq;
typedef struct { typedef struct {
char tbName[TSDB_TABLE_NAME_LEN]; char tbName[TSDB_TABLE_NAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
int32_t numOfTags; int32_t numOfTags;
int32_t numOfColumns; int32_t numOfColumns;
int8_t tableType; int8_t tableType;
int64_t delay1; int64_t delay1;
int64_t delay2; int64_t delay2;
int64_t watermark1; int64_t watermark1;
int64_t watermark2; int64_t watermark2;
int32_t ttl; int32_t ttl;
SArray* pFuncs; SArray* pFuncs;
int32_t commentLen; int32_t commentLen;
char* pComment; char* pComment;
SSchema* pSchemas; SSchema* pSchemas;
int32_t tagsLen; int32_t tagsLen;
char* pTags; char* pTags;
SSchemaExt* pSchemaExt;
} STableCfg; } STableCfg;
typedef STableCfg STableCfgRsp; typedef STableCfg STableCfgRsp;
@ -1153,6 +1240,9 @@ typedef struct {
int32_t sstTrigger; int32_t sstTrigger;
int16_t hashPrefix; int16_t hashPrefix;
int16_t hashSuffix; int16_t hashSuffix;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
@ -1183,6 +1273,8 @@ typedef struct {
int32_t minRows; int32_t minRows;
int32_t walRetentionPeriod; int32_t walRetentionPeriod;
int32_t walRetentionSize; int32_t walRetentionSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
int8_t withArbitrator; int8_t withArbitrator;
@ -1263,6 +1355,20 @@ typedef struct {
int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); int32_t tSerializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq); int32_t tDeserializeSVTrimDbReq(void* buf, int32_t bufLen, SVTrimDbReq* pReq);
typedef struct {
char db[TSDB_DB_FNAME_LEN];
} SS3MigrateDbReq;
int32_t tSerializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
int32_t tDeserializeSS3MigrateDbReq(void* buf, int32_t bufLen, SS3MigrateDbReq* pReq);
typedef struct {
int32_t timestamp;
} SVS3MigrateDbReq;
int32_t tSerializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
int32_t tDeserializeSVS3MigrateDbReq(void* buf, int32_t bufLen, SVS3MigrateDbReq* pReq);
typedef struct { typedef struct {
int32_t timestampSec; int32_t timestampSec;
int32_t ttlDropMaxCount; int32_t ttlDropMaxCount;
@ -1299,6 +1405,9 @@ typedef struct {
int8_t replications; int8_t replications;
int8_t strict; int8_t strict;
int8_t cacheLast; int8_t cacheLast;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int32_t walRetentionPeriod; int32_t walRetentionPeriod;
int32_t walRollPeriod; int32_t walRollPeriod;
@ -1373,8 +1482,14 @@ int32_t tDeserializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp
void tFreeSDnodeListRsp(SDnodeListRsp* pRsp); void tFreeSDnodeListRsp(SDnodeListRsp* pRsp);
typedef struct { typedef struct {
SUseDbRsp* useDbRsp; SArray* pTsmas; // SArray<STableTSMAInfo*>
SDbCfgRsp* cfgRsp; } STableTSMAInfoRsp;
typedef struct {
SUseDbRsp* useDbRsp;
SDbCfgRsp* cfgRsp;
STableTSMAInfoRsp* pTsmaRsp;
int32_t dbTsmaVersion;
} SDbHbRsp; } SDbHbRsp;
typedef struct { typedef struct {
@ -1589,13 +1704,13 @@ int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
void tFreeSStatusReq(SStatusReq* pReq); void tFreeSStatusReq(SStatusReq* pReq);
typedef struct { typedef struct {
int32_t contLen; int32_t contLen;
char* pCont; char* pCont;
} SStatisReq; } SStatisReq;
int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); int32_t tSerializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq); int32_t tDeserializeSStatisReq(void* buf, int32_t bufLen, SStatisReq* pReq);
void tFreeSStatisReq(SStatisReq *pReq); void tFreeSStatisReq(SStatisReq* pReq);
typedef struct { typedef struct {
int32_t dnodeId; int32_t dnodeId;
@ -1700,7 +1815,10 @@ typedef struct {
int16_t hashPrefix; int16_t hashPrefix;
int16_t hashSuffix; int16_t hashSuffix;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int64_t reserved[8]; int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int64_t reserved[6];
int8_t learnerReplica; int8_t learnerReplica;
int8_t learnerSelfIndex; int8_t learnerSelfIndex;
SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA];
@ -1788,13 +1906,15 @@ typedef struct {
int8_t walLevel; int8_t walLevel;
int8_t strict; int8_t strict;
int8_t cacheLast; int8_t cacheLast;
int64_t reserved[8]; int64_t reserved[7];
// 1st modification // 1st modification
int16_t sttTrigger; int16_t sttTrigger;
int32_t minRows; int32_t minRows;
// 2nd modification // 2nd modification
int32_t walRetentionPeriod; int32_t walRetentionPeriod;
int32_t walRetentionSize; int32_t walRetentionSize;
int32_t s3KeepLocal;
int8_t s3Compact;
} SAlterVnodeConfigReq; } SAlterVnodeConfigReq;
int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq); int32_t tSerializeSAlterVnodeConfigReq(void* buf, int32_t bufLen, SAlterVnodeConfigReq* pReq);
@ -1953,7 +2073,7 @@ typedef struct {
int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
// int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); // int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq);
void tFreeSShowReq(SShowReq* pReq); void tFreeSShowReq(SShowReq* pReq);
typedef struct { typedef struct {
int64_t showId; int64_t showId;
@ -2568,6 +2688,7 @@ typedef struct {
SArray* pVgroupVerList; SArray* pVgroupVerList;
// 3.3.0.0 // 3.3.0.0
SArray* pCols; // array of SField SArray* pCols; // array of SField
int64_t smaId;
} SCMCreateStreamReq; } SCMCreateStreamReq;
typedef struct { typedef struct {
@ -2759,15 +2880,17 @@ int32_t tDecodeSRSmaParam(SDecoder* pCoder, SRSmaParam* pRSmaParam);
// TDMT_VND_CREATE_STB ============== // TDMT_VND_CREATE_STB ==============
typedef struct SVCreateStbReq { typedef struct SVCreateStbReq {
char* name; char* name;
tb_uid_t suid; tb_uid_t suid;
int8_t rollup; int8_t rollup;
SSchemaWrapper schemaRow; SSchemaWrapper schemaRow;
SSchemaWrapper schemaTag; SSchemaWrapper schemaTag;
SRSmaParam rsmaParam; SRSmaParam rsmaParam;
int32_t alterOriDataLen; int32_t alterOriDataLen;
void* alterOriData; void* alterOriData;
int8_t source; int8_t source;
int8_t colCmpred;
SColCmprWrapper colCmpr;
} SVCreateStbReq; } SVCreateStbReq;
int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq); int tEncodeSVCreateStbReq(SEncoder* pCoder, const SVCreateStbReq* pReq);
@ -2805,8 +2928,9 @@ typedef struct SVCreateTbReq {
SSchemaWrapper schemaRow; SSchemaWrapper schemaRow;
} ntb; } ntb;
}; };
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
SColCmprWrapper colCmpr;
} SVCreateTbReq; } SVCreateTbReq;
int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq);
@ -2829,6 +2953,7 @@ static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) {
} else if (req->type == TSDB_NORMAL_TABLE) { } else if (req->type == TSDB_NORMAL_TABLE) {
taosMemoryFreeClear(req->ntb.schemaRow.pSchema); taosMemoryFreeClear(req->ntb.schemaRow.pSchema);
} }
taosMemoryFreeClear(req->colCmpr.pColCmpr);
} }
typedef struct { typedef struct {
@ -2837,7 +2962,7 @@ typedef struct {
SVCreateTbReq* pReqs; SVCreateTbReq* pReqs;
SArray* pArray; SArray* pArray;
}; };
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
} SVCreateTbBatchReq; } SVCreateTbBatchReq;
int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq); int tEncodeSVCreateTbBatchReq(SEncoder* pCoder, const SVCreateTbBatchReq* pReq);
@ -2914,12 +3039,10 @@ typedef struct {
int32_t bytes; int32_t bytes;
// TSDB_ALTER_TABLE_DROP_COLUMN // TSDB_ALTER_TABLE_DROP_COLUMN
// TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES // TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES
int8_t colModType; int8_t colModType;
int32_t colModBytes; int32_t colModBytes;
// TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME char* colNewName; // TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME
char* colNewName; char* tagName; // TSDB_ALTER_TABLE_UPDATE_TAG_VAL
// TSDB_ALTER_TABLE_UPDATE_TAG_VAL
char* tagName;
int8_t isNull; int8_t isNull;
int8_t tagType; int8_t tagType;
int8_t tagFree; int8_t tagFree;
@ -2927,12 +3050,13 @@ typedef struct {
uint8_t* pTagVal; uint8_t* pTagVal;
SArray* pTagArray; SArray* pTagArray;
// TSDB_ALTER_TABLE_UPDATE_OPTIONS // TSDB_ALTER_TABLE_UPDATE_OPTIONS
int8_t updateTTL; int8_t updateTTL;
int32_t newTTL; int32_t newTTL;
int32_t newCommentLen; int32_t newCommentLen;
char* newComment; char* newComment;
int64_t ctimeMs; // fill by vnode int64_t ctimeMs; // fill by vnode
int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient int8_t source; // TD_REQ_FROM_TAOX-taosX or TD_REQ_FROM_APP-taosClient
uint32_t compress; // TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS
} SVAlterTbReq; } SVAlterTbReq;
int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq);
@ -3532,6 +3656,10 @@ typedef struct {
char* ast; char* ast;
int64_t deleteMark; int64_t deleteMark;
int64_t lastTs; int64_t lastTs;
int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0
SArray* pVgroupVerList;
int8_t recursiveTsma;
char baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma
} SMCreateSmaReq; } SMCreateSmaReq;
int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq); int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq);
@ -4051,7 +4179,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
#define TD_REQ_FROM_TAOX 0x8 #define TD_REQ_FROM_TAOX 0x8
#define SUBMIT_REQUEST_VERSION (1) #define SUBMIT_REQUEST_VERSION (1)
#define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility #define TD_REQ_FROM_TAOX_OLD 0x1 // for compatibility
typedef struct { typedef struct {
int32_t flags; int32_t flags;
@ -4155,6 +4283,107 @@ typedef struct {
int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp); int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp);
int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp); int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
void tFreeSViewMetaRsp(SViewMetaRsp* pRsp); void tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name
bool fetchingWithTsmaName; // if we are fetching with tsma name
}STableTSMAInfoReq;
int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq);
int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq);
typedef struct {
int32_t funcId;
col_id_t colId;
} STableTSMAFuncInfo;
typedef struct {
char name[TSDB_TABLE_NAME_LEN];
uint64_t tsmaId;
char targetTb[TSDB_TABLE_NAME_LEN];
char targetDbFName[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t suid;
uint64_t destTbUid;
uint64_t dbId;
int32_t version;
int64_t interval;
int8_t unit;
SArray* pFuncs; // SArray<STableTSMAFuncInfo>
SArray* pTags; // SArray<SSchema>
SArray* pUsedCols; // SArray<SSchema>
char* ast;
int64_t streamUid;
int64_t reqTs;
int64_t rspTs;
int64_t delayDuration; // ms
bool fillHistoryFinished;
} STableTSMAInfo;
int32_t tSerializeTableTSMAInfoRsp(void* buf, int32_t bufLen, const STableTSMAInfoRsp* pRsp);
int32_t tDeserializeTableTSMAInfoRsp(void* buf, int32_t bufLen, STableTSMAInfoRsp* pRsp);
int32_t tCloneTbTSMAInfo(STableTSMAInfo* pInfo, STableTSMAInfo** pRes);
void tFreeTableTSMAInfo(void* p);
void tFreeAndClearTableTSMAInfo(void* p);
void tFreeTableTSMAInfoRsp(STableTSMAInfoRsp* pRsp);
#define STSMAHbRsp STableTSMAInfoRsp
#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp
#define tDeserializeTSMAHbRsp tDeserializeTableTSMAInfoRsp
#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp
typedef struct SStreamProgressReq {
int64_t streamId;
int32_t vgId;
int32_t fetchIdx;
int32_t subFetchIdx;
} SStreamProgressReq;
int32_t tSerializeStreamProgressReq(void* buf, int32_t bufLen, const SStreamProgressReq* pReq);
int32_t tDeserializeStreamProgressReq(void* buf, int32_t bufLen, SStreamProgressReq* pReq);
typedef struct SStreamProgressRsp {
int64_t streamId;
int32_t vgId;
bool fillHisFinished;
int64_t progressDelay;
int32_t fetchIdx;
int32_t subFetchIdx;
} SStreamProgressRsp;
int32_t tSerializeStreamProgressRsp(void* buf, int32_t bufLen, const SStreamProgressRsp* pRsp);
int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgressRsp* pRsp);
typedef struct SDropCtbWithTsmaSingleVgReq {
SVgroupInfo vgInfo;
SArray* pTbs; // SVDropTbReq
} SMDropTbReqsOnSingleVg;
int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder* pEncoder, const SMDropTbReqsOnSingleVg* pReq);
int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg* pReq);
void tFreeSMDropTbReqOnSingleVg(void* p);
typedef struct SDropTbsReq {
SArray* pVgReqs; // SMDropTbReqsOnSingleVg
} SMDropTbsReq;
int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq);
int32_t tDeserializeSMDropTbsReq(void* buf, int32_t bufLen, SMDropTbsReq* pReq);
void tFreeSMDropTbsReq(void*);
typedef struct SVFetchTtlExpiredTbsRsp {
SArray* pExpiredTbs; // SVDropTbReq
int32_t vgId;
} SVFetchTtlExpiredTbsRsp;
int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder* pCoder, const SVFetchTtlExpiredTbsRsp* pRsp);
int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder* pCoder, SVFetchTtlExpiredTbsRsp* pRsp);
void tFreeFetchTtlExpiredTbsRsp(void* p);
void setDefaultOptionsForField(SFieldWithOptions* field);
void setFieldWithOptions(SFieldWithOptions* fieldWithOptions, SField* field);
#pragma pack(pop) #pragma pack(pop)

View File

@ -220,7 +220,15 @@
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TSMA, "create-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TSMA, "drop-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STB_DROP, "drop-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_TSMA, "get-table-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_TSMA, "get-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TB_WITH_TSMA, "drop-tb-with-tsma", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG) TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8 TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8
@ -265,6 +273,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
@ -272,6 +281,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_QUERY_COMPACT_PROGRESS, "vnode-query-compact-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_KILL_COMPACT, "kill-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_S3MIGRATE, "vnode-s3migrate", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ARB_HEARTBEAT, "vnode-arb-hb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ARB_CHECK_SYNC, "vnode-arb-check-sync", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL)
@ -307,6 +317,8 @@
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESUME, "stream-task-resume", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESUME, "stream-task-resume", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG) TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8 TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8
@ -354,6 +366,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8 TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8

View File

@ -79,7 +79,7 @@ int64_t taosTimeGetIntervalEnd(int64_t ts, const SInterval* pInterval);
int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, int32_t order); int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, int32_t order);
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision); int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision); int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow);
int32_t taosParseTime(const char* timestr, int64_t* pTime, int32_t len, int32_t timePrec, int8_t dayligth); int32_t taosParseTime(const char* timestr, int64_t* pTime, int32_t len, int32_t timePrec, int8_t dayligth);
void deltaToUtcInitOnce(); void deltaToUtcInitOnce();
@ -88,6 +88,7 @@ char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision); int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit); int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal); int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision); void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);

View File

@ -16,374 +16,393 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_NK_COMMA 33 #define TK_NK_COMMA 33
#define TK_HOST 34 #define TK_HOST 34
#define TK_USER 35 #define TK_USER 35
#define TK_ENABLE 36 #define TK_ENABLE 36
#define TK_NK_INTEGER 37 #define TK_NK_INTEGER 37
#define TK_SYSINFO 38 #define TK_SYSINFO 38
#define TK_ADD 39 #define TK_ADD 39
#define TK_DROP 40 #define TK_DROP 40
#define TK_GRANT 41 #define TK_GRANT 41
#define TK_ON 42 #define TK_ON 42
#define TK_TO 43 #define TK_TO 43
#define TK_REVOKE 44 #define TK_REVOKE 44
#define TK_FROM 45 #define TK_FROM 45
#define TK_SUBSCRIBE 46 #define TK_SUBSCRIBE 46
#define TK_READ 47 #define TK_READ 47
#define TK_WRITE 48 #define TK_WRITE 48
#define TK_NK_DOT 49 #define TK_NK_DOT 49
#define TK_WITH 50 #define TK_WITH 50
#define TK_DNODE 51 #define TK_DNODE 51
#define TK_PORT 52 #define TK_PORT 52
#define TK_DNODES 53 #define TK_DNODES 53
#define TK_RESTORE 54 #define TK_RESTORE 54
#define TK_NK_IPTOKEN 55 #define TK_NK_IPTOKEN 55
#define TK_FORCE 56 #define TK_FORCE 56
#define TK_UNSAFE 57 #define TK_UNSAFE 57
#define TK_CLUSTER 58 #define TK_CLUSTER 58
#define TK_LOCAL 59 #define TK_LOCAL 59
#define TK_QNODE 60 #define TK_QNODE 60
#define TK_BNODE 61 #define TK_BNODE 61
#define TK_SNODE 62 #define TK_SNODE 62
#define TK_MNODE 63 #define TK_MNODE 63
#define TK_VNODE 64 #define TK_VNODE 64
#define TK_DATABASE 65 #define TK_DATABASE 65
#define TK_USE 66 #define TK_USE 66
#define TK_FLUSH 67 #define TK_FLUSH 67
#define TK_TRIM 68 #define TK_TRIM 68
#define TK_COMPACT 69 #define TK_S3MIGRATE 69
#define TK_IF 70 #define TK_COMPACT 70
#define TK_NOT 71 #define TK_IF 71
#define TK_EXISTS 72 #define TK_NOT 72
#define TK_BUFFER 73 #define TK_EXISTS 73
#define TK_CACHEMODEL 74 #define TK_BUFFER 74
#define TK_CACHESIZE 75 #define TK_CACHEMODEL 75
#define TK_COMP 76 #define TK_CACHESIZE 76
#define TK_DURATION 77 #define TK_COMP 77
#define TK_NK_VARIABLE 78 #define TK_DURATION 78
#define TK_MAXROWS 79 #define TK_NK_VARIABLE 79
#define TK_MINROWS 80 #define TK_MAXROWS 80
#define TK_KEEP 81 #define TK_MINROWS 81
#define TK_PAGES 82 #define TK_KEEP 82
#define TK_PAGESIZE 83 #define TK_PAGES 83
#define TK_TSDB_PAGESIZE 84 #define TK_PAGESIZE 84
#define TK_PRECISION 85 #define TK_TSDB_PAGESIZE 85
#define TK_REPLICA 86 #define TK_PRECISION 86
#define TK_VGROUPS 87 #define TK_REPLICA 87
#define TK_SINGLE_STABLE 88 #define TK_VGROUPS 88
#define TK_RETENTIONS 89 #define TK_SINGLE_STABLE 89
#define TK_SCHEMALESS 90 #define TK_RETENTIONS 90
#define TK_WAL_LEVEL 91 #define TK_SCHEMALESS 91
#define TK_WAL_FSYNC_PERIOD 92 #define TK_WAL_LEVEL 92
#define TK_WAL_RETENTION_PERIOD 93 #define TK_WAL_FSYNC_PERIOD 93
#define TK_WAL_RETENTION_SIZE 94 #define TK_WAL_RETENTION_PERIOD 94
#define TK_WAL_ROLL_PERIOD 95 #define TK_WAL_RETENTION_SIZE 95
#define TK_WAL_SEGMENT_SIZE 96 #define TK_WAL_ROLL_PERIOD 96
#define TK_STT_TRIGGER 97 #define TK_WAL_SEGMENT_SIZE 97
#define TK_TABLE_PREFIX 98 #define TK_STT_TRIGGER 98
#define TK_TABLE_SUFFIX 99 #define TK_TABLE_PREFIX 99
#define TK_KEEP_TIME_OFFSET 100 #define TK_TABLE_SUFFIX 100
#define TK_NK_COLON 101 #define TK_S3_CHUNKSIZE 101
#define TK_BWLIMIT 102 #define TK_S3_KEEPLOCAL 102
#define TK_START 103 #define TK_S3_COMPACT 103
#define TK_TIMESTAMP 104 #define TK_KEEP_TIME_OFFSET 104
#define TK_END 105 #define TK_NK_COLON 105
#define TK_TABLE 106 #define TK_BWLIMIT 106
#define TK_NK_LP 107 #define TK_START 107
#define TK_NK_RP 108 #define TK_TIMESTAMP 108
#define TK_STABLE 109 #define TK_END 109
#define TK_COLUMN 110 #define TK_TABLE 110
#define TK_MODIFY 111 #define TK_NK_LP 111
#define TK_RENAME 112 #define TK_NK_RP 112
#define TK_TAG 113 #define TK_STABLE 113
#define TK_SET 114 #define TK_COLUMN 114
#define TK_NK_EQ 115 #define TK_MODIFY 115
#define TK_USING 116 #define TK_RENAME 116
#define TK_TAGS 117 #define TK_TAG 117
#define TK_PRIMARY 118 #define TK_SET 118
#define TK_KEY 119 #define TK_NK_EQ 119
#define TK_BOOL 120 #define TK_USING 120
#define TK_TINYINT 121 #define TK_TAGS 121
#define TK_SMALLINT 122 #define TK_BOOL 122
#define TK_INT 123 #define TK_TINYINT 123
#define TK_INTEGER 124 #define TK_SMALLINT 124
#define TK_BIGINT 125 #define TK_INT 125
#define TK_FLOAT 126 #define TK_INTEGER 126
#define TK_DOUBLE 127 #define TK_BIGINT 127
#define TK_BINARY 128 #define TK_FLOAT 128
#define TK_NCHAR 129 #define TK_DOUBLE 129
#define TK_UNSIGNED 130 #define TK_BINARY 130
#define TK_JSON 131 #define TK_NCHAR 131
#define TK_VARCHAR 132 #define TK_UNSIGNED 132
#define TK_MEDIUMBLOB 133 #define TK_JSON 133
#define TK_BLOB 134 #define TK_VARCHAR 134
#define TK_VARBINARY 135 #define TK_MEDIUMBLOB 135
#define TK_GEOMETRY 136 #define TK_BLOB 136
#define TK_DECIMAL 137 #define TK_VARBINARY 137
#define TK_COMMENT 138 #define TK_GEOMETRY 138
#define TK_MAX_DELAY 139 #define TK_DECIMAL 139
#define TK_WATERMARK 140 #define TK_COMMENT 140
#define TK_ROLLUP 141 #define TK_MAX_DELAY 141
#define TK_TTL 142 #define TK_WATERMARK 142
#define TK_SMA 143 #define TK_ROLLUP 143
#define TK_DELETE_MARK 144 #define TK_TTL 144
#define TK_FIRST 145 #define TK_SMA 145
#define TK_LAST 146 #define TK_DELETE_MARK 146
#define TK_SHOW 147 #define TK_FIRST 147
#define TK_PRIVILEGES 148 #define TK_LAST 148
#define TK_DATABASES 149 #define TK_SHOW 149
#define TK_TABLES 150 #define TK_PRIVILEGES 150
#define TK_STABLES 151 #define TK_DATABASES 151
#define TK_MNODES 152 #define TK_TABLES 152
#define TK_QNODES 153 #define TK_STABLES 153
#define TK_ARBGROUPS 154 #define TK_MNODES 154
#define TK_FUNCTIONS 155 #define TK_QNODES 155
#define TK_INDEXES 156 #define TK_ARBGROUPS 156
#define TK_ACCOUNTS 157 #define TK_FUNCTIONS 157
#define TK_APPS 158 #define TK_INDEXES 158
#define TK_CONNECTIONS 159 #define TK_ACCOUNTS 159
#define TK_LICENCES 160 #define TK_APPS 160
#define TK_GRANTS 161 #define TK_CONNECTIONS 161
#define TK_FULL 162 #define TK_LICENCES 162
#define TK_LOGS 163 #define TK_GRANTS 163
#define TK_MACHINES 164 #define TK_FULL 164
#define TK_QUERIES 165 #define TK_LOGS 165
#define TK_SCORES 166 #define TK_MACHINES 166
#define TK_TOPICS 167 #define TK_QUERIES 167
#define TK_VARIABLES 168 #define TK_SCORES 168
#define TK_BNODES 169 #define TK_TOPICS 169
#define TK_SNODES 170 #define TK_VARIABLES 170
#define TK_TRANSACTIONS 171 #define TK_BNODES 171
#define TK_DISTRIBUTED 172 #define TK_SNODES 172
#define TK_CONSUMERS 173 #define TK_TRANSACTIONS 173
#define TK_SUBSCRIPTIONS 174 #define TK_DISTRIBUTED 174
#define TK_VNODES 175 #define TK_CONSUMERS 175
#define TK_ALIVE 176 #define TK_SUBSCRIPTIONS 176
#define TK_VIEWS 177 #define TK_VNODES 177
#define TK_VIEW 178 #define TK_ALIVE 178
#define TK_COMPACTS 179 #define TK_VIEWS 179
#define TK_NORMAL 180 #define TK_VIEW 180
#define TK_CHILD 181 #define TK_COMPACTS 181
#define TK_LIKE 182 #define TK_NORMAL 182
#define TK_TBNAME 183 #define TK_CHILD 183
#define TK_QTAGS 184 #define TK_LIKE 184
#define TK_AS 185 #define TK_TBNAME 185
#define TK_SYSTEM 186 #define TK_QTAGS 186
#define TK_INDEX 187 #define TK_AS 187
#define TK_FUNCTION 188 #define TK_SYSTEM 188
#define TK_INTERVAL 189 #define TK_TSMA 189
#define TK_COUNT 190 #define TK_INTERVAL 190
#define TK_LAST_ROW 191 #define TK_RECURSIVE 191
#define TK_META 192 #define TK_TSMAS 192
#define TK_ONLY 193 #define TK_FUNCTION 193
#define TK_TOPIC 194 #define TK_INDEX 194
#define TK_CONSUMER 195 #define TK_COUNT 195
#define TK_GROUP 196 #define TK_LAST_ROW 196
#define TK_DESC 197 #define TK_META 197
#define TK_DESCRIBE 198 #define TK_ONLY 198
#define TK_RESET 199 #define TK_TOPIC 199
#define TK_QUERY 200 #define TK_CONSUMER 200
#define TK_CACHE 201 #define TK_GROUP 201
#define TK_EXPLAIN 202 #define TK_DESC 202
#define TK_ANALYZE 203 #define TK_DESCRIBE 203
#define TK_VERBOSE 204 #define TK_RESET 204
#define TK_NK_BOOL 205 #define TK_QUERY 205
#define TK_RATIO 206 #define TK_CACHE 206
#define TK_NK_FLOAT 207 #define TK_EXPLAIN 207
#define TK_OUTPUTTYPE 208 #define TK_ANALYZE 208
#define TK_AGGREGATE 209 #define TK_VERBOSE 209
#define TK_BUFSIZE 210 #define TK_NK_BOOL 210
#define TK_LANGUAGE 211 #define TK_RATIO 211
#define TK_REPLACE 212 #define TK_NK_FLOAT 212
#define TK_STREAM 213 #define TK_OUTPUTTYPE 213
#define TK_INTO 214 #define TK_AGGREGATE 214
#define TK_PAUSE 215 #define TK_BUFSIZE 215
#define TK_RESUME 216 #define TK_LANGUAGE 216
#define TK_TRIGGER 217 #define TK_REPLACE 217
#define TK_AT_ONCE 218 #define TK_STREAM 218
#define TK_WINDOW_CLOSE 219 #define TK_INTO 219
#define TK_IGNORE 220 #define TK_PAUSE 220
#define TK_EXPIRED 221 #define TK_RESUME 221
#define TK_FILL_HISTORY 222 #define TK_PRIMARY 222
#define TK_UPDATE 223 #define TK_KEY 223
#define TK_SUBTABLE 224 #define TK_TRIGGER 224
#define TK_UNTREATED 225 #define TK_AT_ONCE 225
#define TK_KILL 226 #define TK_WINDOW_CLOSE 226
#define TK_CONNECTION 227 #define TK_IGNORE 227
#define TK_TRANSACTION 228 #define TK_EXPIRED 228
#define TK_BALANCE 229 #define TK_FILL_HISTORY 229
#define TK_VGROUP 230 #define TK_UPDATE 230
#define TK_LEADER 231 #define TK_SUBTABLE 231
#define TK_MERGE 232 #define TK_UNTREATED 232
#define TK_REDISTRIBUTE 233 #define TK_KILL 233
#define TK_SPLIT 234 #define TK_CONNECTION 234
#define TK_DELETE 235 #define TK_TRANSACTION 235
#define TK_INSERT 236 #define TK_BALANCE 236
#define TK_NK_BIN 237 #define TK_VGROUP 237
#define TK_NK_HEX 238 #define TK_LEADER 238
#define TK_NULL 239 #define TK_MERGE 239
#define TK_NK_QUESTION 240 #define TK_REDISTRIBUTE 240
#define TK_NK_ALIAS 241 #define TK_SPLIT 241
#define TK_NK_ARROW 242 #define TK_DELETE 242
#define TK_ROWTS 243 #define TK_INSERT 243
#define TK_QSTART 244 #define TK_NK_BIN 244
#define TK_QEND 245 #define TK_NK_HEX 245
#define TK_QDURATION 246 #define TK_NULL 246
#define TK_WSTART 247 #define TK_NK_QUESTION 247
#define TK_WEND 248 #define TK_NK_ALIAS 248
#define TK_WDURATION 249 #define TK_NK_ARROW 249
#define TK_IROWTS 250 #define TK_ROWTS 250
#define TK_ISFILLED 251 #define TK_QSTART 251
#define TK_CAST 252 #define TK_QEND 252
#define TK_NOW 253 #define TK_QDURATION 253
#define TK_TODAY 254 #define TK_WSTART 254
#define TK_TIMEZONE 255 #define TK_WEND 255
#define TK_CLIENT_VERSION 256 #define TK_WDURATION 256
#define TK_SERVER_VERSION 257 #define TK_IROWTS 257
#define TK_SERVER_STATUS 258 #define TK_ISFILLED 258
#define TK_CURRENT_USER 259 #define TK_CAST 259
#define TK_CASE 260 #define TK_NOW 260
#define TK_WHEN 261 #define TK_TODAY 261
#define TK_THEN 262 #define TK_TIMEZONE 262
#define TK_ELSE 263 #define TK_CLIENT_VERSION 263
#define TK_BETWEEN 264 #define TK_SERVER_VERSION 264
#define TK_IS 265 #define TK_SERVER_STATUS 265
#define TK_NK_LT 266 #define TK_CURRENT_USER 266
#define TK_NK_GT 267 #define TK_CASE 267
#define TK_NK_LE 268 #define TK_WHEN 268
#define TK_NK_GE 269 #define TK_THEN 269
#define TK_NK_NE 270 #define TK_ELSE 270
#define TK_MATCH 271 #define TK_BETWEEN 271
#define TK_NMATCH 272 #define TK_IS 272
#define TK_CONTAINS 273 #define TK_NK_LT 273
#define TK_IN 274 #define TK_NK_GT 274
#define TK_JOIN 275 #define TK_NK_LE 275
#define TK_INNER 276 #define TK_NK_GE 276
#define TK_SELECT 277 #define TK_NK_NE 277
#define TK_NK_HINT 278 #define TK_MATCH 278
#define TK_DISTINCT 279 #define TK_NMATCH 279
#define TK_WHERE 280 #define TK_CONTAINS 280
#define TK_PARTITION 281 #define TK_IN 281
#define TK_BY 282 #define TK_JOIN 282
#define TK_SESSION 283 #define TK_INNER 283
#define TK_STATE_WINDOW 284 #define TK_LEFT 284
#define TK_EVENT_WINDOW 285 #define TK_RIGHT 285
#define TK_COUNT_WINDOW 286 #define TK_OUTER 286
#define TK_SLIDING 287 #define TK_SEMI 287
#define TK_FILL 288 #define TK_ANTI 288
#define TK_VALUE 289 #define TK_ASOF 289
#define TK_VALUE_F 290 #define TK_WINDOW 290
#define TK_NONE 291 #define TK_WINDOW_OFFSET 291
#define TK_PREV 292 #define TK_JLIMIT 292
#define TK_NULL_F 293 #define TK_SELECT 293
#define TK_LINEAR 294 #define TK_NK_HINT 294
#define TK_NEXT 295 #define TK_DISTINCT 295
#define TK_HAVING 296 #define TK_WHERE 296
#define TK_RANGE 297 #define TK_PARTITION 297
#define TK_EVERY 298 #define TK_BY 298
#define TK_ORDER 299 #define TK_SESSION 299
#define TK_SLIMIT 300 #define TK_STATE_WINDOW 300
#define TK_SOFFSET 301 #define TK_EVENT_WINDOW 301
#define TK_LIMIT 302 #define TK_COUNT_WINDOW 302
#define TK_OFFSET 303 #define TK_SLIDING 303
#define TK_ASC 304 #define TK_FILL 304
#define TK_NULLS 305 #define TK_VALUE 305
#define TK_ABORT 306 #define TK_VALUE_F 306
#define TK_AFTER 307 #define TK_NONE 307
#define TK_ATTACH 308 #define TK_PREV 308
#define TK_BEFORE 309 #define TK_NULL_F 309
#define TK_BEGIN 310 #define TK_LINEAR 310
#define TK_BITAND 311 #define TK_NEXT 311
#define TK_BITNOT 312 #define TK_HAVING 312
#define TK_BITOR 313 #define TK_RANGE 313
#define TK_BLOCKS 314 #define TK_EVERY 314
#define TK_CHANGE 315 #define TK_ORDER 315
#define TK_COMMA 316 #define TK_SLIMIT 316
#define TK_CONCAT 317 #define TK_SOFFSET 317
#define TK_CONFLICT 318 #define TK_LIMIT 318
#define TK_COPY 319 #define TK_OFFSET 319
#define TK_DEFERRED 320 #define TK_ASC 320
#define TK_DELIMITERS 321 #define TK_NULLS 321
#define TK_DETACH 322 #define TK_ABORT 322
#define TK_DIVIDE 323 #define TK_AFTER 323
#define TK_DOT 324 #define TK_ATTACH 324
#define TK_EACH 325 #define TK_BEFORE 325
#define TK_FAIL 326 #define TK_BEGIN 326
#define TK_FILE 327 #define TK_BITAND 327
#define TK_FOR 328 #define TK_BITNOT 328
#define TK_GLOB 329 #define TK_BITOR 329
#define TK_ID 330 #define TK_BLOCKS 330
#define TK_IMMEDIATE 331 #define TK_CHANGE 331
#define TK_IMPORT 332 #define TK_COMMA 332
#define TK_INITIALLY 333 #define TK_CONCAT 333
#define TK_INSTEAD 334 #define TK_CONFLICT 334
#define TK_ISNULL 335 #define TK_COPY 335
#define TK_MODULES 336 #define TK_DEFERRED 336
#define TK_NK_BITNOT 337 #define TK_DELIMITERS 337
#define TK_NK_SEMI 338 #define TK_DETACH 338
#define TK_NOTNULL 339 #define TK_DIVIDE 339
#define TK_OF 340 #define TK_DOT 340
#define TK_PLUS 341 #define TK_EACH 341
#define TK_PRIVILEGE 342 #define TK_FAIL 342
#define TK_RAISE 343 #define TK_FILE 343
#define TK_RESTRICT 344 #define TK_FOR 344
#define TK_ROW 345 #define TK_GLOB 345
#define TK_SEMI 346 #define TK_ID 346
#define TK_STAR 347 #define TK_IMMEDIATE 347
#define TK_STATEMENT 348 #define TK_IMPORT 348
#define TK_STRICT 349 #define TK_INITIALLY 349
#define TK_STRING 350 #define TK_INSTEAD 350
#define TK_TIMES 351 #define TK_ISNULL 351
#define TK_VALUES 352 #define TK_MODULES 352
#define TK_VARIABLE 353 #define TK_NK_BITNOT 353
#define TK_WAL 354 #define TK_NK_SEMI 354
#define TK_NOTNULL 355
#define TK_OF 356
#define TK_PLUS 357
#define TK_PRIVILEGE 358
#define TK_RAISE 359
#define TK_RESTRICT 360
#define TK_ROW 361
#define TK_STAR 362
#define TK_STATEMENT 363
#define TK_STRICT 364
#define TK_STRING 365
#define TK_TIMES 366
#define TK_VALUES 367
#define TK_VARIABLE 368
#define TK_WAL 369
#define TK_ENCODE 370
#define TK_COMPRESS 371
#define TK_LEVEL 372
#define TK_NK_SPACE 600
#define TK_NK_SPACE 600 #define TK_NK_COMMENT 601
#define TK_NK_COMMENT 601 #define TK_NK_ILLEGAL 602
#define TK_NK_ILLEGAL 602
// #define TK_NK_HEX 603 // hex number 0x123 // #define TK_NK_HEX 603 // hex number 0x123
#define TK_NK_OCT 604 // oct number #define TK_NK_OCT 604 // oct number
// #define TK_NK_BIN 605 // bin format data 0b111 // #define TK_NK_BIN 605 // bin format data 0b111
#define TK_BATCH_SCAN 606 #define TK_BATCH_SCAN 606
#define TK_NO_BATCH_SCAN 607 #define TK_NO_BATCH_SCAN 607
#define TK_SORT_FOR_GROUP 608 #define TK_SORT_FOR_GROUP 608
#define TK_PARTITION_FIRST 609 #define TK_PARTITION_FIRST 609
#define TK_PARA_TABLES_SORT 610 #define TK_PARA_TABLES_SORT 610
#define TK_SMALLDATA_TS_SORT 611 #define TK_SMALLDATA_TS_SORT 611
#define TK_HASH_JOIN 612
#define TK_SKIP_TSMA 613
#define TK_NK_NIL 65535 #define TK_NK_NIL 65535

View File

@ -268,9 +268,11 @@ typedef struct {
#define IS_MATHABLE_TYPE(_t) \ #define IS_MATHABLE_TYPE(_t) \
(IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP)) (IS_NUMERIC_TYPE(_t) || (_t) == (TSDB_DATA_TYPE_BOOL) || (_t) == (TSDB_DATA_TYPE_TIMESTAMP))
#define IS_VAR_DATA_TYPE(t) \ #define IS_VAR_DATA_TYPE(t) \
(((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY)) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR) || \
#define IS_STR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) ((t) == TSDB_DATA_TYPE_JSON) || ((t) == TSDB_DATA_TYPE_GEOMETRY))
#define IS_STR_DATA_TYPE(t) \
(((t) == TSDB_DATA_TYPE_VARCHAR) || ((t) == TSDB_DATA_TYPE_VARBINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX) #define IS_VALID_TINYINT(_t) ((_t) >= INT8_MIN && (_t) <= INT8_MAX)
#define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX) #define IS_VALID_SMALLINT(_t) ((_t) >= INT16_MIN && (_t) <= INT16_MAX)
@ -340,13 +342,28 @@ typedef struct tDataTypeDescriptor {
int32_t nBuf); int32_t nBuf);
} tDataTypeDescriptor; } tDataTypeDescriptor;
typedef struct tDataTypeCompress {
int16_t type;
int16_t nameLen;
int32_t bytes;
char *name;
int64_t minValue;
int64_t maxValue;
int32_t (*compFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t (*decompFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
} tDataTypeCompress;
extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX]; extern tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX];
bool isValidDataType(int32_t type); extern tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX];
bool isValidDataType(int32_t type);
int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); int32_t operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
void assignVal(char *val, const char *src, int32_t len, int32_t type); void assignVal(char *val, const char *src, int32_t len, int32_t type);
void *getDataMin(int32_t type, void* value); void *getDataMin(int32_t type, void *value);
void *getDataMax(int32_t type, void* value); void *getDataMax(int32_t type, void *value);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -40,4 +40,6 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg); int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta, void* pVnode);
#endif // TDENGINE_TQ_COMMON_H #endif // TDENGINE_TQ_COMMON_H

View File

@ -94,6 +94,8 @@ typedef struct SCatalogReq {
SArray* pTableCfg; // element is SNAME SArray* pTableCfg; // element is SNAME
SArray* pTableTag; // element is SNAME SArray* pTableTag; // element is SNAME
SArray* pView; // element is STablesReq SArray* pView; // element is STablesReq
SArray* pTableTSMAs; // element is STablesReq
SArray* pTSMAs; // element is STablesReq
bool qNodeRequired; // valid qnode bool qNodeRequired; // valid qnode
bool dNodeRequired; // valid dnode bool dNodeRequired; // valid dnode
bool svrVerRequired; bool svrVerRequired;
@ -122,6 +124,8 @@ typedef struct SMetaData {
SArray* pTableTag; // pRes = SArray<STagVal>* SArray* pTableTag; // pRes = SArray<STagVal>*
SArray* pDnodeList; // pRes = SArray<SEpSet>* SArray* pDnodeList; // pRes = SArray<SEpSet>*
SArray* pView; // pRes = SViewMeta* SArray* pView; // pRes = SViewMeta*
SArray* pTableTsmas; // pRes = SArray<STableTSMAInfo*>
SArray* pTsmas; // pRes = SArray<STableTSMAInfo*>
SMetaRes* pSvrVer; // pRes = char* SMetaRes* pSvrVer; // pRes = char*
} SMetaData; } SMetaData;
@ -130,9 +134,11 @@ typedef struct SCatalogCfg {
uint32_t maxViewCacheNum; uint32_t maxViewCacheNum;
uint32_t maxDBCacheNum; uint32_t maxDBCacheNum;
uint32_t maxUserCacheNum; uint32_t maxUserCacheNum;
uint32_t maxTSMACacheNum;
uint32_t dbRentSec; uint32_t dbRentSec;
uint32_t stbRentSec; uint32_t stbRentSec;
uint32_t viewRentSec; uint32_t viewRentSec;
uint32_t tsmaRentSec;
} SCatalogCfg; } SCatalogCfg;
typedef struct SSTableVersion { typedef struct SSTableVersion {
@ -152,6 +158,7 @@ typedef struct SDbCacheInfo {
int32_t cfgVersion; int32_t cfgVersion;
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
int64_t stateTs; int64_t stateTs;
int32_t tsmaVersion;
} SDbCacheInfo; } SDbCacheInfo;
typedef struct SDynViewVersion { typedef struct SDynViewVersion {
@ -167,6 +174,14 @@ typedef struct SViewVersion {
int32_t version; int32_t version;
} SViewVersion; } SViewVersion;
typedef struct STSMAVersion {
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
char name[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t tsmaId;
int32_t version;
} STSMAVersion;
typedef struct STbSVersion { typedef struct STbSVersion {
char* tbFName; char* tbFName;
@ -342,6 +357,8 @@ int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* n
int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num);
int32_t catalogGetExpiredTsmas(SCatalog* pCtg, STSMAVersion** tsmas, uint32_t* num);
int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg);
int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* pInfo); int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* pInfo);
@ -388,6 +405,16 @@ int32_t ctgdEnableDebug(char* option, bool enable);
int32_t ctgdHandleDbgCommand(char* command); int32_t ctgdHandleDbgCommand(char* command);
int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma, int32_t tsmaVersion);
int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma);
int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma);
int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma);
/** /**
* Destroy catalog and relase all resources * Destroy catalog and relase all resources
*/ */

View File

@ -78,7 +78,9 @@ typedef struct SMetaEntry {
} smaEntry; } smaEntry;
}; };
uint8_t* pBuf; uint8_t* pBuf;
SColCmprWrapper colCmpr; // col compress alg
} SMetaEntry; } SMetaEntry;
typedef struct SMetaReader { typedef struct SMetaReader {
@ -368,7 +370,8 @@ typedef struct SStateStore {
int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur, int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur,
const SSessionKey* pKey, void** pVal, int32_t* pVLen); const SSessionKey* pKey, void** pVal, int32_t* pVLen);
int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen); int32_t (*streamStateCountWinAddIfNotExist)(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount,
void** ppVal, int32_t* pVLen);
int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t (*streamStateCountWinAdd)(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp); SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp);

View File

@ -88,6 +88,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_LTRIM, FUNCTION_TYPE_LTRIM,
FUNCTION_TYPE_RTRIM, FUNCTION_TYPE_RTRIM,
FUNCTION_TYPE_SUBSTR, FUNCTION_TYPE_SUBSTR,
FUNCTION_TYPE_MD5,
// conversion function // conversion function
FUNCTION_TYPE_CAST = 2000, FUNCTION_TYPE_CAST = 2000,
@ -164,6 +165,18 @@ typedef enum EFunctionType {
FUNCTION_TYPE_STDDEV_MERGE, FUNCTION_TYPE_STDDEV_MERGE,
FUNCTION_TYPE_IRATE_PARTIAL, FUNCTION_TYPE_IRATE_PARTIAL,
FUNCTION_TYPE_IRATE_MERGE, FUNCTION_TYPE_IRATE_MERGE,
FUNCTION_TYPE_AVG_STATE,
FUNCTION_TYPE_AVG_STATE_MERGE,
FUNCTION_TYPE_FIRST_STATE,
FUNCTION_TYPE_FIRST_STATE_MERGE,
FUNCTION_TYPE_LAST_STATE,
FUNCTION_TYPE_LAST_STATE_MERGE,
FUNCTION_TYPE_SPREAD_STATE,
FUNCTION_TYPE_SPREAD_STATE_MERGE,
FUNCTION_TYPE_STDDEV_STATE,
FUNCTION_TYPE_STDDEV_STATE_MERGE,
FUNCTION_TYPE_HYPERLOGLOG_STATE,
FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE,
// geometry functions // geometry functions
FUNCTION_TYPE_GEOM_FROM_TEXT = 4250, FUNCTION_TYPE_GEOM_FROM_TEXT = 4250,
@ -238,6 +251,7 @@ bool fmIsCumulativeFunc(int32_t funcId);
bool fmIsInterpPseudoColumnFunc(int32_t funcId); bool fmIsInterpPseudoColumnFunc(int32_t funcId);
bool fmIsGroupKeyFunc(int32_t funcId); bool fmIsGroupKeyFunc(int32_t funcId);
bool fmIsBlockDistFunc(int32_t funcId); bool fmIsBlockDistFunc(int32_t funcId);
bool fmIsIgnoreNullFunc(int32_t funcId);
bool fmIsConstantResFunc(SFunctionNode* pFunc); bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId); bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId); bool fmIsPrimaryKeyFunc(int32_t funcId);
@ -270,6 +284,13 @@ bool fmIsInvertible(int32_t funcId);
char* fmGetFuncName(int32_t funcId); char* fmGetFuncName(int32_t funcId);
bool fmIsTSMASupportedFunc(func_id_t funcId);
int32_t fmCreateStateFuncs(SNodeList* pFuncs);
int32_t fmCreateStateMergeFuncs(SNodeList* pFuncs);
int32_t fmGetFuncId(const char* name);
bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId);
bool fmIsCountLikeFunc(int32_t funcId);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -23,10 +23,12 @@ extern "C" {
#include "query.h" #include "query.h"
#include "querynodes.h" #include "querynodes.h"
#define DESCRIBE_RESULT_COLS 4 #define DESCRIBE_RESULT_COLS 4
#define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_COLS_COMPRESS 7
#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_NOTE_LEN (16 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_NOTE_LEN (16 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_COPRESS_OPTION_LEN (TSDB_CL_COMPRESS_OPTION_LEN + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_DB_RESULT_COLS 2 #define SHOW_CREATE_DB_RESULT_COLS 2
#define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE)
@ -40,13 +42,12 @@ extern "C" {
#define SHOW_CREATE_VIEW_RESULT_FIELD1_LEN (TSDB_VIEW_FNAME_LEN + 4 + VARSTR_HEADER_SIZE) #define SHOW_CREATE_VIEW_RESULT_FIELD1_LEN (TSDB_VIEW_FNAME_LEN + 4 + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_VIEW_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_VIEW_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_COLS 3 #define SHOW_LOCAL_VARIABLES_RESULT_COLS 3
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD3_LEN (TSDB_CONFIG_SCOPE_LEN + VARSTR_HEADER_SIZE)
#define COMPACT_DB_RESULT_COLS 3 #define COMPACT_DB_RESULT_COLS 3
#define COMPACT_DB_RESULT_FIELD1_LEN 32 #define COMPACT_DB_RESULT_FIELD1_LEN 32
#define COMPACT_DB_RESULT_FIELD3_LEN 128 #define COMPACT_DB_RESULT_FIELD3_LEN 128
@ -100,6 +101,10 @@ typedef struct SDatabaseOptions {
int32_t sstTrigger; int32_t sstTrigger;
int32_t tablePrefix; int32_t tablePrefix;
int32_t tableSuffix; int32_t tableSuffix;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
SValueNode* s3KeepLocalStr;
int8_t s3Compact;
int8_t withArbitrator; int8_t withArbitrator;
} SDatabaseOptions; } SDatabaseOptions;
@ -138,6 +143,11 @@ typedef struct STrimDatabaseStmt {
int32_t maxSpeed; int32_t maxSpeed;
} STrimDatabaseStmt; } STrimDatabaseStmt;
typedef struct SS3MigrateDatabaseStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
} SS3MigrateDatabaseStmt;
typedef struct SCompactDatabaseStmt { typedef struct SCompactDatabaseStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
@ -163,13 +173,21 @@ typedef struct STableOptions {
SNodeList* pSma; SNodeList* pSma;
} STableOptions; } STableOptions;
typedef struct SColumnOptions {
ENodeType type;
bool commentNull;
char comment[TSDB_CL_COMMENT_LEN];
char encode[TSDB_CL_COMPRESS_OPTION_LEN];
char compress[TSDB_CL_COMPRESS_OPTION_LEN];
char compressLevel[TSDB_CL_COMPRESS_OPTION_LEN];
bool bPrimaryKey;
} SColumnOptions;
typedef struct SColumnDefNode { typedef struct SColumnDefNode {
ENodeType type; ENodeType type;
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
SDataType dataType; SDataType dataType;
char comments[TSDB_TB_COMMENT_LEN]; SNode* pOptions;
bool sma; bool sma;
bool is_pk;
} SColumnDefNode; } SColumnDefNode;
typedef struct SCreateTableStmt { typedef struct SCreateTableStmt {
@ -204,11 +222,13 @@ typedef struct SDropTableClause {
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
SArray* pTsmas;
} SDropTableClause; } SDropTableClause;
typedef struct SDropTableStmt { typedef struct SDropTableStmt {
ENodeType type; ENodeType type;
SNodeList* pTables; SNodeList* pTables;
bool withTsma;
} SDropTableStmt; } SDropTableStmt;
typedef struct SDropSuperTableStmt { typedef struct SDropSuperTableStmt {
@ -219,36 +239,37 @@ typedef struct SDropSuperTableStmt {
} SDropSuperTableStmt; } SDropSuperTableStmt;
typedef struct SAlterTableStmt { typedef struct SAlterTableStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
int8_t alterType; int8_t alterType;
char colName[TSDB_COL_NAME_LEN]; char colName[TSDB_COL_NAME_LEN];
char newColName[TSDB_COL_NAME_LEN]; char newColName[TSDB_COL_NAME_LEN];
STableOptions* pOptions; STableOptions* pOptions;
SDataType dataType; SDataType dataType;
SValueNode* pVal; SValueNode* pVal;
SColumnOptions* pColOptions;
} SAlterTableStmt; } SAlterTableStmt;
typedef struct SCreateUserStmt { typedef struct SCreateUserStmt {
ENodeType type; ENodeType type;
char userName[TSDB_USER_LEN]; char userName[TSDB_USER_LEN];
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
int8_t sysinfo; int8_t sysinfo;
int32_t numIpRanges; int32_t numIpRanges;
SIpV4Range* pIpRanges; SIpV4Range* pIpRanges;
SNodeList* pNodeListIpRanges; SNodeList* pNodeListIpRanges;
} SCreateUserStmt; } SCreateUserStmt;
typedef struct SAlterUserStmt { typedef struct SAlterUserStmt {
ENodeType type; ENodeType type;
char userName[TSDB_USER_LEN]; char userName[TSDB_USER_LEN];
int8_t alterType; int8_t alterType;
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
int8_t enable; int8_t enable;
int8_t sysinfo; int8_t sysinfo;
int32_t numIpRanges; int32_t numIpRanges;
SIpV4Range* pIpRanges; SIpV4Range* pIpRanges;
SNodeList* pNodeListIpRanges; SNodeList* pNodeListIpRanges;
@ -286,7 +307,7 @@ typedef struct SShowStmt {
SNode* pDbName; // SValueNode SNode* pDbName; // SValueNode
SNode* pTbName; // SValueNode SNode* pTbName; // SValueNode
EOperatorType tableCondType; EOperatorType tableCondType;
EShowKind showKind; // show databases: user/system, show tables: normal/child, others NULL EShowKind showKind; // show databases: user/system, show tables: normal/child, others NULL
} SShowStmt; } SShowStmt;
typedef struct SShowCreateDatabaseStmt { typedef struct SShowCreateDatabaseStmt {
@ -347,7 +368,7 @@ typedef struct SShowCompactsStmt {
typedef struct SShowCompactDetailsStmt { typedef struct SShowCompactDetailsStmt {
ENodeType type; ENodeType type;
SNode* pCompactId; SNode* pCompactId;
} SShowCompactDetailsStmt; } SShowCompactDetailsStmt;
typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType; typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT, INDEX_TYPE_NORMAL } EIndexType;
@ -525,20 +546,20 @@ typedef struct SDropFunctionStmt {
} SDropFunctionStmt; } SDropFunctionStmt;
typedef struct SCreateViewStmt { typedef struct SCreateViewStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char viewName[TSDB_VIEW_NAME_LEN]; char viewName[TSDB_VIEW_NAME_LEN];
char* pQuerySql; char* pQuerySql;
bool orReplace; bool orReplace;
SNode* pQuery; SNode* pQuery;
SCMCreateViewReq createReq; SCMCreateViewReq createReq;
} SCreateViewStmt; } SCreateViewStmt;
typedef struct SDropViewStmt { typedef struct SDropViewStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char viewName[TSDB_VIEW_NAME_LEN]; char viewName[TSDB_VIEW_NAME_LEN];
bool ignoreNotExists; bool ignoreNotExists;
} SDropViewStmt; } SDropViewStmt;
typedef struct SGrantStmt { typedef struct SGrantStmt {
@ -581,6 +602,34 @@ typedef struct SSplitVgroupStmt {
int32_t vgId; int32_t vgId;
} SSplitVgroupStmt; } SSplitVgroupStmt;
typedef struct STSMAOptions {
ENodeType type;
SNodeList* pFuncs;
SNodeList* pCols;
SNode* pInterval;
uint8_t tsPrecision;
bool recursiveTsma; // true if create recursive tsma
} STSMAOptions;
typedef struct SCreateTSMAStmt {
ENodeType type;
bool ignoreExists;
char tsmaName[TSDB_TABLE_NAME_LEN];
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; // base tb name or base tsma name
char originalTbName[TSDB_TABLE_NAME_LEN];
STSMAOptions* pOptions;
SNode* pPrevQuery;
SMCreateSmaReq* pReq;
} SCreateTSMAStmt;
typedef struct SDropTSMAStmt {
ENodeType type;
bool ignoreNotExists;
char dbName[TSDB_DB_NAME_LEN];
char tsmaName[TSDB_TABLE_NAME_LEN];
} SDropTSMAStmt;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -123,6 +123,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc); int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListMakeStrictAppendList(SNodeList** pTarget, SNodeList* pSrc); int32_t nodesListMakeStrictAppendList(SNodeList** pTarget, SNodeList* pSrc);
int32_t nodesListPushFront(SNodeList* pList, SNode* pNode); int32_t nodesListPushFront(SNodeList* pList, SNode* pNode);
int32_t nodesListMakePushFront(SNodeList** pList, SNode* pNode);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell); SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc); void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc); void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
@ -169,6 +170,7 @@ int32_t nodesMsgToNode(const char* pStr, int32_t len, SNode** pNode);
int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len); int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len);
char* nodesGetNameFromColumnNode(SNode* pNode); char* nodesGetNameFromColumnNode(SNode* pNode);
int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots); int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots);
void nodesSortList(SNodeList** pList, int32_t (*)(SNode* pNode1, SNode* pNode2));
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -66,6 +66,7 @@ typedef struct SLogicNode {
EOrder inputTsOrder; EOrder inputTsOrder;
EOrder outputTsOrder; EOrder outputTsOrder;
bool forceCreateNonBlockingOptr; // true if the operator can use non-blocking(pipeline) mode bool forceCreateNonBlockingOptr; // true if the operator can use non-blocking(pipeline) mode
bool splitDone;
} SLogicNode; } SLogicNode;
typedef enum EScanType { typedef enum EScanType {
@ -108,6 +109,9 @@ typedef struct SScanLogicNode {
int8_t igExpired; int8_t igExpired;
int8_t igCheckUpdate; int8_t igCheckUpdate;
SArray* pSmaIndexes; SArray* pSmaIndexes;
SArray* pTsmas;
SArray* pTsmaTargetTbVgInfo;
SArray* pTsmaTargetTbInfo;
SNodeList* pGroupTags; SNodeList* pGroupTags;
bool groupSort; bool groupSort;
SNodeList* pTags; // for create stream SNodeList* pTags; // for create stream
@ -123,20 +127,38 @@ typedef struct SScanLogicNode {
SArray* pFuncTypes; // for last, last_row SArray* pFuncTypes; // for last, last_row
bool paraTablesSort; // for table merge scan bool paraTablesSort; // for table merge scan
bool smallDataTsSort; // disable row id sort for table merge scan bool smallDataTsSort; // disable row id sort for table merge scan
bool needSplit;
} SScanLogicNode; } SScanLogicNode;
typedef struct SJoinLogicNode { typedef struct SJoinLogicNode {
SLogicNode node; SLogicNode node;
EJoinType joinType; EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
EJoinAlgorithm joinAlgo; EJoinAlgorithm joinAlgo;
SNode* addPrimEqCond;
SNode* pPrimKeyEqCond; SNode* pPrimKeyEqCond;
SNode* pColEqCond; SNode* pColEqCond;
SNode* pColOnCond;
SNode* pTagEqCond; SNode* pTagEqCond;
SNode* pTagOnCond; SNode* pTagOnCond;
SNode* pOtherOnCond; SNode* pFullOnCond; // except prim eq cond
SNodeList* pLeftEqNodes;
SNodeList* pRightEqNodes;
bool allEqTags;
bool isSingleTableJoin; bool isSingleTableJoin;
bool hasSubQuery; bool hasSubQuery;
bool isLowLevelJoin; bool isLowLevelJoin;
bool seqWinGroup;
bool grpJoin;
bool hashJoinHint;
// FOR HASH JOIN
int32_t timeRangeTarget; //table onCond filter
STimeWindow timeRange; //table onCond filter
SNode* pLeftOnCond; //table onCond filter
SNode* pRightOnCond; //table onCond filter
} SJoinLogicNode; } SJoinLogicNode;
typedef struct SAggLogicNode { typedef struct SAggLogicNode {
@ -151,6 +173,7 @@ typedef struct SAggLogicNode {
bool isGroupTb; bool isGroupTb;
bool isPartTb; // true if partition keys has tbname bool isPartTb; // true if partition keys has tbname
bool hasGroup; bool hasGroup;
SNodeList *pTsmaSubplans;
} SAggLogicNode; } SAggLogicNode;
typedef struct SProjectLogicNode { typedef struct SProjectLogicNode {
@ -234,7 +257,9 @@ typedef struct SMergeLogicNode {
SNodeList* pMergeKeys; SNodeList* pMergeKeys;
SNodeList* pInputs; SNodeList* pInputs;
int32_t numOfChannels; int32_t numOfChannels;
int32_t numOfSubplans;
int32_t srcGroupId; int32_t srcGroupId;
int32_t srcEndGroupId;
bool colsMerge; bool colsMerge;
bool needSort; bool needSort;
bool groupSort; bool groupSort;
@ -287,6 +312,7 @@ typedef struct SWindowLogicNode {
bool isPartTb; bool isPartTb;
int64_t windowCount; int64_t windowCount;
int64_t windowSliding; int64_t windowSliding;
SNodeList* pTsmaSubplans;
} SWindowLogicNode; } SWindowLogicNode;
typedef struct SFillLogicNode { typedef struct SFillLogicNode {
@ -480,26 +506,52 @@ typedef struct SInterpFuncPhysiNode {
} SInterpFuncPhysiNode; } SInterpFuncPhysiNode;
typedef struct SSortMergeJoinPhysiNode { typedef struct SSortMergeJoinPhysiNode {
SPhysiNode node; SPhysiNode node;
EJoinType joinType; EJoinType joinType;
SNode* pPrimKeyCond; EJoinSubType subType;
SNode* pColEqCond; SNode* pWindowOffset;
SNode* pOtherOnCond; SNode* pJLimit;
SNodeList* pTargets; int32_t asofOpType;
SNode* leftPrimExpr;
SNode* rightPrimExpr;
int32_t leftPrimSlotId;
int32_t rightPrimSlotId;
SNodeList* pEqLeft;
SNodeList* pEqRight;
SNode* pPrimKeyCond; //remove
SNode* pColEqCond; //remove
SNode* pColOnCond;
SNode* pFullOnCond;
SNodeList* pTargets;
SQueryStat inputStat[2];
bool seqWinGroup;
bool grpJoin;
} SSortMergeJoinPhysiNode; } SSortMergeJoinPhysiNode;
typedef struct SHashJoinPhysiNode { typedef struct SHashJoinPhysiNode {
SPhysiNode node; SPhysiNode node;
EJoinType joinType; EJoinType joinType;
SNodeList* pOnLeft; EJoinSubType subType;
SNodeList* pOnRight; SNode* pWindowOffset;
SNode* pFilterConditions; SNode* pJLimit;
SNodeList* pTargets; SNodeList* pOnLeft;
SQueryStat inputStat[2]; SNodeList* pOnRight;
SNode* leftPrimExpr;
SNode* rightPrimExpr;
int32_t leftPrimSlotId;
int32_t rightPrimSlotId;
int32_t timeRangeTarget; //table onCond filter
STimeWindow timeRange; //table onCond filter
SNode* pLeftOnCond; //table onCond filter
SNode* pRightOnCond; //table onCond filter
SNode* pFullOnCond; //preFilter
SNodeList* pTargets;
SQueryStat inputStat[2];
SNode* pPrimKeyCond; // only in planner internal
SNode* pColEqCond; SNode* pPrimKeyCond;
SNode* pTagEqCond; SNode* pColEqCond;
SNode* pTagEqCond;
} SHashJoinPhysiNode; } SHashJoinPhysiNode;
typedef struct SGroupCachePhysiNode { typedef struct SGroupCachePhysiNode {
@ -533,6 +585,7 @@ typedef struct SAggPhysiNode {
SNodeList* pAggFuncs; SNodeList* pAggFuncs;
bool mergeDataBlock; bool mergeDataBlock;
bool groupKeyOptimized; bool groupKeyOptimized;
bool hasCountLikeFunc;
} SAggPhysiNode; } SAggPhysiNode;
typedef struct SDownstreamSourceNode { typedef struct SDownstreamSourceNode {
@ -562,7 +615,9 @@ typedef struct SMergePhysiNode {
SNodeList* pMergeKeys; SNodeList* pMergeKeys;
SNodeList* pTargets; SNodeList* pTargets;
int32_t numOfChannels; int32_t numOfChannels;
int32_t numOfSubplans;
int32_t srcGroupId; int32_t srcGroupId;
int32_t srcEndGroupId;
bool groupSort; bool groupSort;
bool ignoreGroupId; bool ignoreGroupId;
bool inputWithGroupId; bool inputWithGroupId;

View File

@ -23,6 +23,7 @@ extern "C" {
#include "nodes.h" #include "nodes.h"
#include "tmsg.h" #include "tmsg.h"
#include "tvariant.h" #include "tvariant.h"
#include "tsimplehash.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags) #define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) \ #define TABLE_META_SIZE(pMeta) \
@ -79,6 +80,7 @@ typedef struct SColumnNode {
uint16_t projIdx; // the idx in project list, start from 1 uint16_t projIdx; // the idx in project list, start from 1
EColumnType colType; // column or tag EColumnType colType; // column or tag
bool hasIndex; bool hasIndex;
bool isPrimTs;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN];
char tableAlias[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN];
@ -103,10 +105,16 @@ typedef struct STargetNode {
SNode* pExpr; SNode* pExpr;
} STargetNode; } STargetNode;
#define VALUE_FLAG_IS_DURATION (1 << 0)
#define VALUE_FLAG_IS_TIME_OFFSET (1 << 1)
#define IS_DURATION_VAL(_flag) ((_flag) & VALUE_FLAG_IS_DURATION)
#define IS_TIME_OFFSET_VAL(_flag) ((_flag) & VALUE_FLAG_IS_TIME_OFFSET)
typedef struct SValueNode { typedef struct SValueNode {
SExprNode node; // QUERY_NODE_VALUE SExprNode node; // QUERY_NODE_VALUE
char* literal; char* literal;
bool isDuration; int32_t flag;
bool translate; bool translate;
bool notReserved; bool notReserved;
bool isNull; bool isNull;
@ -133,6 +141,8 @@ typedef enum EHintOption {
HINT_PARTITION_FIRST, HINT_PARTITION_FIRST,
HINT_PARA_TABLES_SORT, HINT_PARA_TABLES_SORT,
HINT_SMALLDATA_TS_SORT, HINT_SMALLDATA_TS_SORT,
HINT_HASH_JOIN,
HINT_SKIP_TSMA,
} EHintOption; } EHintOption;
typedef struct SHintNode { typedef struct SHintNode {
@ -168,6 +178,8 @@ typedef struct SFunctionNode {
int32_t udfBufSize; int32_t udfBufSize;
bool hasPk; bool hasPk;
int32_t pkBytes; int32_t pkBytes;
bool hasOriginalFunc;
int32_t originalFuncId;
} SFunctionNode; } SFunctionNode;
typedef struct STableNode { typedef struct STableNode {
@ -181,6 +193,11 @@ typedef struct STableNode {
struct STableMeta; struct STableMeta;
typedef struct STsmaTargetCTbInfo {
char tableName[TSDB_TABLE_NAME_LEN]; // child table or normal table name
uint64_t uid;
} STsmaTargetTbInfo;
typedef struct SRealTableNode { typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta; struct STableMeta* pMeta;
@ -189,6 +206,9 @@ typedef struct SRealTableNode {
double ratio; double ratio;
SArray* pSmaIndexes; SArray* pSmaIndexes;
int8_t cacheLastMode; int8_t cacheLastMode;
SArray* pTsmas;
SArray* tsmaTargetTbVgInfo; // SArray<SVgroupsInfo*>, used for child table or normal table only
SArray* tsmaTargetTbInfo; // SArray<STsmaTargetTbInfo>, used for child table or normal table only
} SRealTableNode; } SRealTableNode;
typedef struct STempTableNode { typedef struct STempTableNode {
@ -206,12 +226,31 @@ typedef struct SViewNode {
int8_t cacheLastMode; int8_t cacheLastMode;
} SViewNode; } SViewNode;
#define JOIN_JLIMIT_MAX_VALUE 1024
#define IS_INNER_NONE_JOIN(_type, _stype) ((_type) == JOIN_TYPE_INNER && (_stype) == JOIN_STYPE_NONE)
#define IS_SEMI_JOIN(_stype) ((_stype) == JOIN_STYPE_SEMI)
#define IS_WINDOW_JOIN(_stype) ((_stype) == JOIN_STYPE_WIN)
#define IS_ASOF_JOIN(_stype) ((_stype) == JOIN_STYPE_ASOF)
typedef enum EJoinType { typedef enum EJoinType {
JOIN_TYPE_INNER = 1, JOIN_TYPE_INNER = 0,
JOIN_TYPE_LEFT, JOIN_TYPE_LEFT,
JOIN_TYPE_RIGHT, JOIN_TYPE_RIGHT,
JOIN_TYPE_FULL,
JOIN_TYPE_MAX_VALUE
} EJoinType; } EJoinType;
typedef enum EJoinSubType {
JOIN_STYPE_NONE = 0,
JOIN_STYPE_OUTER,
JOIN_STYPE_SEMI,
JOIN_STYPE_ANTI,
JOIN_STYPE_ASOF,
JOIN_STYPE_WIN,
JOIN_STYPE_MAX_VALUE
} EJoinSubType;
typedef enum EJoinAlgorithm { typedef enum EJoinAlgorithm {
JOIN_ALGO_UNKNOWN = 0, JOIN_ALGO_UNKNOWN = 0,
JOIN_ALGO_MERGE, JOIN_ALGO_MERGE,
@ -223,13 +262,18 @@ typedef enum EDynQueryType {
} EDynQueryType; } EDynQueryType;
typedef struct SJoinTableNode { typedef struct SJoinTableNode {
STableNode table; // QUERY_NODE_JOIN_TABLE STableNode table; // QUERY_NODE_JOIN_TABLE
EJoinType joinType; EJoinType joinType;
bool hasSubQuery; EJoinSubType subType;
bool isLowLevelJoin; SNode* pWindowOffset;
SNode* pLeft; SNode* pJLimit;
SNode* pRight; SNode* addPrimCond;
SNode* pOnCond; bool hasSubQuery;
bool isLowLevelJoin;
SNode* pParent;
SNode* pLeft;
SNode* pRight;
SNode* pOnCond;
} SJoinTableNode; } SJoinTableNode;
typedef enum EGroupingSetType { GP_TYPE_NORMAL = 1 } EGroupingSetType; typedef enum EGroupingSetType { GP_TYPE_NORMAL = 1 } EGroupingSetType;
@ -305,6 +349,7 @@ typedef enum EFillMode {
typedef enum ETimeLineMode { typedef enum ETimeLineMode {
TIME_LINE_NONE = 1, TIME_LINE_NONE = 1,
TIME_LINE_BLOCK,
TIME_LINE_MULTI, TIME_LINE_MULTI,
TIME_LINE_GLOBAL, TIME_LINE_GLOBAL,
} ETimeLineMode; } ETimeLineMode;
@ -338,6 +383,13 @@ typedef struct SCaseWhenNode {
SNodeList* pWhenThenList; SNodeList* pWhenThenList;
} SCaseWhenNode; } SCaseWhenNode;
typedef struct SWindowOffsetNode {
ENodeType type; // QUERY_NODE_WINDOW_OFFSET
SNode* pStartOffset; // SValueNode
SNode* pEndOffset; // SValueNode
} SWindowOffsetNode;
typedef struct SSelectStmt { typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT ENodeType type; // QUERY_NODE_SELECT_STMT
bool isDistinct; bool isDistinct;
@ -362,6 +414,7 @@ typedef struct SSelectStmt {
uint8_t precision; uint8_t precision;
int32_t selectFuncNum; int32_t selectFuncNum;
int32_t returnRows; // EFuncReturnRows int32_t returnRows; // EFuncReturnRows
ETimeLineMode timeLineCurMode;
ETimeLineMode timeLineResMode; ETimeLineMode timeLineResMode;
bool isEmptyResult; bool isEmptyResult;
bool isSubquery; bool isSubquery;
@ -385,6 +438,7 @@ typedef struct SSelectStmt {
bool onlyHasKeepOrderFunc; bool onlyHasKeepOrderFunc;
bool groupSort; bool groupSort;
bool tagScan; bool tagScan;
bool joinContains;
} SSelectStmt; } SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
@ -400,6 +454,7 @@ typedef struct SSetOperator {
char stmtName[TSDB_TABLE_NAME_LEN]; char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision; uint8_t precision;
ETimeLineMode timeLineResMode; ETimeLineMode timeLineResMode;
bool joinContains;
} SSetOperator; } SSetOperator;
typedef enum ESqlClause { typedef enum ESqlClause {
@ -537,12 +592,15 @@ typedef struct SQuery {
bool stableQuery; bool stableQuery;
} SQuery; } SQuery;
void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext); void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext); void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType; typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type, int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols); SNodeList** pCols);
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols); int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId); typedef bool (*FFuncClassifier)(int32_t funcId);
@ -579,6 +637,12 @@ const char* logicConditionTypeStr(ELogicConditionType type);
bool nodesIsStar(SNode* pNode); bool nodesIsStar(SNode* pNode);
bool nodesIsTableStar(SNode* pNode); bool nodesIsTableStar(SNode* pNode);
char* getJoinTypeString(EJoinType type);
char* getJoinSTypeString(EJoinSubType type);
char* getFullJoinTypeString(EJoinType type, EJoinSubType stype);
int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -158,6 +158,7 @@ int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap); SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
void destoryCatalogReq(SCatalogReq *pCatalogReq); void destoryCatalogReq(SCatalogReq *pCatalogReq);
bool isPrimaryKeyImpl(SNode* pExpr);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -118,6 +118,7 @@ typedef struct STableMeta {
int32_t sversion; int32_t sversion;
int32_t tversion; int32_t tversion;
STableComInfo tableInfo; STableComInfo tableInfo;
SSchemaExt* schemaExt; // There is no additional memory allocation, and the pointer is fixed to the next address of the schema content.
SSchema schema[]; SSchema schema[];
} STableMeta; } STableMeta;
#pragma pack(pop) #pragma pack(pop)
@ -272,6 +273,7 @@ void initQueryModuleMsgHandle();
const SSchema* tGetTbnameColumnSchema(); const SSchema* tGetTbnameColumnSchema();
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags); bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
int32_t getAsofJoinReverseOp(EOperatorType op);
int32_t queryCreateCTableMetaFromMsg(STableMetaRsp* msg, SCTableMeta* pMeta); int32_t queryCreateCTableMetaFromMsg(STableMetaRsp* msg, SCTableMeta* pMeta);
int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta); int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
@ -330,7 +332,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
(_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW) (_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW || \
(_type) == TDMT_MND_CREATE_TSMA || (_type) == TDMT_MND_DROP_TSMA || (_type) == TDMT_MND_DROP_TB_WITH_TSMA)
#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
(SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \

View File

@ -31,6 +31,15 @@ enum {
FLT_OPTION_NEED_UNIQE = 4, FLT_OPTION_NEED_UNIQE = 4,
}; };
typedef enum EConditionType {
COND_TYPE_PRIMARY_KEY = 1,
COND_TYPE_TAG_INDEX,
COND_TYPE_TAG,
COND_TYPE_NORMAL
} EConditionType;
#define FILTER_RESULT_ALL_QUALIFIED 0x1 #define FILTER_RESULT_ALL_QUALIFIED 0x1
#define FILTER_RESULT_NONE_QUALIFIED 0x2 #define FILTER_RESULT_NONE_QUALIFIED 0x2
#define FILTER_RESULT_PARTIAL_QUALIFIED 0x3 #define FILTER_RESULT_PARTIAL_QUALIFIED 0x3
@ -54,6 +63,8 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg,
/* condition split interface */ /* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,
SNode **pOtherCond); SNode **pOtherCond);
bool filterIsMultiTableColsCond(SNode *pCond);
EConditionType filterClassifyCondition(SNode *pNode);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -72,6 +72,7 @@ int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t md5Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
/* Conversion functions */ /* Conversion functions */
int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@ -82,6 +83,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toTimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toTimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toCharFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t toCharFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int64_t offsetFromTz(char *timezone, int64_t factor);
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

View File

@ -83,7 +83,8 @@ int32_t streamStateClearBuff(SStreamState* pState, void* pVal);
void streamStateFreeVal(void* val); void streamStateFreeVal(void* val);
// count window // count window
int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal, int32_t* pVLen); int32_t streamStateCountWinAddIfNotExist(SStreamState* pState, SSessionKey* pKey, COUNT_TYPE winCount, void** ppVal,
int32_t* pVLen);
int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateCountWinAdd(SStreamState* pState, SSessionKey* pKey, void** pVal, int32_t* pVLen);
SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key);
@ -128,6 +129,8 @@ typedef struct SStateSessionKey {
typedef struct SStreamValue { typedef struct SStreamValue {
int64_t unixTimestamp; int64_t unixTimestamp;
int32_t len; int32_t len;
int32_t rawLen;
int8_t compress;
char* data; char* data;
} SStreamValue; } SStreamValue;

View File

@ -247,11 +247,11 @@ typedef enum {
TASK_SCANHISTORY_CONT = 0x1, TASK_SCANHISTORY_CONT = 0x1,
TASK_SCANHISTORY_QUIT = 0x2, TASK_SCANHISTORY_QUIT = 0x2,
TASK_SCANHISTORY_REXEC = 0x3, TASK_SCANHISTORY_REXEC = 0x3,
} EScanHistoryRet; } EScanHistoryCode;
typedef struct { typedef struct {
EScanHistoryRet ret; EScanHistoryCode ret;
int32_t idleTime; int32_t idleTime;
} SScanhistoryDataInfo; } SScanhistoryDataInfo;
typedef struct { typedef struct {
@ -304,9 +304,9 @@ typedef struct SStreamTaskId {
typedef struct SCheckpointInfo { typedef struct SCheckpointInfo {
int64_t startTs; int64_t startTs;
int64_t checkpointId; int64_t checkpointId; // latest checkpoint id
int64_t checkpointVer; // latest checkpoint offset in wal
int64_t checkpointVer; // latest checkpointId version int64_t checkpointTime; // latest checkpoint time
int64_t processedVer; int64_t processedVer;
int64_t nextProcessVer; // current offset in WAL, not serialize it int64_t nextProcessVer; // current offset in WAL, not serialize it
int64_t failedId; // record the latest failed checkpoint id int64_t failedId; // record the latest failed checkpoint id
@ -384,8 +384,11 @@ typedef struct SSinkRecorder {
typedef struct STaskExecStatisInfo { typedef struct STaskExecStatisInfo {
int64_t created; int64_t created;
int64_t init; int64_t checkTs;
int64_t start; int64_t readyTs;
int64_t startCheckpointId;
int64_t startCheckpointVer;
int64_t step1Start; int64_t step1Start;
double step1El; double step1El;
int64_t step2Start; int64_t step2Start;
@ -429,6 +432,22 @@ typedef struct SUpstreamInfo {
int32_t numOfClosed; int32_t numOfClosed;
} SUpstreamInfo; } SUpstreamInfo;
typedef struct SDownstreamStatusInfo {
int64_t reqId;
int32_t taskId;
int64_t rspTs;
int32_t status;
} SDownstreamStatusInfo;
typedef struct STaskCheckInfo {
SArray* pList;
int64_t startTs;
int32_t notReadyTasks;
int32_t inCheckProcess;
tmr_h checkRspTmr;
TdThreadMutex checkInfoLock;
} STaskCheckInfo;
struct SStreamTask { struct SStreamTask {
int64_t ver; int64_t ver;
SStreamTaskId id; SStreamTaskId id;
@ -442,6 +461,7 @@ struct SStreamTask {
SCheckpointInfo chkInfo; SCheckpointInfo chkInfo;
STaskExec exec; STaskExec exec;
SDataRange dataRange; SDataRange dataRange;
SVersionRange step2Range;
SHistoryTaskInfo hTaskInfo; SHistoryTaskInfo hTaskInfo;
STaskId streamTaskId; STaskId streamTaskId;
STaskExecStatisInfo execInfo; STaskExecStatisInfo execInfo;
@ -451,14 +471,12 @@ struct SStreamTask {
SStreamState* pState; // state backend SStreamState* pState; // state backend
SArray* pRspMsgList; SArray* pRspMsgList;
SUpstreamInfo upstreamInfo; SUpstreamInfo upstreamInfo;
STaskCheckInfo taskCheckInfo;
// the followings attributes don't be serialized // the followings attributes don't be serialized
SScanhistorySchedInfo schedHistoryInfo; SScanhistorySchedInfo schedHistoryInfo;
int32_t notReadyTasks;
int32_t numOfWaitingUpstream; int32_t numOfWaitingUpstream;
int64_t checkReqId;
SArray* checkReqIds; // shuffle
int32_t refCnt; int32_t refCnt;
int32_t transferStateAlignCnt; int32_t transferStateAlignCnt;
struct SStreamMeta* pMeta; struct SStreamMeta* pMeta;
@ -474,7 +492,7 @@ typedef struct STaskStartInfo {
int64_t startTs; int64_t startTs;
int64_t readyTs; int64_t readyTs;
int32_t tasksWillRestart; int32_t tasksWillRestart;
int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. int32_t startAllTasks; // restart flag, sentinel to guard the restart procedure.
SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing
SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed
int64_t elapsedTime; int64_t elapsedTime;
@ -672,24 +690,34 @@ typedef struct {
int32_t tEncodeStreamCheckpointReadyMsg(SEncoder* pEncoder, const SStreamCheckpointReadyMsg* pRsp); int32_t tEncodeStreamCheckpointReadyMsg(SEncoder* pEncoder, const SStreamCheckpointReadyMsg* pRsp);
int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointReadyMsg* pRsp); int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointReadyMsg* pRsp);
typedef struct STaskCkptInfo {
int64_t latestId; // saved checkpoint id
int64_t latestVer; // saved checkpoint ver
int64_t latestTime; // latest checkpoint time
int64_t activeId; // current active checkpoint id
int32_t activeTransId; // checkpoint trans id
int8_t failed; // denote if the checkpoint is failed or not
} STaskCkptInfo;
typedef struct STaskStatusEntry { typedef struct STaskStatusEntry {
STaskId id; STaskId id;
int32_t status; int32_t status;
int32_t statusLastDuration; // to record the last duration of current status int32_t statusLastDuration; // to record the last duration of current status
int64_t stage; int64_t stage;
int32_t nodeId; int32_t nodeId;
int64_t verStart; // start version in WAL, only valid for source task SVersionRange verRange; // start/end version in WAL, only valid for source task
int64_t verEnd; // end version in WAL, only valid for source task int64_t processedVer; // only valid for source task
int64_t processedVer; // only valid for source task bool inputQChanging; // inputQ is changing or not
int64_t checkpointId; // current active checkpoint id int64_t inputQUnchangeCounter;
int32_t chkpointTransId; // checkpoint trans id double inputQUsed; // in MiB
int8_t checkpointFailed; // denote if the checkpoint is failed or not double inputRate;
bool inputQChanging; // inputQ is changing or not double sinkQuota; // existed quota size for sink task
int64_t inputQUnchangeCounter; double sinkDataSize; // sink to dst data size
double inputQUsed; // in MiB int64_t startTime;
double inputRate; int64_t startCheckpointId;
double sinkQuota; // existed quota size for sink task int64_t startCheckpointVer;
double sinkDataSize; // sink to dst data size int64_t hTaskId;
STaskCkptInfo checkpointInfo;
} STaskStatusEntry; } STaskStatusEntry;
typedef struct SStreamHbMsg { typedef struct SStreamHbMsg {
@ -797,7 +825,7 @@ int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq*
int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp); int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp);
int32_t streamLaunchFillHistoryTask(SStreamTask* pTask); int32_t streamLaunchFillHistoryTask(SStreamTask* pTask);
int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated); int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration); int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration);
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
@ -807,8 +835,6 @@ void streamTaskPause(SStreamMeta* pMeta, SStreamTask* pTask);
void streamTaskResume(SStreamTask* pTask); void streamTaskResume(SStreamTask* pTask);
int32_t streamTaskStop(SStreamTask* pTask); int32_t streamTaskStop(SStreamTask* pTask);
int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask);
void streamTaskUpdateUpstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet);
void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet);
void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask); void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDownstreamTask);
int32_t streamTaskReleaseState(SStreamTask* pTask); int32_t streamTaskReleaseState(SStreamTask* pTask);
int32_t streamTaskReloadState(SStreamTask* pTask); int32_t streamTaskReloadState(SStreamTask* pTask);
@ -818,6 +844,15 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key);
bool streamTaskIsSinkTask(const SStreamTask* pTask); bool streamTaskIsSinkTask(const SStreamTask* pTask);
int32_t streamTaskSendCheckpointReq(SStreamTask* pTask); int32_t streamTaskSendCheckpointReq(SStreamTask* pTask);
int32_t streamTaskInitTaskCheckInfo(STaskCheckInfo* pInfo, STaskOutputInfo* pOutputInfo, int64_t startTs);
int32_t streamTaskAddReqInfo(STaskCheckInfo* pInfo, int64_t reqId, int32_t taskId, const char* id);
int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t status, int64_t rspTs, int64_t reqId,
int32_t* pNotReady, const char* id);
void streamTaskCleanCheckInfo(STaskCheckInfo* pInfo);
int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id);
int32_t streamTaskCompleteCheck(STaskCheckInfo* pInfo, const char* id);
int32_t streamTaskStartMonitorCheckRsp(SStreamTask* pTask);
void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask); void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask);
void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc);

View File

@ -31,7 +31,7 @@ extern "C" {
#define TSWAP(a, b) \ #define TSWAP(a, b) \
do { \ do { \
char *__tmp = alloca(sizeof(a)); \ char *__tmp = (char*)alloca(sizeof(a)); \
memcpy(__tmp, &(a), sizeof(a)); \ memcpy(__tmp, &(a), sizeof(a)); \
memcpy(&(a), &(b), sizeof(a)); \ memcpy(&(a), &(b), sizeof(a)); \
memcpy(&(b), __tmp, sizeof(a)); \ memcpy(&(b), __tmp, sizeof(a)); \

View File

@ -22,6 +22,8 @@ extern "C" {
typedef wchar_t TdWchar; typedef wchar_t TdWchar;
typedef int32_t TdUcs4; typedef int32_t TdUcs4;
typedef void *iconv_t;
typedef enum { M2C = 0, C2M } ConvType;
// If the error is in a third-party library, place this header file under the third-party library header file. // If the error is in a third-party library, place this header file under the third-party library header file.
// When you want to use this feature, you should find or add the same function in the following section. // When you want to use this feature, you should find or add the same function in the following section.
@ -59,7 +61,10 @@ int64_t taosStr2int64(const char *str);
int32_t taosConvInit(void); int32_t taosConvInit(void);
void taosConvDestroy(); void taosConvDestroy();
iconv_t taosAcquireConv(int32_t *idx, ConvType type);
void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type);
int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs); int32_t taosUcs4ToMbs(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs);
int32_t taosUcs4ToMbsEx(TdUcs4 *ucs4, int32_t ucs4_max_len, char *mbs, iconv_t conv);
bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len); bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len);
int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes);
TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); TdUcs4 *tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4);

View File

@ -172,7 +172,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E) #define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X022E)
#define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F) #define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022F)
#define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230) #define TSDB_CODE_TSC_STMT_CACHE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0230)
#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231) #define TSDB_CODE_TSC_ENCODE_PARAM_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231)
#define TSDB_CODE_TSC_ENCODE_PARAM_NULL TAOS_DEF_ERROR_CODE(0, 0X0232)
#define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X02FF)
// mnode-common // mnode-common
#define TSDB_CODE_MND_REQ_REJECTED TAOS_DEF_ERROR_CODE(0, 0x0300) #define TSDB_CODE_MND_REQ_REJECTED TAOS_DEF_ERROR_CODE(0, 0x0300)
@ -273,6 +275,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E)
#define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F)
#define TSDB_CODE_MND_FIELD_VALUE_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x0370) #define TSDB_CODE_MND_FIELD_VALUE_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x0370)
#define TSDB_CODE_MND_COLUMN_COMPRESS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0371)
// mnode-func // mnode-func
@ -423,6 +426,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481) #define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481)
#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482) #define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482)
#define TSDB_CODE_MND_INVALID_DROP_TSMA TAOS_DEF_ERROR_CODE(0, 0x0485)
#define TSDB_CODE_MND_MAX_TSMA_NUM_EXCEEDED TAOS_DEF_ERROR_CODE(0, 0x0486)
// mnode-tag-indxe // mnode-tag-indxe
@ -475,6 +480,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_VND_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0533) // internal #define TSDB_CODE_VND_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0533) // internal
#define TSDB_CODE_VND_DIR_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0534) #define TSDB_CODE_VND_DIR_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0534)
#define TSDB_CODE_VND_META_DATA_UNSAFE_DELETE TAOS_DEF_ERROR_CODE(0, 0x0535) #define TSDB_CODE_VND_META_DATA_UNSAFE_DELETE TAOS_DEF_ERROR_CODE(0, 0x0535)
#define TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0536)
#define TSDB_CODE_VND_ARB_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0536) // internal #define TSDB_CODE_VND_ARB_NOT_SYNCED TAOS_DEF_ERROR_CODE(0, 0x0536) // internal
// tsdb // tsdb
@ -547,6 +553,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_QWORKER_QUIT TAOS_DEF_ERROR_CODE(0, 0x0730) #define TSDB_CODE_QRY_QWORKER_QUIT TAOS_DEF_ERROR_CODE(0, 0x0730)
#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731) #define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731)
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732) #define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
#define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733)
// grant // grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800) #define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)
@ -762,12 +769,18 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E) #define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E)
#define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F) #define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F)
#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670) #define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670)
#define TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x2671) #define TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2671)
#define TSDB_CODE_PAR_SECOND_COL_PK TAOS_DEF_ERROR_CODE(0, 0x2672) #define TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR TAOS_DEF_ERROR_CODE(0, 0x2672)
#define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2673) #define TSDB_CODE_PAR_INVALID_WIN_OFFSET_UNIT TAOS_DEF_ERROR_CODE(0, 0x2673)
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x2674) #define TSDB_CODE_PAR_VALID_PRIM_TS_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x2674)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x2675) #define TSDB_CODE_PAR_ORDERBY_UNKNOWN_EXPR TAOS_DEF_ERROR_CODE(0, 0x2675)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x2676) #define TSDB_CODE_PAR_NOT_WIN_FUNC TAOS_DEF_ERROR_CODE(0, 0x2676)
#define TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x2677)
#define TSDB_CODE_PAR_SECOND_COL_PK TAOS_DEF_ERROR_CODE(0, 0x2678)
#define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2679)
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x267A)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x267B)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x267C)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner //planner
@ -816,6 +829,12 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3103) #define TSDB_CODE_TSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3103)
#define TSDB_CODE_TSMA_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x3104) #define TSDB_CODE_TSMA_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x3104)
#define TSDB_CODE_TSMA_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x3105) #define TSDB_CODE_TSMA_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x3105)
#define TSDB_CODE_TSMA_INVALID_TB TAOS_DEF_ERROR_CODE(0, 0x3106)
#define TSDB_CODE_TSMA_INVALID_INTERVAL TAOS_DEF_ERROR_CODE(0, 0x3107)
#define TSDB_CODE_TSMA_INVALID_FUNC_PARAM TAOS_DEF_ERROR_CODE(0, 0x3108)
#define TSDB_CODE_TSMA_UNSUPPORTED_FUNC TAOS_DEF_ERROR_CODE(0, 0x3109)
#define TSDB_CODE_TSMA_MUST_BE_DROPPED TAOS_DEF_ERROR_CODE(0, 0x3110)
#define TSDB_CODE_TSMA_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x3111)
//rsma //rsma
#define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150) #define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150)

View File

@ -83,6 +83,7 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight);
int32_t comparestrRegexMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexMatch(const void *pLeft, const void *pRight);
int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight); int32_t comparestrRegexNMatch(const void *pLeft, const void *pRight);
void DestoryThreadLocalRegComp();
int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight); int32_t comparewcsRegexMatch(const void *pLeft, const void *pRight);
int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight); int32_t comparewcsRegexNMatch(const void *pLeft, const void *pRight);

View File

@ -24,6 +24,20 @@
extern "C" { extern "C" {
#endif #endif
// start compress flag
// |----l1 compAlg----|-----l2 compAlg---|---level--|
// |------8bit--------|------16bit-------|---8bit---|
#define COMPRESS_L1_TYPE_U32(type) (((type) >> 24) & 0xFF)
#define COMPRESS_L2_TYPE_U32(type) (((type) >> 8) & 0xFFFF)
#define COMPRESS_L2_TYPE_LEVEL_U32(type) ((type)&0xFF)
// compress flag
// |----l2lel--|----l2Alg---|---l1Alg--|
// |----2bit---|----3bit----|---3bit---|
#define COMPRESS_L1_TYPE_U8(type) ((type)&0x07)
#define COMPRESS_L2_TYPE_U8(type) (((type) >> 3) & 0x07)
#define COMPRESS_L2_TYPE_LEVEL_U8(type) (((type) >> 6) & 0x03)
// end compress flag
#define COMP_OVERFLOW_BYTES 2 #define COMP_OVERFLOW_BYTES 2
#define BITS_PER_BYTE 8 #define BITS_PER_BYTE 8
// Masks // Masks
@ -58,15 +72,17 @@ extern "C" {
#ifdef TD_TSZ #ifdef TD_TSZ
extern bool lossyFloat; extern bool lossyFloat;
extern bool lossyDouble; extern bool lossyDouble;
int32_t tsCompressInit(char* lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals, uint32_t intervals, int32_t tsCompressInit(char *lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals,
int32_t ifAdtFse, const char* compressor); uint32_t intervals, int32_t ifAdtFse, const char *compressor);
void tsCompressExit(); void tsCompressExit();
int32_t tsCompressFloatLossyImp(const char *const input, const int32_t nelements, char *const output); int32_t tsCompressFloatLossyImp(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressFloatLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output); int32_t tsDecompressFloatLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output);
int32_t tsCompressDoubleLossyImp(const char *const input, const int32_t nelements, char *const output); int32_t tsCompressDoubleLossyImp(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressDoubleLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output); int32_t tsDecompressDoubleLossyImp(const char *const input, int32_t compressedSize, const int32_t nelements,
char *const output);
static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm, char *const output, int32_t outputSize, char algorithm,
@ -139,8 +155,58 @@ int32_t getWordLength(char type);
int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type);
int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressTimestampAvx512(const char* const input, const int32_t nelements, char *const output, bool bigEndian); int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output,
int32_t tsDecompressTimestampAvx2(const char* const input, const int32_t nelements, char *const output, bool bigEndian); bool bigEndian);
int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian);
/*************************************************************************
* REGULAR COMPRESSION 2
*************************************************************************/
int32_t tsCompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsDecompressTimestamp2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressFloat2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressDouble2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressString2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressString2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressBool2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressBool2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressTinyint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressTinyint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressSmallint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsDecompressSmallint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
int32_t tsCompressInt2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressInt2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsCompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg, void *pBuf,
int32_t nBuf);
int32_t tsDecompressBigint2(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint32_t cmprAlg,
void *pBuf, int32_t nBuf);
// for internal usage
int32_t getWordLength(char type);
int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type);
int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output);
int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelements, char *const output,
bool bigEndian);
int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelements, char *const output, bool bigEndian);
/************************************************************************* /*************************************************************************
* STREAM COMPRESSION * STREAM COMPRESSION
@ -153,6 +219,79 @@ int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin); int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin);
int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData);
typedef int32_t (*__data_compress_init)(char *lossyColumns, float fPrecision, double dPrecision, uint32_t maxIntervals,
uint32_t intervals, int32_t ifAdtFse, const char *compressor);
typedef int32_t (*__data_compress_l1_fn_t)(const char *const input, const int32_t nelements, char *const output,
const char type);
typedef int32_t (*__data_decompress_l1_fn_t)(const char *const input, const int32_t nelements, char *const output,
const char type);
typedef int32_t (*__data_compress_l2_fn_t)(const char *const input, const int32_t nelements, char *const output,
int32_t outputSize, const char type, int8_t level);
typedef int32_t (*__data_decompress_l2_fn_t)(const char *const input, const int32_t nelements, char *const output,
int32_t outputSize, const char type);
typedef struct {
char *name;
__data_compress_init initFn;
__data_compress_l1_fn_t comprFn;
__data_decompress_l1_fn_t decomprFn;
} TCompressL1FnSet;
typedef struct {
char *name;
__data_compress_init initFn;
__data_compress_l2_fn_t comprFn;
__data_decompress_l2_fn_t decomprFn;
} TCompressL2FnSet;
typedef struct {
int8_t type;
int8_t level;
__data_compress_init initFn;
__data_compress_l1_fn_t l1CmprFn;
__data_decompress_l1_fn_t l1DecmprFn;
__data_compress_l2_fn_t l2CmprFn;
__data_decompress_l2_fn_t l2DecmprFn;
} TCompressPara;
typedef enum L1Compress {
L1_UNKNOWN = 0,
L1_SIMPLE_8B,
L1_XOR,
L1_RLE,
L1_DELTAD,
L1_DISABLED = 0xFF,
} EL1CompressFuncType;
typedef enum L2Compress {
L2_UNKNOWN = 0,
L2_LZ4,
L2_ZLIB,
L2_ZSTD,
L2_TSZ,
L2_XZ,
L2_DISABLED = 0xFF,
} EL2ComressFuncType;
int32_t tcompressDebug(uint32_t cmprAlg, uint8_t *l1Alg, uint8_t *l2Alg, uint8_t *level);
#define DEFINE_VAR(cmprAlg) \
uint8_t l1 = COMPRESS_L1_TYPE_U32(cmprAlg); \
uint8_t l2 = COMPRESS_L2_TYPE_U32(cmprAlg); \
uint8_t lvl = COMPRESS_L2_TYPE_LEVEL_U32(cmprAlg);
#define SET_COMPRESS(l1, l2, lvl, cmpr) \
do { \
(cmpr) &= 0x00FFFFFF; \
(cmpr) |= ((l1) << 24); \
(cmpr) &= 0xFF0000FF; \
(cmpr) |= ((l2) << 8); \
(cmpr) &= 0xFFFFFF00; \
(cmpr) |= (lvl); \
} while (0)
int8_t tUpdateCompress(uint32_t oldCmpr, uint32_t newCmpr, uint8_t l2Disabled, uint8_t lvlDisabled, uint8_t lvlDefault,
uint32_t *dst);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -51,11 +51,7 @@ typedef enum {
CFG_DTYPE_TIMEZONE CFG_DTYPE_TIMEZONE
} ECfgDataType; } ECfgDataType;
typedef enum { typedef enum { CFG_SCOPE_SERVER, CFG_SCOPE_CLIENT, CFG_SCOPE_BOTH } ECfgScopeType;
CFG_SCOPE_SERVER,
CFG_SCOPE_CLIENT,
CFG_SCOPE_BOTH
} ECfgScopeType;
typedef enum { typedef enum {
CFG_DYN_NONE = 0, CFG_DYN_NONE = 0,
@ -138,6 +134,7 @@ void cfgDumpItemValue(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *p
void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen); void cfgDumpItemScope(SConfigItem *pItem, char *buf, int32_t bufSize, int32_t *pLen);
void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump);
void cfgDumpCfgS3(SConfig *pCfg, bool tsc, bool dump);
int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char *apolloUrl);

View File

@ -149,7 +149,7 @@ typedef enum EOperatorType {
OP_TYPE_BIT_OR, OP_TYPE_BIT_OR,
// binary comparison operator // binary comparison operator
OP_TYPE_GREATER_THAN = 40, OP_TYPE_GREATER_THAN = 40, // MUST KEEP IT FIRST AT COMPARE SECTION
OP_TYPE_GREATER_EQUAL, OP_TYPE_GREATER_EQUAL,
OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_THAN,
OP_TYPE_LOWER_EQUAL, OP_TYPE_LOWER_EQUAL,
@ -170,6 +170,7 @@ typedef enum EOperatorType {
OP_TYPE_IS_NOT_TRUE, OP_TYPE_IS_NOT_TRUE,
OP_TYPE_IS_NOT_FALSE, OP_TYPE_IS_NOT_FALSE,
OP_TYPE_IS_NOT_UNKNOWN, OP_TYPE_IS_NOT_UNKNOWN,
OP_TYPE_COMPARE_MAX_VALUE = 149, // MUST KEEP IT LAST AT COMPARE SECTION
// json operator // json operator
OP_TYPE_JSON_GET_VALUE = 150, OP_TYPE_JSON_GET_VALUE = 150,
@ -197,8 +198,8 @@ typedef enum ELogicConditionType {
#define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788 #define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788
// ACCOUNT is a 32 bit positive integer // ACCOUNT is a 32 bit positive integer
// this is the length of its string representation, including the terminator zero // this is the length of its string representation, including the terminator zero
#define TSDB_ACCT_ID_LEN 11 #define TSDB_ACCT_ID_LEN 11
#define TSDB_NODE_ID_LEN 11 #define TSDB_NODE_ID_LEN 11
#define TSDB_VGROUP_ID_LEN 11 #define TSDB_VGROUP_ID_LEN 11
#define TSDB_MAX_COLUMNS 4096 #define TSDB_MAX_COLUMNS 4096
@ -293,8 +294,8 @@ typedef enum ELogicConditionType {
#define TSDB_DNODE_CONFIG_LEN 128 #define TSDB_DNODE_CONFIG_LEN 128
#define TSDB_DNODE_VALUE_LEN 256 #define TSDB_DNODE_VALUE_LEN 256
#define TSDB_CLUSTER_VALUE_LEN 1000 #define TSDB_CLUSTER_VALUE_LEN 1000
#define TSDB_GRANT_LOG_COL_LEN 15600 #define TSDB_GRANT_LOG_COL_LEN 15600
#define TSDB_ACTIVE_KEY_LEN 109 #define TSDB_ACTIVE_KEY_LEN 109
#define TSDB_CONN_ACTIVE_KEY_LEN 255 #define TSDB_CONN_ACTIVE_KEY_LEN 255
@ -416,6 +417,16 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_SUFFIX 0 #define TSDB_DEFAULT_HASH_SUFFIX 0
#define TSDB_MIN_S3_CHUNK_SIZE (128 * 1024)
#define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024)
#define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024)
#define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute
#define TSDB_MAX_S3_KEEP_LOCAL (365000 * 1440)
#define TSDB_DEFAULT_S3_KEEP_LOCAL (3650 * 1440)
#define TSDB_MIN_S3_COMPACT 0
#define TSDB_MAX_S3_COMPACT 1
#define TSDB_DEFAULT_S3_COMPACT 0
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1
#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_REP_DEF_DB_WAL_RET_PERIOD 3600
#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600 #define TSDB_REPS_DEF_DB_WAL_RET_PERIOD 3600
@ -530,12 +541,12 @@ enum {
SND_WORKER_TYPE__UNIQUE, SND_WORKER_TYPE__UNIQUE,
}; };
#define DEFAULT_HANDLE 0 #define DEFAULT_HANDLE 0
#define MNODE_HANDLE 1 #define MNODE_HANDLE 1
#define QNODE_HANDLE -1 #define QNODE_HANDLE -1
#define SNODE_HANDLE -2 #define SNODE_HANDLE -2
#define VNODE_HANDLE -3 #define VNODE_HANDLE -3
#define CLIENT_HANDLE -5 #define CLIENT_HANDLE -5
#define TSDB_CONFIG_OPTION_LEN 32 #define TSDB_CONFIG_OPTION_LEN 32
#define TSDB_CONFIG_VALUE_LEN 64 #define TSDB_CONFIG_VALUE_LEN 64
@ -555,13 +566,13 @@ enum {
// sort page size by default // sort page size by default
#define DEFAULT_PAGESIZE 4096 #define DEFAULT_PAGESIZE 4096
#define VNODE_TIMEOUT_SEC 60 #define VNODE_TIMEOUT_SEC 60
#define MNODE_TIMEOUT_SEC 60 #define MNODE_TIMEOUT_SEC 60
#define MONITOR_TABLENAME_LEN 200 #define MONITOR_TABLENAME_LEN 200
#define MONITOR_TAG_NAME_LEN 100 #define MONITOR_TAG_NAME_LEN 100
#define MONITOR_TAG_VALUE_LEN 300 #define MONITOR_TAG_VALUE_LEN 300
#define MONITOR_METRIC_NAME_LEN 100 #define MONITOR_METRIC_NAME_LEN 100
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -0,0 +1,219 @@
#!/bin/bash
log_date_time=`date +%m%d%Y-%H%M`
log=${log_date_time}.log
dataDir=/data
logDir=/data/taos/log
echo "#########collect sysinfo############"
cat /etc/os-release
free -g
cat /proc/cpuinfo |grep processor
uname -a
df -h
lsblk
blkid
cat /etc/fstab
mount
cat /etc/sysctl.conf
cat /etc/security/limits.conf
sysctl -p
######check hostname #####
echo "#########hostname"
hostname
hostname -f
echo "#########/etc/hostname"
cat /etc/hostname
echo "#########/etc/hosts"
cat /etc/hosts
echo "#########Step 1############"
echo "$#"
######single node without config file#####
if [ "$#" -eq 0 ] ;then
echo "#########This is for single node !!"
echo "#########Please provide config.ini file for cluster installation !!"
first_ep=`hostname -f`:6030
second_ep=""
fi
echo "#########Step 2############"
######use config file to configre cluster!#####
echo $1
config_file=$1
if [ "$#" -eq 1 ] ;then
echo "#########Use $1 file to configure cluster."
echo "#########The configure of cluster"
cat $config_file
first_ep=`sed -n '1p' $config_file |awk '{print $2}'`:6030
second_ep=`sed -n '2p' $config_file |awk '{print $2}'`:6030
fi
echo "#########Step 3############"
################################################################
echo "#########create data directory"
mkdir -p ${dataDir}/taos/{data,core,tmp,soft}
mkdir -p ${logDir}
chmod 777 ${dataDir}/taos/tmp
chmod 777 ${logDir}
echo "#########install suggest package and disable firewall"
os=$(cat /etc/os-release| grep PRETTY_NAME | awk '{print $1}'|awk -F '=' '{print $2}' | sed 's/"//g')
if [ $os = 'Ubuntu' ]
then
echo "This is Ununtu!"
#apt install screen tmux gdb fio iperf3 sysstat net-tools ntp tree wget
ufw status
ufw stop
ufw disable
elif [ $os = 'CentOS' -o $os = 'Red' ]
then
echo "This is Centos/Red"
#yum install -y screen tmux gdb fio iperf3 sysstat net-tools ntp tree wget
systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld
else
echo "####$os"
fi
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
echo "#########system tunning"
echo "fs.nr_open = 10485760" >>/etc/sysctl.conf
echo "net.core.somaxconn=10240" >> /etc/sysctl.conf
echo "net.core.netdev_max_backlog=20480" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_syn_backlog=10240" >> /etc/sysctl.conf
echo "net.ipv4.tcp_retries2=5" >> /etc/sysctl.conf
echo "net.ipv4.tcp_syn_retries=2" >> /etc/sysctl.conf
echo "net.ipv4.tcp_synack_retries=2" >> /etc/sysctl.conf
echo "net.ipv4.tcp_tw_reuse=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_tw_recycle=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_keepalive_time=600" >> /etc/sysctl.conf
echo "net.ipv4.tcp_abort_on_overflow=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_tw_buckets=5000" >> /etc/sysctl.conf
echo "net.ipv4.ip_local_port_range=10000 60999" >> /etc/sysctl.conf
echo "* soft ntaosc 65536" >>/etc/security/limits.conf
echo "* soft nofile 1048576" >>/etc/security/limits.conf
echo "* soft stack 65536" >>/etc/security/limits.conf
echo "* hard ntaosc 65536" >>/etc/security/limits.conf
echo "* hard nofile 1048576" >>/etc/security/limits.conf
echo "* hard stack 65536" >>/etc/security/limits.conf
echo "root soft ntaosc 65536" >>/etc/security/limits.conf
echo "root soft nofile 1048576" >>/etc/security/limits.conf
echo "root soft stack 65536" >>/etc/security/limits.conf
echo "root hard ntaosc 65536" >>/etc/security/limits.conf
echo "root hard nofile 1048576" >>/etc/security/limits.conf
echo "root hard stack 65536" >>/etc/security/limits.conf
echo "ulimit -c unlimited" >>/etc/profile
echo "kernel.core_pattern=${dataDir}/taos/core/core-%e-%p" >>/etc/sysctl.conf
sysctl -p
date
####to do check date timezone and ntp########
#cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
#NTP
#SWAP
#reboot
echo "#########install TDengine"
./install.sh -e no
pwd
echo "#########Create /etc/taos/taos.cfg"
mv /etc/taos/taos.cfg /etc/taos/taos.cfg.bak
cat /proc/cpuinfo | grep "processor" | wc -l
num=`cat /proc/cpuinfo | grep "processor" | wc -l`
cpunum=$(expr $num + $num)
host_name=`hostname -f`
#####vi /etc/taos/taos.cfg
echo "firstEp $first_ep
secondEp $second_ep
fqdn $host_name
supportVnodes $cpunum
logDir $logDir
dataDir ${dataDir}/taos/data
tempDir ${dataDir}/taos/tmp
keepColumnName 0
maxNumOfDistinctRes 10000000
timezone UTC-8
locale en_US.UTF-8
charset UTF-8
maxShellConns 100000
maxConnections 100000
audit 0
auditFqdn localhost
monitor 1
monitorFqdn localhost
logKeepDays 10
debugflag 131
shellActivityTimer 120
numOfRpcSessions 30000" >> /etc/taos/taos.cfg
########################
########################tunning taosadapter and taosx log directory
sed -i "/^\#*path/c\path\ =\ \"${logDir}\"" /etc/taos/taosadapter.toml
sed -i "/^\#*logs_home/c\logs_home\ =\ \"${logDir}\"" /etc/taos/taosx.toml
##########################single nodes
echo "#########Start toasd"
./start-all.sh
date
if [ "$#" -eq 0 ] ;then
sleep 5
taos -s "show dnodes;"
taos -s "show cluster\G;"
echo "########"
taosd -k
curl -u root:taosdata ${host_name}:6041/rest/sql -d "select server_version()"
echo -e "\r"
echo "#########Installation completed########"
echo "#########Suggest reboot OS##########"
date
exit
fi
##########################use $config_file to config cluster
if test "$host_name"x = `sed -n '1p' $config_file |awk '{print $2}'`x ;then
echo "#########Add dnodes and mnodes"
while read ip_address host_name
do
echo "$host_name"
taos -s "create dnode '$host_name:6030';"
sleep 5
done <$config_file
fi
host_name=`hostname -f`
if test "$host_name"x = `sed -n '3p' $config_file |awk '{print $2}'`x ;then
echo "#########Add mnodes"
taos -s "create mnode on dnode 2;"
sleep 5
taos -s "create mnode on dnode 3;"
fi
sleep 5
taos -s "show dnodes;"
taos -s "show mnodes;"
taos -s "show grants\G;"
taos -s "show cluster\G;"
echo "########"
taosd -k
echo "#########Test TDengine"
curl -u root:taosdata ${host_name}:6041/rest/sql -d "select server_version()"
echo -e "\r"
echo "#########Installation completed########"
echo "#########Suggest reboot OS##########"
date
cd ..

View File

@ -234,6 +234,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
catalogUpdateDbCfg(pCatalog, rsp->cfgRsp->db, rsp->cfgRsp->dbId, rsp->cfgRsp); catalogUpdateDbCfg(pCatalog, rsp->cfgRsp->db, rsp->cfgRsp->dbId, rsp->cfgRsp);
rsp->cfgRsp = NULL; rsp->cfgRsp = NULL;
} }
if (rsp->pTsmaRsp) {
if (rsp->pTsmaRsp->pTsmas) {
for (int32_t i = 0; i < rsp->pTsmaRsp->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(rsp->pTsmaRsp->pTsmas, i);
catalogAsyncUpdateTSMA(pCatalog, &pTsma, rsp->dbTsmaVersion);
}
taosArrayClear(rsp->pTsmaRsp->pTsmas);
}
}
} }
_return: _return:
@ -327,6 +336,33 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog* pCatalog) {
int32_t code = 0;
STSMAHbRsp hbRsp = {0};
if (tDeserializeTSMAHbRsp(value, valueLen, &hbRsp)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int32_t numOfTsma = taosArrayGetSize(hbRsp.pTsmas);
for (int32_t i = 0; i < numOfTsma; ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(hbRsp.pTsmas, i);
if (!pTsmaInfo->pFuncs) {
tscDebug("hb to remove tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
catalogRemoveTSMA(pCatalog, pTsmaInfo);
tFreeAndClearTableTSMAInfo(pTsmaInfo);
} else {
tscDebug("hb to update tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
catalogUpdateTSMA(pCatalog, &pTsmaInfo);
tFreeAndClearTableTSMAInfo(pTsmaInfo);
}
}
taosArrayDestroy(hbRsp.pTsmas);
return TSDB_CODE_SUCCESS;
}
static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
for (int32_t i = 0; i < kvNum; ++i) { for (int32_t i = 0; i < kvNum; ++i) {
@ -379,6 +415,13 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p
break; break;
} }
#endif #endif
case HEARTBEAT_KEY_TSMA: {
if (kv->valueLen <= 0 || !kv->value) {
tscError("Invalid tsma info, len: %d, value: %p", kv->valueLen, kv->value);
}
hbprocessTSMARsp(kv->value, kv->valueLen, pCatalog);
break;
}
default: default:
tscError("invalid hb key type:%d", kv->key); tscError("invalid hb key type:%d", kv->key);
break; break;
@ -763,6 +806,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
db->cfgVersion = htonl(db->cfgVersion); db->cfgVersion = htonl(db->cfgVersion);
db->numOfTable = htonl(db->numOfTable); db->numOfTable = htonl(db->numOfTable);
db->stateTs = htobe64(db->stateTs); db->stateTs = htobe64(db->stateTs);
db->tsmaVersion = htonl(db->tsmaVersion);
} }
SKv kv = { SKv kv = {
@ -871,6 +915,39 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t hbGetExpiredTSMAInfo(SClientHbKey* connKey, struct SCatalog* pCatalog, SClientHbReq* pReq) {
int32_t code = 0;
uint32_t tsmaNum = 0;
STSMAVersion *tsmas = NULL;
code = catalogGetExpiredTsmas(pCatalog, &tsmas, &tsmaNum);
if (code) {
taosMemoryFree(tsmas);
return code;
}
if (tsmaNum <= 0) {
taosMemoryFree(tsmas);
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < tsmaNum; ++i) {
STSMAVersion* tsma = &tsmas[i];
tsma->dbId = htobe64(tsma->dbId);
tsma->tsmaId = htobe64(tsma->tsmaId);
tsma->version = htonl(tsma->version);
}
tscDebug("hb got %d expred tsmas, valueLen: %lu", tsmaNum, sizeof(STSMAVersion) * tsmaNum);
if (!pReq->info) {
pReq->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
}
SKv kv = {.key = HEARTBEAT_KEY_TSMA, .valueLen = sizeof(STSMAVersion) * tsmaNum, .value = tsmas};
taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
return TSDB_CODE_SUCCESS;
}
int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) {
SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId));
@ -935,6 +1012,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
return code; return code;
} }
#endif #endif
code = hbGetExpiredTSMAInfo(connKey, pCatalog, req);
} else { } else {
req->app.appId = 0; req->app.appId = 0;
} }

View File

@ -1041,6 +1041,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->retry, pRequest->requestId); tstrerror(code), pRequest->retry, pRequest->requestId);
removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
restartAsyncQuery(pRequest, code); restartAsyncQuery(pRequest, code);
return; return;
} }
@ -1803,6 +1804,10 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) {
} }
static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) {
int32_t idx = -1;
iconv_t conv = taosAcquireConv(&idx, C2M);
if (!conv) return TSDB_CODE_TSC_INTERNAL_ERROR;
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
int32_t type = pResultInfo->fields[i].type; int32_t type = pResultInfo->fields[i].type;
int32_t bytes = pResultInfo->fields[i].bytes; int32_t bytes = pResultInfo->fields[i].bytes;
@ -1810,6 +1815,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) { if (type == TSDB_DATA_TYPE_NCHAR && colLength[i] > 0) {
char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]); char* p = taosMemoryRealloc(pResultInfo->convertBuf[i], colLength[i]);
if (p == NULL) { if (p == NULL) {
taosReleaseConv(idx, conv, C2M);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -1820,12 +1826,13 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
if (pCol->offset[j] != -1) { if (pCol->offset[j] != -1) {
char* pStart = pCol->offset[j] + pCol->pData; char* pStart = pCol->offset[j] + pCol->pData;
int32_t len = taosUcs4ToMbs((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p)); int32_t len = taosUcs4ToMbsEx((TdUcs4*)varDataVal(pStart), varDataLen(pStart), varDataVal(p), conv);
if (len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) { if (len < 0 || len > bytes || (p + len) >= (pResultInfo->convertBuf[i] + colLength[i])) {
tscError( tscError(
"doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + " "doConvertUCS4 error, invalid data. len:%d, bytes:%d, (p + len):%p, (pResultInfo->convertBuf[i] + "
"colLength[i]):%p", "colLength[i]):%p",
len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i])); len, bytes, (p + len), (pResultInfo->convertBuf[i] + colLength[i]));
taosReleaseConv(idx, conv, C2M);
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
@ -1839,7 +1846,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int
pResultInfo->row[i] = pResultInfo->pCol[i].pData; pResultInfo->row[i] = pResultInfo->pCol[i].pData;
} }
} }
taosReleaseConv(idx, conv, C2M);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2619,8 +2626,7 @@ static void fetchCallback(void* pResult, void* param, int32_t code) {
setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4); setQueryResultFromRsp(pResultInfo, (const SRetrieveTableRsp*)pResultInfo->pData, pResultInfo->convertUcs4);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0; pResultInfo->numOfRows = 0;
pRequest->code = code; tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(pRequest->code),
tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId); pRequest->requestId);
} else { } else {
tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64, tscDebug("0x%" PRIx64 " fetch results, numOfRows:%" PRId64 " total Rows:%" PRId64 ", complete:%d, reqId:0x%" PRIx64,

View File

@ -936,6 +936,8 @@ int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) {
pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL); pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL);
pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL); pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL);
pTarget->pView = taosArrayDup(pSrc->pView, NULL); pTarget->pView = taosArrayDup(pSrc->pView, NULL);
pTarget->pTableTSMAs = taosArrayDup(pSrc->pTableTSMAs, NULL);
pTarget->pTSMAs = taosArrayDup(pSrc->pTSMAs, NULL);
pTarget->qNodeRequired = pSrc->qNodeRequired; pTarget->qNodeRequired = pSrc->qNodeRequired;
pTarget->dNodeRequired = pSrc->dNodeRequired; pTarget->dNodeRequired = pSrc->dNodeRequired;
pTarget->svrVerRequired = pSrc->svrVerRequired; pTarget->svrVerRequired = pSrc->svrVerRequired;

View File

@ -712,11 +712,13 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) {
goto end; goto end;
} }
// build create stable // build create stable
pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField)); pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SFieldWithOptions));
for (int32_t i = 0; i < req.schemaRow.nCols; i++) { for (int32_t i = 0; i < req.schemaRow.nCols; i++) {
SSchema* pSchema = req.schemaRow.pSchema + i; SSchema* pSchema = req.schemaRow.pSchema + i;
SField field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes}; SFieldWithOptions field = {.type = pSchema->type, .flags = pSchema->flags, .bytes = pSchema->bytes};
strcpy(field.name, pSchema->name); strcpy(field.name, pSchema->name);
// todo get active compress param
setDefaultOptionsForField(&field);
taosArrayPush(pReq.pColumns, &field); taosArrayPush(pReq.pColumns, &field);
} }
pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField));

View File

@ -108,7 +108,7 @@ int64_t smlToMilli[] = {3600000LL, 60000LL, 1000LL};
int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1}; int64_t smlFactorNS[] = {NANOSECOND_PER_MSEC, NANOSECOND_PER_USEC, 1};
int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL}; int64_t smlFactorS[] = {1000LL, 1000000LL, 1000000000LL};
static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const char* pTabName, AUTH_TYPE type){ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo *conn, const char *pTabName, AUTH_TYPE type) {
SUserAuthInfo pAuth = {0}; SUserAuthInfo pAuth = {0};
snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user); snprintf(pAuth.user, sizeof(pAuth.user), "%s", info->taos->user);
if (NULL == pTabName) { if (NULL == pTabName) {
@ -124,8 +124,9 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const cha
code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes); code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes);
nodesDestroyNode(authRes.pCond[AUTH_RES_BASIC]); nodesDestroyNode(authRes.pCond[AUTH_RES_BASIC]);
return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; return (code == TSDB_CODE_SUCCESS)
? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED)
: code;
} }
int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) { int32_t smlBuildInvalidDataMsg(SSmlMsgBuf *pBuf, const char *msg1, const char *msg2) {
@ -182,7 +183,7 @@ cleanup:
return NULL; return NULL;
} }
void smlBuildTsKv(SSmlKv *kv, int64_t ts){ void smlBuildTsKv(SSmlKv *kv, int64_t ts) {
kv->key = tsSmlTsDefaultName; kv->key = tsSmlTsDefaultName;
kv->keyLen = strlen(tsSmlTsDefaultName); kv->keyLen = strlen(tsSmlTsDefaultName);
kv->type = TSDB_DATA_TYPE_TIMESTAMP; kv->type = TSDB_DATA_TYPE_TIMESTAMP;
@ -190,10 +191,10 @@ void smlBuildTsKv(SSmlKv *kv, int64_t ts){
kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; kv->length = (size_t)tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes;
} }
SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement){ SSmlSTableMeta *smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currElement) {
SSmlSTableMeta* sMeta = NULL; SSmlSTableMeta *sMeta = NULL;
char *measure = currElement->measure; char *measure = currElement->measure;
int measureLen = currElement->measureLen; int measureLen = currElement->measureLen;
if (currElement->measureEscaped) { if (currElement->measureEscaped) {
measure = (char *)taosMemoryMalloc(measureLen); measure = (char *)taosMemoryMalloc(measureLen);
memcpy(measure, currElement->measure, measureLen); memcpy(measure, currElement->measure, measureLen);
@ -211,7 +212,7 @@ SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currEleme
return sMeta; return sMeta;
} }
sMeta = smlBuildSTableMeta(info->dataFormat); sMeta = smlBuildSTableMeta(info->dataFormat);
if(sMeta == NULL){ if (sMeta == NULL) {
taosMemoryFreeClear(pTableMeta); taosMemoryFreeClear(pTableMeta);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return sMeta; return sMeta;
@ -223,22 +224,23 @@ SSmlSTableMeta* smlBuildSuperTableInfo(SSmlHandle *info, SSmlLineInfo *currEleme
SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type}; SSmlKv kv = {.key = col->name, .keyLen = strlen(col->name), .type = col->type};
if (col->type == TSDB_DATA_TYPE_NCHAR) { if (col->type == TSDB_DATA_TYPE_NCHAR) {
kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; kv.length = (col->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
} else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY || col->type == TSDB_DATA_TYPE_VARBINARY) { } else if (col->type == TSDB_DATA_TYPE_BINARY || col->type == TSDB_DATA_TYPE_GEOMETRY ||
col->type == TSDB_DATA_TYPE_VARBINARY) {
kv.length = col->bytes - VARSTR_HEADER_SIZE; kv.length = col->bytes - VARSTR_HEADER_SIZE;
} else{ } else {
kv.length = col->bytes; kv.length = col->bytes;
} }
if(i < pTableMeta->tableInfo.numOfColumns){ if (i < pTableMeta->tableInfo.numOfColumns) {
taosArrayPush(sMeta->cols, &kv); taosArrayPush(sMeta->cols, &kv);
}else{ } else {
taosArrayPush(sMeta->tags, &kv); taosArrayPush(sMeta->tags, &kv);
} }
} }
return sMeta; return sMeta;
} }
bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv) {
// cnt begin 0, add ts so + 2 // cnt begin 0, add ts so + 2
if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) { if (unlikely(cnt + 2 > info->currSTableMeta->tableInfo.numOfColumns)) {
info->dataFormat = false; info->dataFormat = false;
@ -273,7 +275,7 @@ bool isSmlColAligned(SSmlHandle *info, int cnt, SSmlKv *kv){
return true; return true;
} }
bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv){ bool isSmlTagAligned(SSmlHandle *info, int cnt, SSmlKv *kv) {
if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) { if (unlikely(cnt + 1 > info->currSTableMeta->tableInfo.numOfTags)) {
goto END; goto END;
} }
@ -299,9 +301,9 @@ END:
return false; return false;
} }
int32_t smlJoinMeasureTag(SSmlLineInfo *elements){ int32_t smlJoinMeasureTag(SSmlLineInfo *elements) {
elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen); elements->measureTag = (char *)taosMemoryMalloc(elements->measureLen + elements->tagsLen);
if(elements->measureTag == NULL){ if (elements->measureTag == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
memcpy(elements->measureTag, elements->measure, elements->measureLen); memcpy(elements->measureTag, elements->measure, elements->measureLen);
@ -322,7 +324,7 @@ static bool smlIsPKTable(STableMeta *pTableMeta){
int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) { int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
bool isSameMeasure = IS_SAME_SUPER_TABLE; bool isSameMeasure = IS_SAME_SUPER_TABLE;
if(isSameMeasure) { if (isSameMeasure) {
return 0; return 0;
} }
SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen); SSmlSTableMeta **tmp = (SSmlSTableMeta **)taosHashGet(info->superTables, elements->measure, elements->measureLen);
@ -330,8 +332,8 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
SSmlSTableMeta *sMeta = NULL; SSmlSTableMeta *sMeta = NULL;
if (unlikely(tmp == NULL)) { if (unlikely(tmp == NULL)) {
sMeta = smlBuildSuperTableInfo(info, elements); sMeta = smlBuildSuperTableInfo(info, elements);
if(sMeta == NULL) return -1; if (sMeta == NULL) return -1;
}else{ } else {
sMeta = *tmp; sMeta = *tmp;
} }
ASSERT(sMeta != NULL); ASSERT(sMeta != NULL);
@ -346,7 +348,7 @@ int32_t smlProcessSuperTable(SSmlHandle *info, SSmlLineInfo *elements) {
return 0; return 0;
} }
int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements) {
SSmlTableInfo **oneTable = SSmlTableInfo **oneTable =
(SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen); (SSmlTableInfo **)taosHashGet(info->childTables, elements->measureTag, elements->measureTagsLen);
SSmlTableInfo *tinfo = NULL; SSmlTableInfo *tinfo = NULL;
@ -375,7 +377,7 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
} }
}else{ } else {
tinfo = *oneTable; tinfo = *oneTable;
} }
ASSERT(tinfo != NULL); ASSERT(tinfo != NULL);
@ -383,7 +385,7 @@ int32_t smlProcessChildTable(SSmlHandle *info, SSmlLineInfo *elements){
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv){ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs, SSmlKv *kv) {
if (info->dataFormat) { if (info->dataFormat) {
uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i); uDebug("SML:0x%" PRIx64 " smlParseEndTelnetJson format true, ts:%" PRId64, info->id, kvTs->i);
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
@ -411,7 +413,7 @@ int32_t smlParseEndTelnetJson(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs) {
if (info->dataFormat) { if (info->dataFormat) {
uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i); uDebug("SML:0x%" PRIx64 " smlParseEndLine format true, ts:%" PRId64, info->id, kvTs->i);
int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0); int32_t ret = smlBuildCol(info->currTableDataCtx, info->currSTableMeta->schema, kvTs, 0);
@ -434,31 +436,31 @@ int32_t smlParseEndLine(SSmlHandle *info, SSmlLineInfo *elements, SSmlKv *kvTs){
} }
static int32_t smlParseTableName(SArray *tags, char *childTableName) { static int32_t smlParseTableName(SArray *tags, char *childTableName) {
bool autoChildName = false; bool autoChildName = false;
size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter); size_t delimiter = strlen(tsSmlAutoChildTableNameDelimiter);
if(delimiter > 0){ if (delimiter > 0) {
size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1); size_t totalNameLen = delimiter * (taosArrayGetSize(tags) - 1);
for (int i = 0; i < taosArrayGetSize(tags); i++) { for (int i = 0; i < taosArrayGetSize(tags); i++) {
SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
totalNameLen += tag->length; totalNameLen += tag->length;
} }
if(totalNameLen < TSDB_TABLE_NAME_LEN){ if (totalNameLen < TSDB_TABLE_NAME_LEN) {
autoChildName = true; autoChildName = true;
} }
} }
if(autoChildName){ if (autoChildName) {
memset(childTableName, 0, TSDB_TABLE_NAME_LEN); memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
for (int i = 0; i < taosArrayGetSize(tags); i++) { for (int i = 0; i < taosArrayGetSize(tags); i++) {
SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i); SSmlKv *tag = (SSmlKv *)taosArrayGet(tags, i);
strncat(childTableName, tag->value, tag->length); strncat(childTableName, tag->value, tag->length);
if(i != taosArrayGetSize(tags) - 1){ if (i != taosArrayGetSize(tags) - 1) {
strcat(childTableName, tsSmlAutoChildTableNameDelimiter); strcat(childTableName, tsSmlAutoChildTableNameDelimiter);
} }
} }
if(tsSmlDot2Underline){ if (tsSmlDot2Underline) {
smlStrReplace(childTableName, strlen(childTableName)); smlStrReplace(childTableName, strlen(childTableName));
} }
}else{ } else {
size_t childTableNameLen = strlen(tsSmlChildTableName); size_t childTableNameLen = strlen(tsSmlChildTableName);
if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS; if (childTableNameLen <= 0) return TSDB_CODE_SUCCESS;
@ -468,7 +470,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) {
if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) {
memset(childTableName, 0, TSDB_TABLE_NAME_LEN); memset(childTableName, 0, TSDB_TABLE_NAME_LEN);
strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN));
if(tsSmlDot2Underline){ if (tsSmlDot2Underline) {
smlStrReplace(childTableName, strlen(childTableName)); smlStrReplace(childTableName, strlen(childTableName));
} }
taosArrayRemove(tags, i); taosArrayRemove(tags, i);
@ -484,15 +486,15 @@ int32_t smlSetCTableName(SSmlTableInfo *oneTable) {
smlParseTableName(oneTable->tags, oneTable->childTableName); smlParseTableName(oneTable->tags, oneTable->childTableName);
if (strlen(oneTable->childTableName) == 0) { if (strlen(oneTable->childTableName) == 0) {
SArray *dst = taosArrayDup(oneTable->tags, NULL); SArray *dst = taosArrayDup(oneTable->tags, NULL);
ASSERT(oneTable->sTableNameLen < TSDB_TABLE_NAME_LEN); ASSERT(oneTable->sTableNameLen < TSDB_TABLE_NAME_LEN);
char superName[TSDB_TABLE_NAME_LEN] = {0}; char superName[TSDB_TABLE_NAME_LEN] = {0};
RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName}; RandTableName rName = {dst, NULL, (uint8_t)oneTable->sTableNameLen, oneTable->childTableName};
if(tsSmlDot2Underline){ if (tsSmlDot2Underline) {
memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen); memcpy(superName, oneTable->sTableName, oneTable->sTableNameLen);
smlStrReplace(superName, oneTable->sTableNameLen); smlStrReplace(superName, oneTable->sTableNameLen);
rName.stbFullName = superName; rName.stbFullName = superName;
}else{ } else {
rName.stbFullName = oneTable->sTableName; rName.stbFullName = oneTable->sTableName;
} }
@ -506,7 +508,7 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin
char key[TSDB_TABLE_NAME_LEN * 2 + 1] = {0}; char key[TSDB_TABLE_NAME_LEN * 2 + 1] = {0};
size_t nLen = strlen(tinfo->childTableName); size_t nLen = strlen(tinfo->childTableName);
memcpy(key, currElement->measure, currElement->measureLen); memcpy(key, currElement->measure, currElement->measureLen);
if(tsSmlDot2Underline){ if (tsSmlDot2Underline) {
smlStrReplace(key, currElement->measureLen); smlStrReplace(key, currElement->measureLen);
} }
memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen); memcpy(key + currElement->measureLen + 1, tinfo->childTableName, nLen);
@ -522,7 +524,7 @@ void getTableUid(SSmlHandle *info, SSmlLineInfo *currElement, SSmlTableInfo *tin
} }
static void smlDestroySTableMeta(void *para) { static void smlDestroySTableMeta(void *para) {
SSmlSTableMeta *meta = *(SSmlSTableMeta**)para; SSmlSTableMeta *meta = *(SSmlSTableMeta **)para;
taosHashCleanup(meta->tagHash); taosHashCleanup(meta->tagHash);
taosHashCleanup(meta->colHash); taosHashCleanup(meta->colHash);
taosArrayDestroy(meta->tags); taosArrayDestroy(meta->tags);
@ -783,7 +785,8 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm
return TSDB_CODE_SML_INVALID_DATA; return TSDB_CODE_SML_INVALID_DATA;
} }
if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARBINARY || colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) && if (((colField[*index].type == TSDB_DATA_TYPE_VARCHAR || colField[*index].type == TSDB_DATA_TYPE_VARBINARY ||
colField[*index].type == TSDB_DATA_TYPE_GEOMETRY) &&
(colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) || (colField[*index].bytes - VARSTR_HEADER_SIZE) < kv->length) ||
(colField[*index].type == TSDB_DATA_TYPE_NCHAR && (colField[*index].type == TSDB_DATA_TYPE_NCHAR &&
((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) { ((colField[*index].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE < kv->length))) {
@ -814,7 +817,8 @@ static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) {
} }
} }
if ((type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { if ((type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_GEOMETRY) &&
result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) {
result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE;
} else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { } else if (type == TSDB_DATA_TYPE_NCHAR && result > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
result = (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; result = (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE;
@ -866,7 +870,8 @@ static int32_t smlCheckMeta(SSchema *schema, int32_t length, SArray *cols, bool
} }
static int32_t getBytes(uint8_t type, int32_t length) { static int32_t getBytes(uint8_t type, int32_t length) {
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_GEOMETRY) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_NCHAR ||
type == TSDB_DATA_TYPE_GEOMETRY) {
return smlFindNearestPowerOf2(length, type); return smlFindNearestPowerOf2(length, type);
} else { } else {
return tDataTypes[type].bytes; return tDataTypes[type].bytes;
@ -907,7 +912,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO
SField *field = taosArrayGet(results, j); SField *field = taosArrayGet(results, j);
len += field->bytes; len += field->bytes;
} }
if(len > maxLen){ if (len > maxLen) {
return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH; return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH;
} }
@ -924,7 +929,15 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns,
// put front for free // put front for free
pReq.numOfColumns = taosArrayGetSize(pColumns); pReq.numOfColumns = taosArrayGetSize(pColumns);
pReq.pColumns = pColumns; pReq.pColumns = taosArrayInit(pReq.numOfColumns, sizeof(SFieldWithOptions));
for (int32_t i = 0; i < pReq.numOfColumns; ++i) {
SField *pField = taosArrayGet(pColumns, i);
SFieldWithOptions fieldWithOption;
setFieldWithOptions(&fieldWithOption, pField);
setDefaultOptionsForField(&fieldWithOption);
taosArrayPush(pReq.pColumns, &fieldWithOption);
}
pReq.numOfTags = taosArrayGetSize(pTags); pReq.numOfTags = taosArrayGetSize(pTags);
pReq.pTags = pTags; pReq.pTags = pTags;
@ -946,7 +959,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns,
pReq.suid = pTableMeta->uid; pReq.suid = pTableMeta->uid;
pReq.source = TD_REQ_FROM_TAOX; pReq.source = TD_REQ_FROM_TAOX;
pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size";
} else{ } else {
uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action); uError("SML:0x%" PRIx64 " invalid action:%d", info->id, action);
goto end; goto end;
} }
@ -1044,7 +1057,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) { if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST || code == TSDB_CODE_MND_STB_NOT_EXIST) {
code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE); code = smlCheckAuth(info, &conn, NULL, AUTH_TYPE_WRITE);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname); uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas create table:%s", info->id, pName.tname);
@ -1065,6 +1078,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
goto end; goto end;
} }
code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE);
taosArrayDestroy(pColumns);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
goto end; goto end;
@ -1098,7 +1112,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
if (action != SCHEMA_ACTION_NULL) { if (action != SCHEMA_ACTION_NULL) {
code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname, uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table tag, table:%s, action:%d", info->id, pName.tname,
@ -1144,6 +1158,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
taosArrayDestroy(pColumns);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
goto end; goto end;
@ -1172,7 +1187,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
if (action != SCHEMA_ACTION_NULL) { if (action != SCHEMA_ACTION_NULL) {
code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname, uDebug("SML:0x%" PRIx64 " smlModifyDBSchemas change table col, table:%s, action:%d", info->id, pName.tname,
@ -1212,6 +1227,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) {
} }
code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action);
taosArrayDestroy(pColumns);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname);
goto end; goto end;
@ -1324,7 +1340,7 @@ static int32_t smlUpdateMeta(SHashObj *metaHash, SArray *metaArray, SArray *cols
} }
void smlDestroyTableInfo(void *para) { void smlDestroyTableInfo(void *para) {
SSmlTableInfo *tag = *(SSmlTableInfo**)para; SSmlTableInfo *tag = *(SSmlTableInfo **)para;
for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) { for (size_t i = 0; i < taosArrayGetSize(tag->cols); i++) {
SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i); SHashObj *kvHash = (SHashObj *)taosArrayGetP(tag->cols, i);
taosHashCleanup(kvHash); taosHashCleanup(kvHash);
@ -1508,7 +1524,7 @@ static int32_t smlParseLineBottom(SSmlHandle *info) {
uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat, uDebug("SML:0x%" PRIx64 " smlParseLineBottom add meta, format:%d, linenum:%d", info->id, info->dataFormat,
info->lineNum); info->lineNum);
SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat); SSmlSTableMeta *meta = smlBuildSTableMeta(info->dataFormat);
if(meta == NULL){ if (meta == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES); taosHashPut(info->superTables, elements->measure, elements->measureLen, &meta, POINTER_BYTES);
@ -1563,7 +1579,7 @@ static int32_t smlInsertData(SSmlHandle *info) {
conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp); conn.mgmtEps = getEpSet_s(&info->taos->pAppInfo->mgmtEp);
code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE); code = smlCheckAuth(info, &conn, pName.tname, AUTH_TYPE_WRITE);
if(code != TSDB_CODE_SUCCESS){ if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(measure); taosMemoryFree(measure);
taosHashCancelIterate(info->childTables, oneTable); taosHashCancelIterate(info->childTables, oneTable);
return code; return code;
@ -1633,7 +1649,6 @@ static void smlPrintStatisticInfo(SSmlHandle *info) {
info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime, info->cost.numOfAlterColSTables, info->cost.schemaTime - info->cost.parseTime,
info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime, info->cost.insertBindTime - info->cost.schemaTime, info->cost.insertRpcTime - info->cost.insertBindTime,
info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime); info->cost.endTime - info->cost.insertRpcTime, info->cost.endTime - info->cost.parseTime);
} }
int32_t smlClearForRerun(SSmlHandle *info) { int32_t smlClearForRerun(SSmlHandle *info) {
@ -1661,8 +1676,8 @@ int32_t smlClearForRerun(SSmlHandle *info) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd, static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLineEnd, int numLines, int i, char **tmp,
int numLines, int i, char** tmp, int *len){ int *len) {
if (lines) { if (lines) {
*tmp = lines[i]; *tmp = lines[i];
*len = strlen(*tmp); *len = strlen(*tmp);
@ -1679,15 +1694,15 @@ static bool getLine(SSmlHandle *info, char *lines[], char **rawLine, char *rawLi
} }
} }
if(*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)){ if (*rawLine != NULL && (uDebugFlag & DEBUG_DEBUG)) {
char* print = taosMemoryCalloc(*len + 1, 1); char *print = taosMemoryCalloc(*len + 1, 1);
memcpy(print, *tmp, *len); memcpy(print, *tmp, *len);
uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, uDebug("SML:0x%" PRIx64 " smlParseLine is raw, numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines,
numLines, info->protocol, *len, print); info->protocol, *len, print);
taosMemoryFree(print); taosMemoryFree(print);
}else{ } else {
uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, uDebug("SML:0x%" PRIx64 " smlParseLine is not numLines:%d, protocol:%d, len:%d, data:%s", info->id, numLines,
numLines, info->protocol, *len, *tmp); info->protocol, *len, *tmp);
} }
return true; return true;
} }
@ -1713,7 +1728,7 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
while (i < numLines) { while (i < numLines) {
char *tmp = NULL; char *tmp = NULL;
int len = 0; int len = 0;
if(!getLine(info, lines, &rawLine, rawLineEnd, numLines, i, &tmp, &len)){ if (!getLine(info, lines, &rawLine, rawLineEnd, numLines, i, &tmp, &len)) {
continue; continue;
} }
if (info->protocol == TSDB_SML_LINE_PROTOCOL) { if (info->protocol == TSDB_SML_LINE_PROTOCOL) {
@ -1735,12 +1750,12 @@ static int32_t smlParseLine(SSmlHandle *info, char *lines[], char *rawLine, char
code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE; code = TSDB_CODE_SML_INVALID_PROTOCOL_TYPE;
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
if(rawLine != NULL){ if (rawLine != NULL) {
char* print = taosMemoryCalloc(len + 1, 1); char *print = taosMemoryCalloc(len + 1, 1);
memcpy(print, tmp, len); memcpy(print, tmp, len);
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, print);
taosMemoryFree(print); taosMemoryFree(print);
}else{ } else {
uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp); uError("SML:0x%" PRIx64 " smlParseLine failed. line %d : %s", info->id, i, tmp);
} }
return code; return code;
@ -1787,7 +1802,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL
do { do {
code = smlModifyDBSchemas(info); code = smlModifyDBSchemas(info);
if (code != TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER && code != TSDB_CODE_SDB_OBJ_CREATING && code != TSDB_CODE_MND_TRANS_CONFLICT) { if (code != TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER && code != TSDB_CODE_SDB_OBJ_CREATING &&
code != TSDB_CODE_MND_TRANS_CONFLICT) {
break; break;
} }
taosMsleep(100); taosMsleep(100);
@ -1813,7 +1829,7 @@ void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawL
if (tsSlowLogScope & SLOW_LOG_TYPE_INSERT) { if (tsSlowLogScope & SLOW_LOG_TYPE_INSERT) {
int32_t len = 0; int32_t len = 0;
int32_t rlen = 0; int32_t rlen = 0;
char* p = NULL; char *p = NULL;
if (lines && lines[0]) { if (lines && lines[0]) {
len = strlen(lines[0]); len = strlen(lines[0]);
@ -1912,8 +1928,8 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine,
request->code = code; request->code = code;
info->cost.endTime = taosGetTimestampUs(); info->cost.endTime = taosGetTimestampUs();
info->cost.code = code; info->cost.code = code;
if (NEED_CLIENT_HANDLE_ERROR(code) || code == TSDB_CODE_SDB_OBJ_CREATING || if (NEED_CLIENT_HANDLE_ERROR(code) || code == TSDB_CODE_SDB_OBJ_CREATING || code == TSDB_CODE_PAR_VALUE_TOO_LONG ||
code == TSDB_CODE_PAR_VALUE_TOO_LONG || code == TSDB_CODE_MND_TRANS_CONFLICT) { code == TSDB_CODE_MND_TRANS_CONFLICT) {
if (cnt++ >= 10) { if (cnt++ >= 10) {
uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code)); uInfo("SML:%" PRIx64 " retry:%d/10 end code:%d, msg:%s", info->id, cnt, code, tstrerror(code));
break; break;
@ -1976,7 +1992,7 @@ TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLi
reqid); reqid);
} }
static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol){ static void getRawLineLen(char *lines, int len, int32_t *totalRows, int protocol) {
int numLines = 0; int numLines = 0;
*totalRows = 0; *totalRows = 0;
char *tmp = lines; char *tmp = lines;

View File

@ -16,14 +16,15 @@ ENDIF ()
IF (TD_STORAGE) IF (TD_STORAGE)
ADD_DEFINITIONS(-D_STORAGE) ADD_DEFINITIONS(-D_STORAGE)
TARGET_LINK_LIBRARIES(common PRIVATE storage) TARGET_LINK_LIBRARIES(common PRIVATE storage)
ENDIF ()
IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3}) IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3) add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS}) ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS) add_definitions(-DUSE_COS)
ENDIF() ENDIF()
ENDIF()
ENDIF ()
target_include_directories( target_include_directories(
common common

View File

@ -53,6 +53,100 @@ int32_t s3Init() { return 0; /*s3Begin();*/ }
void s3CleanUp() { /*s3End();*/ void s3CleanUp() { /*s3End();*/
} }
static int32_t s3ListBucket(char const *bucketname);
int32_t s3CheckCfg() {
int32_t code = 0;
code = s3Begin();
if (code != 0) {
fprintf(stderr, "failed to initialize s3.\n");
goto _exit;
}
// test put
char testdata[17] = "0123456789abcdef";
const char *objectname[] = {"s3test.txt"};
char path[PATH_MAX] = {0};
int ds_len = strlen(TD_DIRSEP);
int tmp_len = strlen(tsTempDir);
snprintf(path, PATH_MAX, "%s", tsTempDir);
if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) {
snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP);
snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]);
} else {
snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]);
}
TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
if (!fp) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to open test file: %s.\n", path);
// uError("ERROR: %s Failed to open %s", __func__, path);
goto _exit;
}
if (taosWriteFile(fp, testdata, strlen(testdata)) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to write test file: %s.\n", path);
goto _exit;
}
if (taosFsyncFile(fp) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
fprintf(stderr, "failed to fsync test file: %s.\n", path);
goto _exit;
}
taosCloseFile(&fp);
fprintf(stderr, "\nstart to put object: %s, file: %s content: %s\n", objectname[0], path, testdata);
code = s3PutObjectFromFileOffset(path, objectname[0], 0, 16);
if (code != 0) {
fprintf(stderr, "put object %s : failed.\n", objectname[0]);
goto _exit;
}
fprintf(stderr, "put object %s: success.\n\n", objectname[0]);
// list buckets
fprintf(stderr, "start to list bucket %s by prefix s3.\n", tsS3BucketName);
code = s3ListBucket(tsS3BucketName);
if (code != 0) {
fprintf(stderr, "listing bucket %s : failed.\n", tsS3BucketName);
goto _exit;
}
fprintf(stderr, "listing bucket %s: success.\n\n", tsS3BucketName);
// test range get
uint8_t *pBlock = NULL;
int c_offset = 10;
int c_len = 6;
fprintf(stderr, "start to range get object %s offset: %d len: %d.\n", objectname[0], c_offset, c_len);
code = s3GetObjectBlock(objectname[0], c_offset, c_len, true, &pBlock);
if (code != 0) {
fprintf(stderr, "get object %s : failed.\n", objectname[0]);
goto _exit;
}
char buf[7] = {0};
memcpy(buf, pBlock, c_len);
taosMemoryFree(pBlock);
fprintf(stderr, "object content: %s\n", buf);
fprintf(stderr, "get object %s: success.\n\n", objectname[0]);
// delete test object
fprintf(stderr, "start to delete object: %s.\n", objectname[0]);
code = s3DeleteObjects(objectname, 1);
if (code != 0) {
fprintf(stderr, "delete object %s : failed.\n", objectname[0]);
goto _exit;
}
fprintf(stderr, "delete object %s: success.\n\n", objectname[0]);
s3End();
_exit:
return code;
}
static int should_retry() { static int should_retry() {
/* /*
if (retriesG--) { if (retriesG--) {
@ -85,7 +179,7 @@ typedef struct {
} TS3GetData; } TS3GetData;
typedef struct { typedef struct {
char err_msg[128]; char err_msg[512];
S3Status status; S3Status status;
uint64_t content_length; uint64_t content_length;
char *buf; char *buf;
@ -137,6 +231,30 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro
} }
} }
static SArray *getListByPrefix(const char *prefix);
static void s3FreeObjectKey(void *pItem);
static int32_t s3ListBucket(char const *bucketname) {
int32_t code = 0;
SArray *objectArray = getListByPrefix("s3");
if (objectArray == NULL) {
return -1;
}
const char **object_name = TARRAY_DATA(objectArray);
int size = TARRAY_SIZE(objectArray);
fprintf(stderr, "objects:\n");
for (int i = 0; i < size; ++i) {
fprintf(stderr, "%s\n", object_name[i]);
}
taosArrayDestroyEx(objectArray, s3FreeObjectKey);
return code;
}
typedef struct growbuffer { typedef struct growbuffer {
// The total number of bytes, and the start byte // The total number of bytes, and the start byte
int size; int size;
@ -471,10 +589,11 @@ static int32_t s3PutObjectFromFileSimple(S3BucketContext *bucket_context, char c
} while (S3_status_is_retryable(data->status) && should_retry()); } while (S3_status_is_retryable(data->status) && should_retry());
if (data->status != S3StatusOK) { if (data->status != S3StatusOK) {
s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg); // s3PrintError(__FILE__, __LINE__, __func__, data->status, data->err_msg);
s3PrintError(NULL, __LINE__, __func__, data->status, data->err_msg);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
} else if (data->contentLength) { } else if (data->contentLength) {
uError("%s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data->contentLength); uError("%s Failed to put remaining %llu bytes", __func__, (unsigned long long)data->contentLength);
code = TAOS_SYSTEM_ERROR(EIO); code = TAOS_SYSTEM_ERROR(EIO);
} }
@ -830,6 +949,66 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object_name, int8_t w
return code; return code;
} }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) {
int32_t code = 0;
int32_t lmtime = 0;
const char *filename = 0;
uint64_t contentLength = 0;
const char *cacheControl = 0, *contentType = 0, *md5 = 0;
const char *contentDispositionFilename = 0, *contentEncoding = 0;
int64_t expires = -1;
S3CannedAcl cannedAcl = S3CannedAclPrivate;
int metaPropertiesCount = 0;
S3NameValue metaProperties[S3_MAX_METADATA_COUNT];
char useServerSideEncryption = 0;
put_object_callback_data data = {0};
if (taosStatFile(file, &contentLength, &lmtime, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to stat file %s: ", __func__, file);
return code;
}
contentLength = size;
if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) {
code = TAOS_SYSTEM_ERROR(errno);
uError("ERROR: %s Failed to open file %s: ", __func__, file);
return code;
}
if (taosLSeekFile(data.infileFD, offset, SEEK_SET) < 0) {
taosCloseFile(&data.infileFD);
code = TAOS_SYSTEM_ERROR(errno);
return code;
}
data.totalContentLength = data.totalOriginalContentLength = data.contentLength = data.originalContentLength =
contentLength;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG};
S3PutProperties putProperties = {contentType, md5,
cacheControl, contentDispositionFilename,
contentEncoding, expires,
cannedAcl, metaPropertiesCount,
metaProperties, useServerSideEncryption};
if (contentLength <= MULTIPART_CHUNK_SIZE) {
code = s3PutObjectFromFileSimple(&bucketContext, object_name, contentLength, &putProperties, &data);
} else {
code = s3PutObjectFromFileWithoutCp(&bucketContext, object_name, contentLength, &putProperties, &data);
}
if (data.infileFD) {
taosCloseFile(&data.infileFD);
} else if (data.gb) {
growbuffer_destroy(data.gb);
}
return code;
}
typedef struct list_bucket_callback_data { typedef struct list_bucket_callback_data {
char err_msg[512]; char err_msg[512];
S3Status status; S3Status status;
@ -888,7 +1067,7 @@ static SArray *getListByPrefix(const char *prefix) {
const char *marker = 0, *delimiter = 0; const char *marker = 0, *delimiter = 0;
int maxkeys = 0, allDetails = 0; int maxkeys = 0, allDetails = 0;
list_bucket_callback_data data; list_bucket_callback_data data = {0};
data.objectArray = taosArrayInit(32, sizeof(void *)); data.objectArray = taosArrayInit(32, sizeof(void *));
if (!data.objectArray) { if (!data.objectArray) {
uError("%s: %s", __func__, "out of memoty"); uError("%s: %s", __func__, "out of memoty");
@ -918,14 +1097,17 @@ static SArray *getListByPrefix(const char *prefix) {
return data.objectArray; return data.objectArray;
} }
} else { } else {
s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); uError("failed to list with prefix %s: %s", prefix, S3_get_status_name(data.status));
// s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg);
} }
taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); taosArrayDestroyEx(data.objectArray, s3FreeObjectKey);
return NULL; return NULL;
} }
void s3DeleteObjects(const char *object_name[], int nobject) { int32_t s3DeleteObjects(const char *object_name[], int nobject) {
int32_t code = 0;
S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret,
0, awsRegionG}; 0, awsRegionG};
S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback};
@ -938,8 +1120,11 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) {
s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg);
code = -1;
} }
} }
return code;
} }
void s3DeleteObjectsByPrefix(const char *prefix) { void s3DeleteObjectsByPrefix(const char *prefix) {
@ -991,12 +1176,12 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size,
} while (S3_status_is_retryable(cbd.status) && should_retry()); } while (S3_status_is_retryable(cbd.status) && should_retry());
if (cbd.status != S3StatusOK) { if (cbd.status != S3StatusOK) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO); return TAOS_SYSTEM_ERROR(EIO);
} }
if (check && cbd.buf_pos != size) { if (check && cbd.buf_pos != size) {
uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); uError("%s: %d/%s(%s)", __func__, cbd.status, S3_get_status_name(cbd.status), cbd.err_msg);
return TAOS_SYSTEM_ERROR(EIO); return TAOS_SYSTEM_ERROR(EIO);
} }
@ -1233,7 +1418,7 @@ void s3DeleteObjectsByPrefix(const char *prefix_str) {
cos_pool_destroy(p); cos_pool_destroy(p);
} }
void s3DeleteObjects(const char *object_name[], int nobject) { int32_t s3DeleteObjects(const char *object_name[], int nobject) {
cos_pool_t *p = NULL; cos_pool_t *p = NULL;
int is_cname = 0; int is_cname = 0;
cos_string_t bucket; cos_string_t bucket;
@ -1267,6 +1452,8 @@ void s3DeleteObjects(const char *object_name[], int nobject) {
} else { } else {
cos_warn_log("delete objects failed\n"); cos_warn_log("delete objects failed\n");
} }
return 0;
} }
bool s3Exists(const char *object_name) { bool s3Exists(const char *object_name) {
@ -1535,8 +1722,9 @@ int32_t s3Init() { return 0; }
void s3CleanUp() {} void s3CleanUp() {}
int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; } int32_t s3PutObjectFromFile(const char *file, const char *object) { return 0; }
int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; } int32_t s3PutObjectFromFile2(const char *file, const char *object, int8_t withcp) { return 0; }
int32_t s3PutObjectFromFileOffset(const char *file, const char *object_name, int64_t offset, int64_t size) { return 0; }
void s3DeleteObjectsByPrefix(const char *prefix) {} void s3DeleteObjectsByPrefix(const char *prefix) {}
void s3DeleteObjects(const char *object_name[], int nobject) {} int32_t s3DeleteObjects(const char *object_name[], int nobject) { return 0; }
bool s3Exists(const char *object_name) { return false; } bool s3Exists(const char *object_name) { return false; }
bool s3Get(const char *object_name, const char *path) { return false; } bool s3Get(const char *object_name, const char *path) { return false; }
int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) {

View File

@ -24,7 +24,7 @@
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
// clang-format off // clang-format off
static const SSysDbTableSchema dnodesSchema[] = { static const SSysDbTableSchema dnodesSchema[] = {
@ -117,6 +117,9 @@ static const SSysDbTableSchema userDBSchema[] = {
{.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true},
{.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "keep_time_offset", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "s3_chunksize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
{.name = "s3_keeplocal", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "s3_compact", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "with_arbitrator", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
}; };
@ -159,6 +162,8 @@ static const SSysDbTableSchema userStbsSchema[] = {
static const SSysDbTableSchema streamSchema[] = { static const SSysDbTableSchema streamSchema[] = {
{.name = "stream_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "stream_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "stream_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "history_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "status", .bytes = 20 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "source_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
@ -180,7 +185,16 @@ static const SSysDbTableSchema streamTaskSchema[] = {
{.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
{.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
// {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "info", .bytes = 35, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "start_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
{.name = "start_ver", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
{.name = "checkpoint_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "checkpoint_id", .bytes = 25, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
{.name = "checkpoint_version", .bytes = 25, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false},
{.name = "ds_err_info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "history_task_id", .bytes = 16 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "history_task_status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
}; };
static const SSysDbTableSchema userTblsSchema[] = { static const SSysDbTableSchema userTblsSchema[] = {
@ -358,6 +372,18 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
}; };
static const SSysDbTableSchema tsmaSchema[] = {
{.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target_stb", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "interval", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "func_list", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema userGrantsFullSchema[] = { static const SSysDbTableSchema userGrantsFullSchema[] = {
{.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@ -413,6 +439,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true}, {TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true},
{TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true}, {TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true},
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true}, {TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
}; };
static const SSysDbTableSchema connectionsSchema[] = { static const SSysDbTableSchema connectionsSchema[] = {

404
source/common/src/tcol.c Normal file
View File

@ -0,0 +1,404 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tcol.h"
#include "tcompression.h"
#include "tutil.h"
const char* supportedEncode[5] = {TSDB_COLUMN_ENCODE_SIMPLE8B, TSDB_COLUMN_ENCODE_XOR, TSDB_COLUMN_ENCODE_RLE,
TSDB_COLUMN_ENCODE_DELTAD, TSDB_COLUMN_ENCODE_DISABLED};
const char* supportedCompress[6] = {TSDB_COLUMN_COMPRESS_LZ4, TSDB_COLUMN_COMPRESS_TSZ,
TSDB_COLUMN_COMPRESS_XZ, TSDB_COLUMN_COMPRESS_ZLIB,
TSDB_COLUMN_COMPRESS_ZSTD, TSDB_COLUMN_COMPRESS_DISABLED};
const char* supportedLevel[3] = {TSDB_COLUMN_LEVEL_HIGH, TSDB_COLUMN_LEVEL_MEDIUM, TSDB_COLUMN_LEVEL_LOW};
const int supportedEncodeNum = sizeof(supportedEncode) / sizeof(char*);
const int supportedCompressNum = sizeof(supportedCompress) / sizeof(char*);
const int supportedLevelNum = sizeof(supportedLevel) / sizeof(char*);
uint8_t getDefaultEncode(uint8_t type) {
switch (type) {
case TSDB_DATA_TYPE_NULL:
case TSDB_DATA_TYPE_BOOL:
return TSDB_COLVAL_ENCODE_RLE;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
return TSDB_COLVAL_ENCODE_DELTAD;
case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_TIMESTAMP:
return TSDB_COLVAL_ENCODE_XOR;
case TSDB_DATA_TYPE_NCHAR:
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_UTINYINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_USMALLINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_UINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_UBIGINT:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_JSON:
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_VARBINARY:
return TSDB_COLVAL_ENCODE_DISABLED;
case TSDB_DATA_TYPE_DECIMAL:
return TSDB_COLVAL_ENCODE_DELTAD;
case TSDB_DATA_TYPE_BLOB:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
case TSDB_DATA_TYPE_MEDIUMBLOB:
case TSDB_DATA_TYPE_GEOMETRY:
case TSDB_DATA_TYPE_MAX:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
default:
return TSDB_COLVAL_ENCODE_SIMPLE8B;
}
}
const char* getDefaultEncodeStr(uint8_t type) { return columnEncodeStr(getDefaultEncode(type)); }
uint16_t getDefaultCompress(uint8_t type) {
switch (type) {
case TSDB_DATA_TYPE_NULL:
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_VARCHAR: // TSDB_DATA_TYPE_BINARY
case TSDB_DATA_TYPE_TIMESTAMP:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_VARBINARY:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
case TSDB_DATA_TYPE_MEDIUMBLOB:
case TSDB_DATA_TYPE_GEOMETRY:
case TSDB_DATA_TYPE_MAX:
return TSDB_COLVAL_COMPRESS_LZ4;
default:
return TSDB_COLVAL_COMPRESS_LZ4;
}
}
const char* getDefaultCompressStr(uint8_t type) { return columnCompressStr(getDefaultCompress(type)); }
uint8_t getDefaultLevel(uint8_t type) { return TSDB_COLVAL_LEVEL_MEDIUM; }
const char* getDefaultLevelStr(uint8_t type) { return columnLevelStr(getDefaultLevel(type)); }
const char* columnEncodeStr(uint8_t type) {
const char* encode = NULL;
switch (type) {
case TSDB_COLVAL_ENCODE_SIMPLE8B:
encode = TSDB_COLUMN_ENCODE_SIMPLE8B;
break;
case TSDB_COLVAL_ENCODE_XOR:
encode = TSDB_COLUMN_ENCODE_XOR;
break;
case TSDB_COLVAL_ENCODE_RLE:
encode = TSDB_COLUMN_ENCODE_RLE;
break;
case TSDB_COLVAL_ENCODE_DELTAD:
encode = TSDB_COLUMN_ENCODE_DELTAD;
break;
case TSDB_COLVAL_ENCODE_DISABLED:
encode = TSDB_COLUMN_ENCODE_DISABLED;
break;
default:
break;
}
return encode;
}
const char* columnCompressStr(uint16_t type) {
const char* compress = NULL;
switch (type) {
case TSDB_COLVAL_COMPRESS_LZ4:
compress = TSDB_COLUMN_COMPRESS_LZ4;
break;
case TSDB_COLVAL_COMPRESS_TSZ:
compress = TSDB_COLUMN_COMPRESS_TSZ;
break;
case TSDB_COLVAL_COMPRESS_XZ:
compress = TSDB_COLUMN_COMPRESS_XZ;
break;
case TSDB_COLVAL_COMPRESS_ZLIB:
compress = TSDB_COLUMN_COMPRESS_ZLIB;
break;
case TSDB_COLVAL_COMPRESS_ZSTD:
compress = TSDB_COLUMN_COMPRESS_ZSTD;
break;
case TSDB_COLVAL_COMPRESS_DISABLED:
compress = TSDB_COLUMN_COMPRESS_DISABLED;
break;
default:
break;
}
return compress;
}
uint8_t columnLevelVal(const char* level) {
uint8_t l = TSDB_COLVAL_LEVEL_MEDIUM;
if (0 == strcmp(level, "h") || 0 == strcmp(level, TSDB_COLUMN_LEVEL_HIGH)) {
l = TSDB_COLVAL_LEVEL_HIGH;
} else if (0 == strcmp(level, "m") || 0 == strcmp(level, TSDB_COLUMN_LEVEL_MEDIUM)) {
l = TSDB_COLVAL_LEVEL_MEDIUM;
} else if (0 == strcmp(level, "l") || 0 == strcmp(level, TSDB_COLUMN_LEVEL_LOW)) {
l = TSDB_COLVAL_LEVEL_LOW;
} else {
l = TSDB_COLVAL_LEVEL_NOCHANGE;
}
return l;
}
uint16_t columnCompressVal(const char* compress) {
uint16_t c = TSDB_COLVAL_COMPRESS_LZ4;
if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_LZ4)) {
c = TSDB_COLVAL_COMPRESS_LZ4;
} else if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_TSZ)) {
c = TSDB_COLVAL_COMPRESS_TSZ;
} else if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_XZ)) {
c = TSDB_COLVAL_COMPRESS_XZ;
} else if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_ZLIB)) {
c = TSDB_COLVAL_COMPRESS_ZLIB;
} else if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_ZSTD)) {
c = TSDB_COLVAL_COMPRESS_ZSTD;
} else if (0 == strcmp(compress, TSDB_COLUMN_COMPRESS_DISABLED)) {
c = TSDB_COLVAL_COMPRESS_DISABLED;
} else {
c = TSDB_COLVAL_COMPRESS_NOCHANGE;
}
return c;
}
uint8_t columnEncodeVal(const char* encode) {
uint8_t e = TSDB_COLVAL_ENCODE_SIMPLE8B;
if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_SIMPLE8B)) {
e = TSDB_COLVAL_ENCODE_SIMPLE8B;
} else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_XOR)) {
e = TSDB_COLVAL_ENCODE_XOR;
} else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_RLE)) {
e = TSDB_COLVAL_ENCODE_RLE;
} else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_DELTAD)) {
e = TSDB_COLVAL_ENCODE_DELTAD;
} else if (0 == strcmp(encode, TSDB_COLUMN_ENCODE_DISABLED)) {
e = TSDB_COLVAL_ENCODE_DISABLED;
} else {
e = TSDB_COLVAL_ENCODE_NOCHANGE;
}
return e;
}
const char* columnLevelStr(uint8_t type) {
const char* level = NULL;
switch (type) {
case TSDB_COLVAL_LEVEL_HIGH:
level = TSDB_COLUMN_LEVEL_HIGH;
break;
case TSDB_COLVAL_LEVEL_MEDIUM:
level = TSDB_COLUMN_LEVEL_MEDIUM;
break;
case TSDB_COLVAL_LEVEL_LOW:
level = TSDB_COLUMN_LEVEL_LOW;
break;
default:
level = TSDB_COLUMN_LEVEL_UNKNOWN;
break;
}
return level;
}
bool checkColumnEncode(char encode[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(encode)) return true;
strtolower(encode, encode);
for (int i = 0; i < supportedEncodeNum; ++i) {
if (0 == strcmp((const char*)encode, supportedEncode[i])) {
return true;
}
}
return false;
}
bool checkColumnEncodeOrSetDefault(uint8_t type, char encode[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(encode)) {
strncpy(encode, getDefaultEncodeStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnEncode(encode);
}
bool checkColumnCompress(char compress[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(compress)) return true;
strtolower(compress, compress);
for (int i = 0; i < supportedCompressNum; ++i) {
if (0 == strcmp((const char*)compress, supportedCompress[i])) {
return true;
}
}
return false;
}
bool checkColumnCompressOrSetDefault(uint8_t type, char compress[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(compress)) {
strncpy(compress, getDefaultCompressStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnCompress(compress);
}
bool checkColumnLevel(char level[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(level)) return true;
if (1 == strlen(level)) {
if ('h' == level[0] || 'm' == level[0] || 'l' == level[0]) return true;
} else {
for (int i = 0; i < supportedLevelNum; ++i) {
if (0 == strcmp((const char*)level, supportedLevel[i])) {
return true;
}
}
}
return false;
}
bool checkColumnLevelOrSetDefault(uint8_t type, char level[TSDB_CL_COMPRESS_OPTION_LEN]) {
if (0 == strlen(level)) {
strncpy(level, getDefaultLevelStr(type), TSDB_CL_COMPRESS_OPTION_LEN);
return true;
}
return checkColumnLevel(level);
}
void setColEncode(uint32_t* compress, uint8_t l1) {
*compress &= 0x00FFFFFF;
*compress |= (l1 << 24);
return;
}
void setColCompress(uint32_t* compress, uint16_t l2) {
*compress &= 0xFF0000FF;
*compress |= (l2 << 8);
return;
}
void setColLevel(uint32_t* compress, uint8_t level) {
*compress &= 0xFFFFFF00;
*compress |= level;
return;
}
int8_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressType, uint8_t level, bool check,
uint32_t* compress) {
if (check && !validColEncode(type, encode)) return 0;
setColEncode(compress, encode);
if (compressType == TSDB_COLVAL_COMPRESS_DISABLED) {
setColCompress(compress, compressType);
setColLevel(compress, TSDB_COLVAL_LEVEL_DISABLED);
} else {
if (check && !validColCompress(type, compressType)) return 0;
setColCompress(compress, compressType);
if (check && !validColCompressLevel(type, level)) return 0;
setColLevel(compress, level);
}
return 1;
}
bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; }
int8_t validColCompressLevel(uint8_t type, uint8_t level) {
if (level == TSDB_COLVAL_LEVEL_DISABLED) return 1;
if (level < TSDB_COLVAL_LEVEL_NOCHANGE || level > TSDB_COLVAL_LEVEL_HIGH) {
return 0;
}
return 1;
}
int8_t validColCompress(uint8_t type, uint8_t l2) {
if (l2 > TSDB_COLVAL_COMPRESS_XZ && l2 < TSDB_COLVAL_COMPRESS_DISABLED) {
return 0;
}
if (l2 == TSDB_COLVAL_COMPRESS_TSZ) {
if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) {
return 1;
} else {
return 0;
}
}
return 1;
}
//
// | --------type----------|----- supported encode ----|
// |tinyint/smallint/int/bigint/utinyint/usmallinit/uint/ubiginint| simple8b |
// | timestamp/bigint/ubigint | delta-i |
// | bool | bit-packing |
// | flout/double | delta-d |
//
int8_t validColEncode(uint8_t type, uint8_t l1) {
if (l1 == TSDB_COLVAL_ENCODE_NOCHANGE) {
return 1;
}
if (type == TSDB_DATA_TYPE_BOOL) {
return TSDB_COLVAL_ENCODE_RLE == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_TINYINT && type <= TSDB_DATA_TYPE_INT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 ? 1 : 0;
} else if (type == TSDB_DATA_TYPE_BIGINT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 || TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) {
return TSDB_COLVAL_ENCODE_DELTAD == l1 ? 1 : 0;
} else if ((type == TSDB_DATA_TYPE_VARCHAR || type == TSDB_DATA_TYPE_NCHAR) || type == TSDB_DATA_TYPE_JSON ||
type == TSDB_DATA_TYPE_VARBINARY || type == TSDB_DATA_TYPE_BINARY) {
return l1 == TSDB_COLVAL_ENCODE_DISABLED ? 1 : 0;
// if (l1 >= TSDB_COLVAL_ENCODE_NOCHANGE || l1 <= TSDB_COLVAL_ENCODE_DELTAD) {
// return 1;
// } else if (l1 == TSDB_COLVAL_ENCODE_DISABLED) {
// return 1;
// } else {
// return 0;
// }
} else if (type == TSDB_DATA_TYPE_TIMESTAMP) {
return TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0;
} else if (type >= TSDB_DATA_TYPE_UTINYINT && type <= TSDB_DATA_TYPE_UINT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 ? 1 : 0;
} else if (type == TSDB_DATA_TYPE_UBIGINT) {
return TSDB_COLVAL_ENCODE_SIMPLE8B == l1 || TSDB_COLVAL_ENCODE_XOR == l1 ? 1 : 0;
} else if (type == TSDB_DATA_TYPE_GEOMETRY) {
return 1;
}
return 0;
}
uint32_t createDefaultColCmprByType(uint8_t type) {
uint32_t ret = 0;
uint8_t encode = getDefaultEncode(type);
uint8_t compress = getDefaultCompress(type);
uint8_t lvl = getDefaultLevel(type);
SET_COMPRESS(encode, compress, lvl, ret);
return ret;
}
bool validColCmprByType(uint8_t type, uint32_t cmpr) {
DEFINE_VAR(cmpr);
if (validColEncode(type, l1) && validColCompress(type, l2) && validColCompressLevel(type, lvl)) {
return true;
}
return false;
}

View File

@ -191,15 +191,17 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
} }
size_t start = 1; size_t start = 1;
// the first item
memcpy(pColumnInfoData->pData, pData, itemLen);
int32_t t = 0; int32_t t = 0;
int32_t count = log(numOfRows) / log(2); int32_t count = log(numOfRows) / log(2);
uint32_t startOffset = (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) ? pColumnInfoData->varmeta.length : (currentRow * itemLen);
// the first item
memcpy(pColumnInfoData->pData + startOffset, pData, itemLen);
while (t < count) { while (t < count) {
int32_t xlen = 1 << t; int32_t xlen = 1 << t;
memcpy(pColumnInfoData->pData + start * itemLen + pColumnInfoData->varmeta.length, pColumnInfoData->pData, memcpy(pColumnInfoData->pData + start * itemLen + startOffset,
pColumnInfoData->pData + startOffset,
xlen * itemLen); xlen * itemLen);
t += 1; t += 1;
start += xlen; start += xlen;
@ -207,7 +209,8 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
// the tail part // the tail part
if (numOfRows > start) { if (numOfRows > start) {
memcpy(pColumnInfoData->pData + start * itemLen + currentRow * itemLen, pColumnInfoData->pData, memcpy(pColumnInfoData->pData + start * itemLen + startOffset,
pColumnInfoData->pData + startOffset,
(numOfRows - start) * itemLen); (numOfRows - start) * itemLen);
} }
@ -226,9 +229,13 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
bool trimValue) { bool trimValue) {
int32_t len = pColumnInfoData->info.bytes; int32_t len = pColumnInfoData->info.bytes;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
len = varDataTLen(pData); if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
if (pColumnInfoData->varmeta.allocLen < (numOfRows + currentRow) * len) { len = getJsonValueLen(pData);
int32_t code = colDataReserve(pColumnInfoData, (numOfRows + currentRow) * len); } else {
len = varDataTLen(pData);
}
if (pColumnInfoData->varmeta.allocLen < (numOfRows * len + pColumnInfoData->varmeta.length)) {
int32_t code = colDataReserve(pColumnInfoData, (numOfRows * len + pColumnInfoData->varmeta.length));
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -239,10 +246,12 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
} }
void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows) { void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows) {
pColumnInfoData->hasNull = true;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
memset(&pColumnInfoData->varmeta.offset[currentRow], -1, sizeof(int32_t) * numOfRows); memset(&pColumnInfoData->varmeta.offset[currentRow], -1, sizeof(int32_t) * numOfRows);
} else { } else {
if (numOfRows < sizeof(char) * 2) { if (numOfRows < 16) {
for (int32_t i = 0; i < numOfRows; ++i) { for (int32_t i = 0; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i); colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
} }
@ -256,8 +265,9 @@ void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
} }
} }
memset(&BMCharPos(pColumnInfoData->nullbitmap, currentRow + i), 0xFF, (numOfRows - i) / sizeof(char)); int32_t bytes = (numOfRows - i) / 8;
i += (numOfRows - i) / sizeof(char) * sizeof(char); memset(&BMCharPos(pColumnInfoData->nullbitmap, currentRow + i), 0xFF, bytes);
i += bytes * 8;
for (; i < numOfRows; ++i) { for (; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i); colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
@ -481,6 +491,111 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
return 0; return 0;
} }
int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, int32_t numOfRows) {
if (pDst->info.type != pSrc->info.type || pDst->info.bytes != pSrc->info.bytes || pSrc->reassigned) {
return TSDB_CODE_FAILED;
}
if (numOfRows <= 0) {
return numOfRows;
}
if (IS_VAR_DATA_TYPE(pDst->info.type)) {
int32_t allLen = 0;
void* srcAddr = NULL;
if (pSrc->hasNull) {
for (int32_t i = 0; i < numOfRows; ++i) {
if (colDataIsNull_var(pSrc, srcIdx + i)) {
pDst->varmeta.offset[dstIdx + i] = -1;
pDst->hasNull = true;
continue;
}
char* pData = colDataGetVarData(pSrc, srcIdx + i);
if (NULL == srcAddr) {
srcAddr = pData;
}
int32_t dataLen = 0;
if (pSrc->info.type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
} else {
dataLen = varDataTLen(pData);
}
pDst->varmeta.offset[dstIdx + i] = pDst->varmeta.length + allLen;
allLen += dataLen;
}
} else {
for (int32_t i = 0; i < numOfRows; ++i) {
char* pData = colDataGetVarData(pSrc, srcIdx + i);
int32_t dataLen = 0;
if (pSrc->info.type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
} else {
dataLen = varDataTLen(pData);
}
pDst->varmeta.offset[dstIdx + i] = pDst->varmeta.length + allLen;
allLen += dataLen;
}
}
if (allLen > 0) {
// copy data
if (pDst->varmeta.allocLen < pDst->varmeta.length + allLen) {
char* tmp = taosMemoryRealloc(pDst->pData, pDst->varmeta.length + allLen);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pDst->pData = tmp;
pDst->varmeta.allocLen = pDst->varmeta.length + allLen;
}
if (pSrc->hasNull) {
memcpy(pDst->pData + pDst->varmeta.length, srcAddr, allLen);
} else {
memcpy(pDst->pData + pDst->varmeta.length, colDataGetVarData(pSrc, srcIdx), allLen);
}
pDst->varmeta.length = pDst->varmeta.length + allLen;
}
} else {
if (pSrc->hasNull) {
if (BitPos(dstIdx) == BitPos(srcIdx)) {
for (int32_t i = 0; i < numOfRows; ++i) {
if (0 == BitPos(dstIdx) && (i + (1 << NBIT) <= numOfRows)) {
BMCharPos(pDst->nullbitmap, dstIdx + i) = BMCharPos(pSrc->nullbitmap, srcIdx + i);
if (BMCharPos(pDst->nullbitmap, dstIdx + i)) {
pDst->hasNull = true;
}
i += (1 << NBIT) - 1;
} else {
if (colDataIsNull_f(pSrc->nullbitmap, srcIdx + i)) {
colDataSetNull_f(pDst->nullbitmap, dstIdx + i);
pDst->hasNull = true;
} else {
colDataClearNull_f(pDst->nullbitmap, dstIdx + i);
}
}
}
} else {
for (int32_t i = 0; i < numOfRows; ++i) {
if (colDataIsNull_f(pSrc->nullbitmap, srcIdx + i)) {
colDataSetNull_f(pDst->nullbitmap, dstIdx + i);
pDst->hasNull = true;
} else {
colDataClearNull_f(pDst->nullbitmap, dstIdx + i);
}
}
}
}
if (pSrc->pData != NULL) {
memcpy(pDst->pData + pDst->info.bytes * dstIdx, pSrc->pData + pSrc->info.bytes * srcIdx, pDst->info.bytes * numOfRows);
}
}
return 0;
}
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); } size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); }
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
@ -574,6 +689,60 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t srcIdx, int32_t numOfRows) {
if (pDest->info.rows + numOfRows > pDest->info.capacity) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
size_t numOfCols = taosArrayGetSize(pDest->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
colDataAssignNRows(pCol2, pDest->info.rows, pCol1, srcIdx, numOfRows);
}
pDest->info.rows += numOfRows;
return TSDB_CODE_SUCCESS;
}
void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows) {
if (numOfRows >= pBlock->info.rows) {
blockDataCleanup(pBlock);
return;
}
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
pCol->varmeta.length = pCol->varmeta.offset[pBlock->info.rows - numOfRows];
memset(pCol->varmeta.offset + pBlock->info.rows - numOfRows, 0, sizeof(*pCol->varmeta.offset) * numOfRows);
} else {
int32_t i = pBlock->info.rows - numOfRows;
for (; i < pBlock->info.rows; ++i) {
if (BitPos(i)) {
colDataClearNull_f(pCol->nullbitmap, i);
} else {
break;
}
}
int32_t bytes = (pBlock->info.rows - i) / 8;
memset(&BMCharPos(pCol->nullbitmap, i), 0, bytes);
i += bytes * 8;
for (; i < pBlock->info.rows; ++i) {
colDataClearNull_f(pCol->nullbitmap, i);
}
}
}
pBlock->info.rows -= numOfRows;
}
size_t blockDataGetSize(const SSDataBlock* pBlock) { size_t blockDataGetSize(const SSDataBlock* pBlock) {
size_t total = 0; size_t total = 0;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@ -653,25 +822,26 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
return NULL; return NULL;
} }
SSDataBlock* pDst = createDataBlock(); SSDataBlock* pDst = createOneDataBlock(pBlock, false);
if (pDst == NULL) { if (pDst == NULL) {
return NULL; return NULL;
} }
pDst->info = pBlock->info;
pDst->info.rows = 0;
pDst->info.capacity = 0;
pDst->info.rowSize = 0;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0};
SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i);
colInfo.info = pSrcCol->info;
blockDataAppendColInfo(pDst, &colInfo);
}
blockDataEnsureCapacity(pDst, rowCount); blockDataEnsureCapacity(pDst, rowCount);
/* may have disorder varchar data, TODO
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
colDataAssignNRows(pDstCol, 0, pColData, startIndex, rowCount);
}
*/
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
@ -1206,18 +1376,43 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
pInfo->window.skey = 0; pInfo->window.skey = 0;
} }
void blockDataReset(SSDataBlock* pDataBlock) {
SDataBlockInfo* pInfo = &pDataBlock->info;
if (pInfo->capacity == 0) {
return;
}
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
p->hasNull = false;
p->reassigned = false;
if (IS_VAR_DATA_TYPE(p->info.type)) {
p->varmeta.length = 0;
}
}
pInfo->rows = 0;
pInfo->dataLoad = 0;
pInfo->window.ekey = 0;
pInfo->window.skey = 0;
pInfo->id.uid = 0;
pInfo->id.groupId = 0;
}
/* /*
* NOTE: the type of the input column may be TSDB_DATA_TYPE_NULL, which is used to denote * NOTE: the type of the input column may be TSDB_DATA_TYPE_NULL, which is used to denote
* the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to * the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to
* any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case. * any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case.
*/ */
static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows, int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
bool clearPayload) { bool clearPayload) {
if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) { if (numOfRows <= 0 || pBlockInfo && numOfRows <= pBlockInfo->capacity) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t existedRows = pBlockInfo->rows; int32_t existedRows = pBlockInfo ? pBlockInfo->rows : 0;
if (IS_VAR_DATA_TYPE(pColumn->info.type)) { if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
@ -1329,8 +1524,6 @@ void* blockDataDestroy(SSDataBlock* pBlock) {
} }
if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) { if (IS_VAR_DATA_TYPE(pBlock->info.pks[0].type)) {
uInfo("1====free pk:%p, %p pBlock", pBlock->info.pks[0].pData, pBlock);
uInfo("2====free pk:%p, %p pBlock", pBlock->info.pks[1].pData, pBlock);
taosMemoryFreeClear(pBlock->info.pks[0].pData); taosMemoryFreeClear(pBlock->info.pks[0].pData);
taosMemoryFreeClear(pBlock->info.pks[1].pData); taosMemoryFreeClear(pBlock->info.pks[1].pData);
} }
@ -1510,10 +1703,6 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) {
p->pData = taosMemoryCalloc(1, pDataBlock->info.pks[1].nData); p->pData = taosMemoryCalloc(1, pDataBlock->info.pks[1].nData);
p->nData = pDataBlock->info.pks[1].nData; p->nData = pDataBlock->info.pks[1].nData;
memcpy(p->pData, pDataBlock->info.pks[1].pData, p->nData); memcpy(p->pData, pDataBlock->info.pks[1].pData, p->nData);
uInfo("===========clone block, with varchar, %p, 0---addr:%p, src:%p, %p", pBlock, pBlock->info.pks[0].pData, pDataBlock, pDataBlock->info.pks[0].pData);
uInfo("===========clone block, with varchar, %p, 1---addr:%p, src:%p, %p", pBlock, pBlock->info.pks[1].pData, pDataBlock, pDataBlock->info.pks[1].pData);
} else {
uInfo("===========clone block without varchar pk, %p, src:%p", pBlock, pDataBlock);
} }
if (copyData) { if (copyData) {
@ -2505,6 +2694,8 @@ void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList
int32_t numOfRows = 0; int32_t numOfRows = 0;
if (IS_VAR_DATA_TYPE(pDst->info.type)) { if (IS_VAR_DATA_TYPE(pDst->info.type)) {
pDst->varmeta.length = 0; pDst->varmeta.length = 0;
} else {
memset(pDst->nullbitmap, 0, bmLen);
} }
} }
return; return;

View File

@ -2450,7 +2450,7 @@ static int32_t tColDataUpdateValue72(SColData *pColData, uint8_t *pData, uint32_
pColData->numOfValue--; pColData->numOfValue--;
pColData->nVal--; pColData->nVal--;
if (pColData->numOfValue) { if (pColData->numOfValue) {
if (IS_STR_DATA_TYPE(pColData->type)) { if (IS_VAR_DATA_TYPE(pColData->type)) {
pColData->nData = pColData->aOffset[pColData->nVal]; pColData->nData = pColData->aOffset[pColData->nVal];
} else { } else {
pColData->nData -= TYPE_BYTES[pColData->type]; pColData->nData -= TYPE_BYTES[pColData->type];
@ -4278,7 +4278,7 @@ int32_t tCompressData(void *input, // input
if (info->cmprAlg == NO_COMPRESSION) { if (info->cmprAlg == NO_COMPRESSION) {
memcpy(output, input, info->originalSize); memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize; info->compressedSize = info->originalSize;
} else { } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local; SBuffer local;
tBufferInit(&local); tBufferInit(&local);
@ -4310,6 +4310,38 @@ int32_t tCompressData(void *input, // input
} }
tBufferDestroy(&local); tBufferDestroy(&local);
} else {
DEFINE_VAR(info->cmprAlg)
if ((l1 == L1_UNKNOWN && l2 == L2_UNKNOWN) || (l1 == L1_DISABLED && l2 == L2_DISABLED)) {
memcpy(output, input, info->originalSize);
info->compressedSize = info->originalSize;
return 0;
}
SBuffer local;
tBufferInit(&local);
if (buffer == NULL) {
buffer = &local;
}
code = tBufferEnsureCapacity(buffer, extraSizeNeeded);
info->compressedSize = tDataCompress[info->dataType].compFunc( //
input, // input
info->originalSize, // input size
info->originalSize / tDataTypes[info->dataType].bytes, // number of elements
output, // output
outputSize, // output size
info->cmprAlg, // compression algorithm
buffer->data, // buffer
buffer->capacity // buffer size
);
if (info->compressedSize < 0) {
tBufferDestroy(&local);
return TSDB_CODE_COMPRESS_ERROR;
}
tBufferDestroy(&local);
// new col compress
} }
return 0; return 0;
@ -4328,7 +4360,7 @@ int32_t tDecompressData(void *input, // input
if (info->cmprAlg == NO_COMPRESSION) { if (info->cmprAlg == NO_COMPRESSION) {
ASSERT(info->compressedSize == info->originalSize); ASSERT(info->compressedSize == info->originalSize);
memcpy(output, input, info->compressedSize); memcpy(output, input, info->compressedSize);
} else { } else if (info->cmprAlg == ONE_STAGE_COMP || info->cmprAlg == TWO_STAGE_COMP) {
SBuffer local; SBuffer local;
tBufferInit(&local); tBufferInit(&local);
@ -4359,6 +4391,37 @@ int32_t tDecompressData(void *input, // input
return TSDB_CODE_COMPRESS_ERROR; return TSDB_CODE_COMPRESS_ERROR;
} }
ASSERT(decompressedSize == info->originalSize);
tBufferDestroy(&local);
} else {
DEFINE_VAR(info->cmprAlg);
if (l1 == L1_DISABLED && l2 == L2_DISABLED) {
memcpy(output, input, info->compressedSize);
return 0;
}
SBuffer local;
tBufferInit(&local);
if (buffer == NULL) {
buffer = &local;
}
code = tBufferEnsureCapacity(buffer, info->originalSize + COMP_OVERFLOW_BYTES);
int32_t decompressedSize = tDataCompress[info->dataType].decompFunc(
input, // input
info->compressedSize, // inputSize
info->originalSize / tDataTypes[info->dataType].bytes, // number of elements
output, // output
outputSize, // output size
info->cmprAlg, // compression algorithm
buffer->data, // helper buffer
buffer->capacity // extra buffer size
);
if (decompressedSize < 0) {
tBufferDestroy(&local);
return TSDB_CODE_COMPRESS_ERROR;
}
ASSERT(decompressedSize == info->originalSize); ASSERT(decompressedSize == info->originalSize);
tBufferDestroy(&local); tBufferDestroy(&local);
} }

View File

@ -171,6 +171,7 @@ int32_t tsMetaCacheMaxSize = -1; // MB
int32_t tsSlowLogThreshold = 3; // seconds int32_t tsSlowLogThreshold = 3; // seconds
int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL;
int32_t tsTimeSeriesThreshold = 50; int32_t tsTimeSeriesThreshold = 50;
bool tsMultiResultFunctionStarReturnTags = false;
/* /*
* denote if the server needs to compress response message at the application layer to client, including query rsp, * denote if the server needs to compress response message at the application layer to client, including query rsp,
@ -265,7 +266,9 @@ float tsSinkDataRate = 2.0;
int32_t tsStreamNodeCheckInterval = 16; int32_t tsStreamNodeCheckInterval = 16;
int32_t tsTtlUnit = 86400; int32_t tsTtlUnit = 86400;
int32_t tsTtlPushIntervalSec = 10; int32_t tsTtlPushIntervalSec = 10;
int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups int32_t tsTrimVDbIntervalSec = 60 * 60; // interval of trimming db in all vgroups
int32_t tsS3MigrateIntervalSec = 60 * 60; // interval of s3migrate db in all vgroups
bool tsS3MigrateEnabled = 1;
int32_t tsGrantHBInterval = 60; int32_t tsGrantHBInterval = 60;
int32_t tsUptimeInterval = 300; // seconds int32_t tsUptimeInterval = 300; // seconds
char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits
@ -291,10 +294,13 @@ char tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) int32_t tsS3BlockSize = -1; // number of tsdb pages (4096)
int32_t tsS3BlockCacheSize = 16; // number of blocks int32_t tsS3BlockCacheSize = 16; // number of blocks
int32_t tsS3PageCacheSize = 4096; // number of pages int32_t tsS3PageCacheSize = 4096; // number of pages
int32_t tsS3UploadDelaySec = 60 * 60 * 24; int32_t tsS3UploadDelaySec = 60;
bool tsExperimental = true; bool tsExperimental = true;
int32_t tsMaxTsmaNum = 8;
int32_t tsMaxTsmaCalcDelay = 600;
#ifndef _STORAGE #ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -351,7 +357,9 @@ int32_t taosSetS3Cfg(SConfig *pCfg) {
} }
if (tsS3BucketName[0] != '<') { if (tsS3BucketName[0] != '<') {
#if defined(USE_COS) || defined(USE_S3) #if defined(USE_COS) || defined(USE_S3)
if (tsDiskCfgNum > 1) tsS3Enabled = true; #ifdef TD_ENTERPRISE
/*if (tsDiskCfgNum > 1) */ tsS3Enabled = true;
#endif
tsS3StreamEnabled = true; tsS3StreamEnabled = true;
#endif #endif
} }
@ -546,6 +554,13 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) !=
0)
return -1;
return 0; return 0;
} }
@ -600,8 +615,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
@ -674,8 +687,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0) CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, if (cfgAddInt32(pCfg, "syncSnapReplMaxWaitN", tsSnapReplMaxWaitN, 16, (TSDB_SYNC_SNAP_BUFFER_SIZE >> 2),
(TSDB_SYNC_SNAP_BUFFER_SIZE >> 2), CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "arbHeartBeatIntervalSec", tsArbHeartBeatIntervalSec, 1, 60 * 24 * 2, CFG_SCOPE_SERVER,
@ -700,7 +713,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "monitorLogProtocol", tsMonitorLogProtocol, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt32(pCfg, "monitorIntervalForBasic", tsMonitorIntervalForBasic, 1, 200000, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddBool(pCfg, "monitorForceV2", tsMonitorForceV2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -724,6 +738,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 12, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
@ -745,6 +760,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "s3MigrateIntervalSec", tsS3MigrateIntervalSec, 600, 100000, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddBool(pCfg, "s3MigrateEnabled", tsS3MigrateEnabled, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
@ -763,8 +782,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1; return -1;
if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) if (cfgAddInt64(pCfg, "streamAggCnt", tsStreamAggCnt, 2, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
return -1;
if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0) CFG_DYN_ENT_SERVER) != 0)
@ -795,6 +813,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
/*
if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1; return -1;
if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) { if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) {
@ -804,10 +823,11 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0) 0)
return -1; return -1;
*/
if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0) CFG_DYN_ENT_SERVER) != 0)
return -1; return -1;
if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER,
CFG_DYN_ENT_SERVER) != 0) CFG_DYN_ENT_SERVER) != 0)
return -1; return -1;
@ -1096,6 +1116,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) {
return -1; return -1;
} }
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
@ -1107,6 +1128,9 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32;
tsExperimental = cfgGetItem(pCfg, "experimental")->bval; tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
tsMultiResultFunctionStarReturnTags = cfgGetItem(pCfg, "multiResultFunctionStarReturnTags")->bval;
tsMaxTsmaCalcDelay = cfgGetItem(pCfg, "maxTsmaCalcDelay")->i32;
return 0; return 0;
} }
@ -1138,7 +1162,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32;
tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
@ -1188,6 +1211,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32; tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
tmqRowSize = cfgGetItem(pCfg, "tmqRowSize")->i32; tmqRowSize = cfgGetItem(pCfg, "tmqRowSize")->i32;
tsMaxTsmaNum = cfgGetItem(pCfg, "maxTsmaNum")->i32;
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32; tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
@ -1247,8 +1271,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32; tsResolveFQDNRetryTime = cfgGetItem(pCfg, "resolveFQDNRetryTime")->i32;
tsMinDiskFreeSize = cfgGetItem(pCfg, "minDiskFreeSize")->i64; tsMinDiskFreeSize = cfgGetItem(pCfg, "minDiskFreeSize")->i64;
tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32; // tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32;
tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32; // tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32;
tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32; tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32;
tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32; tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32;
@ -1526,12 +1550,15 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"ttlBatchDropNum", &tsTtlBatchDropNum}, {"ttlBatchDropNum", &tsTtlBatchDropNum},
{"ttlFlushThreshold", &tsTtlFlushThreshold}, {"ttlFlushThreshold", &tsTtlFlushThreshold},
{"ttlPushInterval", &tsTtlPushIntervalSec}, {"ttlPushInterval", &tsTtlPushIntervalSec},
{"s3MigrateIntervalSec", &tsS3MigrateIntervalSec},
{"s3MigrateEnabled", &tsS3MigrateEnabled},
//{"s3BlockSize", &tsS3BlockSize}, //{"s3BlockSize", &tsS3BlockSize},
{"s3BlockCacheSize", &tsS3BlockCacheSize}, {"s3BlockCacheSize", &tsS3BlockCacheSize},
{"s3PageCacheSize", &tsS3PageCacheSize}, {"s3PageCacheSize", &tsS3PageCacheSize},
{"s3UploadDelaySec", &tsS3UploadDelaySec}, {"s3UploadDelaySec", &tsS3UploadDelaySec},
{"supportVnodes", &tsNumOfSupportVnodes}, {"supportVnodes", &tsNumOfSupportVnodes},
{"experimental", &tsExperimental}}; {"experimental", &tsExperimental},
{"maxTsmaNum", &tsMaxTsmaNum}};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
taosCfgSetOption(options, tListLen(options), pItem, false); taosCfgSetOption(options, tListLen(options), pItem, false);
@ -1748,7 +1775,9 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
{"shellActivityTimer", &tsShellActivityTimer}, {"shellActivityTimer", &tsShellActivityTimer},
{"slowLogThreshold", &tsSlowLogThreshold}, {"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter}, {"useAdapter", &tsUseAdapter},
{"experimental", &tsExperimental}}; {"experimental", &tsExperimental},
{"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags},
{"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
taosCfgSetOption(options, tListLen(options), pItem, false); taosCfgSetOption(options, tListLen(options), pItem, false);

File diff suppressed because it is too large Load Diff

View File

@ -581,7 +581,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) { int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
switch (unit) { switch (unit) {
case 's': case 's':
if (val > INT64_MAX / MILLISECOND_PER_SECOND) { if (val > INT64_MAX / MILLISECOND_PER_SECOND) {
@ -663,12 +663,12 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura
return getDuration(timestamp, *unit, duration, timePrecision); return getDuration(timestamp, *unit, duration, timePrecision);
} }
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision) { int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) {
errno = 0; errno = 0;
/* get the basic numeric value */ /* get the basic numeric value */
*duration = taosStr2Int64(token, NULL, 10); *duration = taosStr2Int64(token, NULL, 10);
if (*duration < 0 || errno != 0) { if ((*duration < 0 && !negativeAllow) || errno != 0) {
return -1; return -1;
} }

View File

@ -61,13 +61,44 @@ tDataTypeDescriptor tDataTypes[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt}, {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint}, {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint},
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString}, {TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString, tsDecompressString},
{TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString, tsDecompressString}, // placeholder, not implemented {TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString,
tsDecompressString}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented {TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString, tsDecompressString}, {TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString, tsDecompressString},
}; };
tDataTypeCompress tDataCompress[TSDB_DATA_TYPE_MAX] = {
{TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL},
{TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool2, tsDecompressBool2},
{TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint2, tsDecompressTinyint2},
{TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint2,
tsDecompressSmallint2},
{TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt2, tsDecompressInt2},
{TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint2, tsDecompressBigint2},
{TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat2, tsDecompressFloat2},
{TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble2, tsDecompressDouble2},
{TSDB_DATA_TYPE_VARCHAR, 6, 1, "VARCHAR", 0, 0, tsCompressString2, tsDecompressString2},
{TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp2,
tsDecompressTimestamp2},
{TSDB_DATA_TYPE_NCHAR, 5, 1, "NCHAR", 0, 0, tsCompressString2, tsDecompressString2},
{TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint2,
tsDecompressTinyint2},
{TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint2,
tsDecompressSmallint2},
{TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt2, tsDecompressInt2},
{TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint2, tsDecompressBigint2},
{TSDB_DATA_TYPE_JSON, 4, TSDB_MAX_JSON_TAG_LEN, "JSON", 0, 0, tsCompressString2, tsDecompressString2},
{TSDB_DATA_TYPE_VARBINARY, 9, 1, "VARBINARY", 0, 0, tsCompressString2,
tsDecompressString2}, // placeholder, not implemented
{TSDB_DATA_TYPE_DECIMAL, 7, 1, "DECIMAL", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_BLOB, 4, 1, "BLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_MEDIUMBLOB, 10, 1, "MEDIUMBLOB", 0, 0, NULL, NULL}, // placeholder, not implemented
{TSDB_DATA_TYPE_GEOMETRY, 8, 1, "GEOMETRY", 0, 0, tsCompressString2, tsDecompressString2},
};
static float floatMin = -FLT_MAX, floatMax = FLT_MAX; static float floatMin = -FLT_MAX, floatMax = FLT_MAX;
static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; static double doubleMin = -DBL_MAX, doubleMax = DBL_MAX;

View File

@ -14,6 +14,14 @@ target_include_directories(
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc"
) )
IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS)
ENDIF()
ENDIF()
IF (TD_LINUX_64 AND JEMALLOC_ENABLED) IF (TD_LINUX_64 AND JEMALLOC_ENABLED)
ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc)
SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc")

View File

@ -27,15 +27,15 @@
#include "cus_name.h" #include "cus_name.h"
#else #else
#ifndef CUS_NAME #ifndef CUS_NAME
#define CUS_NAME "TDengine" #define CUS_NAME "TDengine"
#endif #endif
#ifndef CUS_PROMPT #ifndef CUS_PROMPT
#define CUS_PROMPT "taos" #define CUS_PROMPT "taos"
#endif #endif
#ifndef CUS_EMAIL #ifndef CUS_EMAIL
#define CUS_EMAIL "<support@taosdata.com>" #define CUS_EMAIL "<support@taosdata.com>"
#endif #endif
#endif #endif
// clang-format off // clang-format off
@ -58,6 +58,7 @@ static struct {
bool dumpSdb; bool dumpSdb;
bool generateGrant; bool generateGrant;
bool memDbg; bool memDbg;
bool checkS3;
bool printAuth; bool printAuth;
bool printVersion; bool printVersion;
bool printHelp; bool printHelp;
@ -169,7 +170,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
return -1; return -1;
} }
} else if (strcmp(argv[i], "-a") == 0) { } else if (strcmp(argv[i], "-a") == 0) {
if(i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("apollo url overflow"); printf("apollo url overflow");
return -1; return -1;
@ -182,7 +183,7 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
} else if (strcmp(argv[i], "-s") == 0) { } else if (strcmp(argv[i], "-s") == 0) {
global.dumpSdb = true; global.dumpSdb = true;
} else if (strcmp(argv[i], "-E") == 0) { } else if (strcmp(argv[i], "-E") == 0) {
if(i < argc - 1) { if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) { if (strlen(argv[++i]) >= PATH_MAX) {
printf("env file path overflow"); printf("env file path overflow");
return -1; return -1;
@ -207,6 +208,8 @@ static int32_t dmParseArgs(int32_t argc, char const *argv[]) {
cmdEnvIndex++; cmdEnvIndex++;
} else if (strcmp(argv[i], "-dm") == 0) { } else if (strcmp(argv[i], "-dm") == 0) {
global.memDbg = true; global.memDbg = true;
} else if (strcmp(argv[i], "--checks3") == 0) {
global.checkS3 = true;
} else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 ||
strcmp(argv[i], "-?") == 0) { strcmp(argv[i], "-?") == 0) {
global.printHelp = true; global.printHelp = true;
@ -267,8 +270,21 @@ static void dmDumpCfg() {
cfgDumpCfg(pCfg, 0, true); cfgDumpCfg(pCfg, 0, true);
} }
static int32_t dmCheckS3() {
int32_t code = 0;
SConfig *pCfg = taosGetCfg();
cfgDumpCfgS3(pCfg, 0, true);
#if defined(USE_S3)
extern int32_t s3CheckCfg();
code = s3CheckCfg();
#endif
return code;
}
static int32_t dmInitLog() { static int32_t dmInitLog() {
return taosCreateLog(CUS_PROMPT"dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); return taosCreateLog(CUS_PROMPT "dlog", 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs,
0);
} }
static void taosCleanupArgs() { static void taosCleanupArgs() {
@ -355,6 +371,15 @@ int mainWindows(int argc, char **argv) {
return -1; return -1;
} }
if (global.checkS3) {
int32_t code = dmCheckS3();
taosCleanupCfg();
taosCloseLog();
taosCleanupArgs();
taosConvDestroy();
return code;
}
if (global.dumpConfig) { if (global.dumpConfig) {
dmDumpCfg(); dmDumpCfg();
taosCleanupCfg(); taosCleanupCfg();

View File

@ -141,6 +141,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_COMPACT_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TRIM_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_S3MIGRATE_DB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_DB_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_VGROUP_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_REDISTRIBUTE_VGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -163,6 +164,14 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STB_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STB_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TB_WITH_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@ -208,6 +217,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -220,6 +230,10 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

View File

@ -91,6 +91,7 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0; code = 0;
_OVER: _OVER:

View File

@ -53,7 +53,7 @@ void vmGetVnodeLoadsLite(SVnodeMgmt *pMgmt, SMonVloadInfo *pInfo) {
SVnodeObj **ppVnode = pIter; SVnodeObj **ppVnode = pIter;
if (ppVnode == NULL || *ppVnode == NULL) continue; if (ppVnode == NULL || *ppVnode == NULL) continue;
SVnodeObj *pVnode = *ppVnode; SVnodeObj *pVnode = *ppVnode;
if (!pVnode->failed) { if (!pVnode->failed) {
SVnodeLoadLite vload = {0}; SVnodeLoadLite vload = {0};
if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) { if (vnodeGetLoadLite(pVnode->pImpl, &vload) == 0) {
@ -159,6 +159,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->hashSuffix = pCreate->hashSuffix; pCfg->hashSuffix = pCreate->hashSuffix;
pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024;
pCfg->s3ChunkSize = pCreate->s3ChunkSize;
pCfg->s3KeepLocal = pCreate->s3KeepLocal;
pCfg->s3Compact = pCreate->s3Compact;
pCfg->standby = 0; pCfg->standby = 0;
pCfg->syncCfg.replicaNum = 0; pCfg->syncCfg.replicaNum = 0;
pCfg->syncCfg.totalReplicaNum = 0; pCfg->syncCfg.totalReplicaNum = 0;
@ -238,17 +242,18 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo( dInfo(
"vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d " "vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d "
"szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 "szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64
", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d" ", days:%d keep0:%d keep1:%d keep2:%d keepTimeOffset%d s3ChunkSize:%d s3KeepLocal:%d s3Compact:%d tsma:%d "
"precision:%d compression:%d minRows:%d maxRows:%d"
", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64 ", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64
", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d " ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d "
"learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d", "learnerReplica:%d learnerSelfIndex:%d strict:%d changeVersion:%d",
req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024,
(uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize,
req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2,
req.keepTimeOffset, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, req.keepTimeOffset, req.s3ChunkSize, req.s3KeepLocal, req.s3Compact, req.isTsma, req.precision, req.compression,
req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize,
req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica, req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, req.hashSuffix,
req.learnerSelfIndex, req.strict, req.changeVersion); req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict, req.changeVersion);
for (int32_t i = 0; i < req.replica; ++i) { for (int32_t i = 0; i < req.replica; ++i) {
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port,
@ -347,7 +352,7 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
_OVER: _OVER:
if (code != 0) { if (code != 0) {
vnodeClose(pImpl); vnodeClose(pImpl);
vnodeDestroy(0, path, pMgmt->pTfs); vnodeDestroy(0, path, pMgmt->pTfs, 0);
} else { } else {
dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId, dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", req.vgId,
TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
@ -358,7 +363,7 @@ _OVER:
return code; return code;
} }
//alter replica doesn't use this, but restore dnode still use this // alter replica doesn't use this, but restore dnode still use this
int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
SAlterVnodeTypeReq req = {0}; SAlterVnodeTypeReq req = {0};
if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) { if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) {
@ -400,8 +405,8 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name); dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name);
int32_t vgId = req.vgId; int32_t vgId = req.vgId;
dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d changeVersion:%d", vgId, req.replica,
vgId, req.replica, req.selfIndex, req.strict, req.changeVersion); req.selfIndex, req.strict, req.changeVersion);
for (int32_t i = 0; i < req.replica; ++i) { for (int32_t i = 0; i < req.replica; ++i) {
SReplica *pReplica = &req.replicas[i]; SReplica *pReplica = &req.replicas[i];
dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id);
@ -484,12 +489,12 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
return -1; return -1;
} }
if(req.learnerReplicas == 0){ if (req.learnerReplicas == 0) {
req.learnerSelfIndex = -1; req.learnerSelfIndex = -1;
} }
dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", dInfo("vgId:%d, vnode management handle msgType:%s, start to process check-learner-catchup-request", req.vgId,
req.vgId, TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId);
if (pVnode == NULL) { if (pVnode == NULL) {
@ -501,7 +506,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
ESyncRole role = vnodeGetRole(pVnode->pImpl); ESyncRole role = vnodeGetRole(pVnode->pImpl);
dInfo("vgId:%d, checking node role:%d", req.vgId, role); dInfo("vgId:%d, checking node role:%d", req.vgId, role);
if(role == TAOS_SYNC_ROLE_VOTER){ if (role == TAOS_SYNC_ROLE_VOTER) {
dError("vgId:%d, failed to alter vnode type since node already is role:%d", req.vgId, role); dError("vgId:%d, failed to alter vnode type since node already is role:%d", req.vgId, role);
terrno = TSDB_CODE_VND_ALREADY_IS_VOTER; terrno = TSDB_CODE_VND_ALREADY_IS_VOTER;
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
@ -509,7 +514,7 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
} }
dInfo("vgId:%d, checking node catch up", req.vgId); dInfo("vgId:%d, checking node catch up", req.vgId);
if(vnodeIsCatchUp(pVnode->pImpl) != 1){ if (vnodeIsCatchUp(pVnode->pImpl) != 1) {
terrno = TSDB_CODE_VND_NOT_CATCH_UP; terrno = TSDB_CODE_VND_NOT_CATCH_UP;
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
return -1; return -1;
@ -519,8 +524,8 @@ int32_t vmProcessCheckLearnCatchupReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
vmReleaseVnode(pMgmt, pVnode); vmReleaseVnode(pMgmt, pVnode);
dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", dInfo("vgId:%d, vnode management handle msgType:%s, end to process check-learner-catchup-request", req.vgId,
req.vgId, TMSG_INFO(pMsg->msgType)); TMSG_INFO(pMsg->msgType));
return 0; return 0;
} }
@ -879,6 +884,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -925,6 +931,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -933,6 +940,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_S3MIGRATE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;

View File

@ -15,9 +15,9 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "vmInt.h" #include "vmInt.h"
#include "libs/function/tudf.h"
#include "tfs.h" #include "tfs.h"
#include "vnd.h" #include "vnd.h"
#include "libs/function/tudf.h"
int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) { int32_t vmGetPrimaryDisk(SVnodeMgmt *pMgmt, int32_t vgId) {
int32_t diskId = -1; int32_t diskId = -1;
@ -234,6 +234,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal)
dInfo("vgId:%d, commit data finished", pVnode->vgId); dInfo("vgId:%d, commit data finished", pVnode->vgId);
} }
int32_t nodeId = vnodeNodeId(pVnode->pImpl);
vnodeClose(pVnode->pImpl); vnodeClose(pVnode->pImpl);
pVnode->pImpl = NULL; pVnode->pImpl = NULL;
@ -250,7 +251,7 @@ _closed:
if (pVnode->dropped) { if (pVnode->dropped) {
dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped); dInfo("vgId:%d, vnode is destroyed, dropped:%d", pVnode->vgId, pVnode->dropped);
snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId); snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pVnode->vgId);
vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs); vnodeDestroy(pVnode->vgId, path, pMgmt->pTfs, nodeId);
} }
vmFreeVnodeObj(&pVnode); vmFreeVnodeObj(&pVnode);

View File

@ -4,15 +4,13 @@ target_link_libraries(
dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode monitorfw
) )
IF (TD_STORAGE) IF (TD_ENTERPRISE)
IF(${BUILD_WITH_S3}) IF(${BUILD_WITH_S3})
add_definitions(-DUSE_S3) add_definitions(-DUSE_S3)
ELSEIF(${BUILD_WITH_COS}) ELSEIF(${BUILD_WITH_COS})
add_definitions(-DUSE_COS) add_definitions(-DUSE_COS)
ENDIF() ENDIF()
ENDIF()
ENDIF ()
IF (DEFINED GRANT_CFG_INCLUDE_DIR) IF (DEFINED GRANT_CFG_INCLUDE_DIR)
add_definitions(-DGRANTS_CFG) add_definitions(-DGRANTS_CFG)

View File

@ -29,6 +29,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb);
int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen);
int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq); int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq);
bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb); bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb);
void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList);
SSdbRaw *mndDbActionEncode(SDbObj *pDb); SSdbRaw *mndDbActionEncode(SDbObj *pDb);
const char *mndGetDbStr(const char *src); const char *mndGetDbStr(const char *src);

View File

@ -379,6 +379,9 @@ typedef struct {
int32_t walRollPeriod; int32_t walRollPeriod;
int64_t walRetentionSize; int64_t walRetentionSize;
int64_t walSegmentSize; int64_t walSegmentSize;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
int8_t withArbitrator; int8_t withArbitrator;
} SDbCfg; } SDbCfg;
@ -395,6 +398,7 @@ typedef struct {
SRWLatch lock; SRWLatch lock;
int64_t stateTs; int64_t stateTs;
int64_t compactStartTime; int64_t compactStartTime;
int32_t tsmaVersion;
} SDbObj; } SDbObj;
typedef struct { typedef struct {
@ -454,12 +458,14 @@ typedef struct {
int32_t tagsFilterLen; int32_t tagsFilterLen;
int32_t sqlLen; int32_t sqlLen;
int32_t astLen; int32_t astLen;
int32_t version;
char* expr; char* expr;
char* tagsFilter; char* tagsFilter;
char* sql; char* sql;
char* ast; char* ast;
SSchemaWrapper schemaRow; // for dstVgroup SSchemaWrapper schemaRow; // for dstVgroup
SSchemaWrapper schemaTag; // for dstVgroup SSchemaWrapper schemaTag; // for dstVgroup
char baseSmaName[TSDB_TABLE_FNAME_LEN];
} SSmaObj; } SSmaObj;
typedef struct { typedef struct {
@ -475,33 +481,38 @@ typedef struct {
} SIdxObj; } SIdxObj;
typedef struct { typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; col_id_t colId;
char db[TSDB_DB_FNAME_LEN]; int32_t cmprAlg;
int64_t createdTime; } SCmprObj;
int64_t updateTime; typedef struct {
int64_t uid; char name[TSDB_TABLE_FNAME_LEN];
int64_t dbUid; char db[TSDB_DB_FNAME_LEN];
int32_t tagVer; int64_t createdTime;
int32_t colVer; int64_t updateTime;
int32_t smaVer; int64_t uid;
int32_t nextColId; int64_t dbUid;
int64_t maxdelay[2]; int32_t tagVer;
int64_t watermark[2]; int32_t colVer;
int32_t ttl; int32_t smaVer;
int32_t numOfColumns; int32_t nextColId;
int32_t numOfTags; int64_t maxdelay[2];
int32_t numOfFuncs; int64_t watermark[2];
int32_t commentLen; int32_t ttl;
int32_t ast1Len; int32_t numOfColumns;
int32_t ast2Len; int32_t numOfTags;
SArray* pFuncs; int32_t numOfFuncs;
SSchema* pColumns; int32_t commentLen;
SSchema* pTags; int32_t ast1Len;
char* comment; int32_t ast2Len;
char* pAst1; SArray* pFuncs;
char* pAst2; SSchema* pColumns;
SRWLatch lock; SSchema* pTags;
int8_t source; char* comment;
char* pAst1;
char* pAst2;
SRWLatch lock;
int8_t source;
SColCmpr* pCmpr;
} SStbObj; } SStbObj;
typedef struct { typedef struct {

View File

@ -29,6 +29,9 @@ void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma);
int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb);
int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist); int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist);
int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t numOfTsmas, void **ppRsp,
int32_t *pRspLen);
int32_t mndGetDbTsmas(SMnode *pMnode, const char *dbFName, uint64_t dbUid, STableTSMAInfoRsp *pRsp, bool *exist);
#ifdef __cplusplus #ifdef __cplusplus
} }

View File

@ -36,7 +36,7 @@
#include "tjson.h" #include "tjson.h"
#define DB_VER_NUMBER 1 #define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 41 #define DB_RESERVE_SIZE 28
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb); static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -49,6 +49,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq);
static int32_t mndProcessDropDbReq(SRpcMsg *pReq); static int32_t mndProcessDropDbReq(SRpcMsg *pReq);
static int32_t mndProcessUseDbReq(SRpcMsg *pReq); static int32_t mndProcessUseDbReq(SRpcMsg *pReq);
static int32_t mndProcessTrimDbReq(SRpcMsg *pReq); static int32_t mndProcessTrimDbReq(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq);
static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity);
static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter);
static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq); static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq);
@ -76,6 +77,7 @@ int32_t mndInitDb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq);
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq); mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB, mndProcessTrimDbReq);
mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB, mndProcessS3MigrateDbReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb);
@ -140,7 +142,11 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER)
SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pDb->compactStartTime, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.keepTimeOffset, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3ChunkSize, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER) SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER);
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -232,11 +238,27 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER)
SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pDb->compactStartTime, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.keepTimeOffset, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3ChunkSize, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER) SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER);
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock); taosInitRWLatch(&pDb->lock);
if (pDb->cfg.s3ChunkSize == 0) {
pDb->cfg.s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
mInfo("db:%s, s3ChunkSize set from %d to default %d", pDb->name, pDb->cfg.s3ChunkSize, TSDB_DEFAULT_S3_CHUNK_SIZE);
}
if (pDb->cfg.s3KeepLocal == 0) {
pDb->cfg.s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
mInfo("db:%s, s3KeepLocal set from %d to default %d", pDb->name, pDb->cfg.s3KeepLocal, TSDB_DEFAULT_S3_KEEP_LOCAL);
}
if (pDb->cfg.tsdbPageSize != TSDB_MIN_TSDB_PAGESIZE) { if (pDb->cfg.tsdbPageSize != TSDB_MIN_TSDB_PAGESIZE) {
mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize, mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize,
TSDB_DEFAULT_TSDB_PAGESIZE); TSDB_DEFAULT_TSDB_PAGESIZE);
@ -322,8 +344,12 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.minRows = pNew->cfg.minRows; pOld->cfg.minRows = pNew->cfg.minRows;
pOld->cfg.maxRows = pNew->cfg.maxRows; pOld->cfg.maxRows = pNew->cfg.maxRows;
pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize;
pOld->cfg.s3ChunkSize = pNew->cfg.s3ChunkSize;
pOld->cfg.s3KeepLocal = pNew->cfg.s3KeepLocal;
pOld->cfg.s3Compact = pNew->cfg.s3Compact;
pOld->cfg.withArbitrator = pNew->cfg.withArbitrator; pOld->cfg.withArbitrator = pNew->cfg.withArbitrator;
pOld->compactStartTime = pNew->compactStartTime; pOld->compactStartTime = pNew->compactStartTime;
pOld->tsmaVersion = pNew->tsmaVersion;
taosWUnLockLatch(&pOld->lock); taosWUnLockLatch(&pOld->lock);
return 0; return 0;
} }
@ -422,6 +448,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1; if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
if (pCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
if (pCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
if (pCfg->s3Compact < TSDB_MIN_S3_COMPACT || pCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
terrno = 0; terrno = 0;
return terrno; return terrno;
} }
@ -471,6 +501,9 @@ static int32_t mndCheckInChangeDbCfg(SMnode *pMnode, SDbCfg *pOldCfg, SDbCfg *pN
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1; return -1;
} }
if (pNewCfg->s3ChunkSize < TSDB_MIN_S3_CHUNK_SIZE || pNewCfg->s3ChunkSize > TSDB_MAX_S3_CHUNK_SIZE) return -1;
if (pNewCfg->s3KeepLocal < TSDB_MIN_S3_KEEP_LOCAL || pNewCfg->s3KeepLocal > TSDB_MAX_S3_KEEP_LOCAL) return -1;
if (pNewCfg->s3Compact < TSDB_MIN_S3_COMPACT || pNewCfg->s3Compact > TSDB_MAX_S3_COMPACT) return -1;
terrno = 0; terrno = 0;
return terrno; return terrno;
@ -507,6 +540,9 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
if (pCfg->s3ChunkSize <= 0) pCfg->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
if (pCfg->s3KeepLocal <= 0) pCfg->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
if (pCfg->s3Compact <= 0) pCfg->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR; if (pCfg->withArbitrator < 0) pCfg->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
} }
@ -648,6 +684,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_DB_FNAME_LEN); dbObj.uid = mndGenerateUid(dbObj.name, TSDB_DB_FNAME_LEN);
dbObj.cfgVersion = 1; dbObj.cfgVersion = 1;
dbObj.vgVersion = 1; dbObj.vgVersion = 1;
dbObj.tsmaVersion = 1;
memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN); memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN);
dbObj.cfg = (SDbCfg){ dbObj.cfg = (SDbCfg){
.numOfVgroups = pCreate->numOfVgroups, .numOfVgroups = pCreate->numOfVgroups,
@ -679,6 +716,9 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
.sstTrigger = pCreate->sstTrigger, .sstTrigger = pCreate->sstTrigger,
.hashPrefix = pCreate->hashPrefix, .hashPrefix = pCreate->hashPrefix,
.hashSuffix = pCreate->hashSuffix, .hashSuffix = pCreate->hashSuffix,
.s3ChunkSize = pCreate->s3ChunkSize,
.s3KeepLocal = pCreate->s3KeepLocal,
.s3Compact = pCreate->s3Compact,
.tsdbPageSize = pCreate->tsdbPageSize, .tsdbPageSize = pCreate->tsdbPageSize,
.withArbitrator = pCreate->withArbitrator, .withArbitrator = pCreate->withArbitrator,
}; };
@ -945,6 +985,18 @@ static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) {
terrno = 0; terrno = 0;
} }
if (pAlter->s3KeepLocal > TSDB_MIN_S3_KEEP_LOCAL && pAlter->s3KeepLocal != pDb->cfg.s3KeepLocal) {
pDb->cfg.s3KeepLocal = pAlter->s3KeepLocal;
pDb->vgVersion++;
terrno = 0;
}
if (pAlter->s3Compact > TSDB_MIN_S3_COMPACT && pAlter->s3Compact != pDb->cfg.s3Compact) {
pDb->cfg.s3Compact = pAlter->s3Compact;
pDb->vgVersion++;
terrno = 0;
}
if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) { if (pAlter->withArbitrator >= TSDB_MIN_DB_WITH_ARBITRATOR && pAlter->withArbitrator != pDb->cfg.withArbitrator) {
pDb->cfg.withArbitrator = pAlter->withArbitrator; pDb->cfg.withArbitrator = pAlter->withArbitrator;
pDb->vgVersion++; pDb->vgVersion++;
@ -1151,6 +1203,9 @@ static void mndDumpDbCfgInfo(SDbCfgRsp *cfgRsp, SDbObj *pDb) {
cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL); cfgRsp->pRetensions = taosArrayDup(pDb->cfg.pRetensions, NULL);
cfgRsp->schemaless = pDb->cfg.schemaless; cfgRsp->schemaless = pDb->cfg.schemaless;
cfgRsp->sstTrigger = pDb->cfg.sstTrigger; cfgRsp->sstTrigger = pDb->cfg.sstTrigger;
cfgRsp->s3ChunkSize = pDb->cfg.s3ChunkSize;
cfgRsp->s3KeepLocal = pDb->cfg.s3KeepLocal;
cfgRsp->s3Compact = pDb->cfg.s3Compact;
cfgRsp->withArbitrator = pDb->cfg.withArbitrator; cfgRsp->withArbitrator = pDb->cfg.withArbitrator;
} }
@ -1218,7 +1273,7 @@ static int32_t mndSetDropDbPrepareLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p
if (pIter == NULL) break; if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) { if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupPrepareLogs(pTrans,pArbGroup) != 0) { if (mndSetDropArbGroupPrepareLogs(pTrans, pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup); sdbRelease(pSdb, pArbGroup);
return -1; return -1;
@ -1246,7 +1301,7 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD
if (pIter == NULL) break; if (pIter == NULL) break;
if (pArbGroup->dbUid == pDb->uid) { if (pArbGroup->dbUid == pDb->uid) {
if (mndSetDropArbGroupCommitLogs(pTrans,pArbGroup) != 0) { if (mndSetDropArbGroupCommitLogs(pTrans, pArbGroup) != 0) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pArbGroup); sdbRelease(pSdb, pArbGroup);
return -1; return -1;
@ -1470,7 +1525,7 @@ static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) {
return numOfTables; return numOfTables;
} }
static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
int32_t vindex = 0; int32_t vindex = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -1631,6 +1686,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
pDbCacheInfo->cfgVersion = htonl(pDbCacheInfo->cfgVersion); pDbCacheInfo->cfgVersion = htonl(pDbCacheInfo->cfgVersion);
pDbCacheInfo->numOfTable = htonl(pDbCacheInfo->numOfTable); pDbCacheInfo->numOfTable = htonl(pDbCacheInfo->numOfTable);
pDbCacheInfo->stateTs = be64toh(pDbCacheInfo->stateTs); pDbCacheInfo->stateTs = be64toh(pDbCacheInfo->stateTs);
pDbCacheInfo->tsmaVersion = htonl(pDbCacheInfo->tsmaVersion);
SDbHbRsp rsp = {0}; SDbHbRsp rsp = {0};
@ -1669,7 +1725,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); int32_t numOfTable = mndGetDBTableNum(pDb, pMnode);
if (pDbCacheInfo->vgVersion >= pDb->vgVersion && pDbCacheInfo->cfgVersion >= pDb->cfgVersion && if (pDbCacheInfo->vgVersion >= pDb->vgVersion && pDbCacheInfo->cfgVersion >= pDb->cfgVersion &&
numOfTable == pDbCacheInfo->numOfTable && pDbCacheInfo->stateTs == pDb->stateTs) { numOfTable == pDbCacheInfo->numOfTable && pDbCacheInfo->stateTs == pDb->stateTs &&
pDbCacheInfo->tsmaVersion >= pDb->tsmaVersion) {
mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64 mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64
" numOfTables:%d, not changed vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d", " numOfTables:%d, not changed vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d",
pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs, pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs,
@ -1688,6 +1745,16 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
mndDumpDbCfgInfo(rsp.cfgRsp, pDb); mndDumpDbCfgInfo(rsp.cfgRsp, pDb);
} }
if (pDbCacheInfo->tsmaVersion != pDb->tsmaVersion) {
rsp.pTsmaRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (rsp.pTsmaRsp) rsp.pTsmaRsp->pTsmas = taosArrayInit(4, POINTER_BYTES);
if (rsp.pTsmaRsp && rsp.pTsmaRsp->pTsmas) {
rsp.dbTsmaVersion = pDb->tsmaVersion;
bool exist = false;
mndGetDbTsmas(pMnode, 0, pDb->uid, rsp.pTsmaRsp, &exist);
}
}
if (pDbCacheInfo->vgVersion < pDb->vgVersion || numOfTable != pDbCacheInfo->numOfTable || if (pDbCacheInfo->vgVersion < pDb->vgVersion || numOfTable != pDbCacheInfo->numOfTable ||
pDbCacheInfo->stateTs != pDb->stateTs) { pDbCacheInfo->stateTs != pDb->stateTs) {
rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp)); rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp));
@ -1798,6 +1865,77 @@ _OVER:
return code; return code;
} }
static int32_t mndS3MigrateDb(SMnode *pMnode, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
int32_t contLen = reqLen + sizeof(SMsgHead);
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
if (pVgroup->dbUid != pDb->uid) continue;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), contLen, &s3migrateReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("vgId:%d, failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
} else {
mInfo("vgId:%d, send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndProcessS3MigrateDbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
int32_t code = -1;
SDbObj *pDb = NULL;
SS3MigrateDbReq s3migrateReq = {0};
if (tDeserializeSS3MigrateDbReq(pReq->pCont, pReq->contLen, &s3migrateReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
mInfo("db:%s, start to s3migrate", s3migrateReq.db);
pDb = mndAcquireDb(pMnode, s3migrateReq.db);
if (pDb == NULL) {
goto _OVER;
}
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_TRIM_DB, pDb) != 0) {
goto _OVER;
}
code = mndS3MigrateDb(pMnode, pDb);
_OVER:
if (code != 0) {
mError("db:%s, failed to process s3migrate db req since %s", s3migrateReq.db, terrstr());
}
mndReleaseDb(pMnode, pDb);
return code;
}
const char *mndGetDbStr(const char *src) { const char *mndGetDbStr(const char *src) {
char *pos = strstr(src, TS_PATH_DELIMITER); char *pos = strstr(src, TS_PATH_DELIMITER);
if (pos != NULL) ++pos; if (pos != NULL) ++pos;
@ -2096,6 +2234,18 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3ChunkSize, false);
char keeplocalVstr[128] = {0};
len = sprintf(&keeplocalVstr[VARSTR_HEADER_SIZE], "%dm", pDb->cfg.s3KeepLocal);
varDataSetLen(keeplocalVstr, len);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)keeplocalVstr, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.s3Compact, false);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false); colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.withArbitrator, false);
} }

View File

@ -668,6 +668,7 @@ _OVER:
if (newStb.pTags != NULL) { if (newStb.pTags != NULL) {
taosMemoryFree(newStb.pTags); taosMemoryFree(newStb.pTags);
taosMemoryFree(newStb.pColumns); taosMemoryFree(newStb.pColumns);
taosMemoryFree(newStb.pCmpr);
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
return code; return code;
@ -784,6 +785,7 @@ static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *p
_OVER: _OVER:
taosMemoryFree(newObj.pTags); taosMemoryFree(newObj.pTags);
taosMemoryFree(newObj.pColumns); taosMemoryFree(newObj.pColumns);
taosMemoryFree(newObj.pCmpr);
mndTransDrop(pTrans); mndTransDrop(pTrans);
mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pStb);

View File

@ -88,7 +88,6 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
return -1; return -1;
} }
*pRsp = *pMeta; *pRsp = *pMeta;
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
@ -130,6 +129,8 @@ int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbN
} }
memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema));
pRsp->pSchemaExt = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchemaExt));
return 0; return 0;
} }

View File

@ -134,6 +134,14 @@ static void mndPullupTtl(SMnode *pMnode) {
} }
static void mndPullupTrimDb(SMnode *pMnode) { static void mndPullupTrimDb(SMnode *pMnode) {
mTrace("pullup s3migrate");
int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_S3MIGRATE_DB_TIMER, .pCont = pReq, .contLen = contLen};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
static void mndPullupS3MigrateDb(SMnode *pMnode) {
mTrace("pullup trim"); mTrace("pullup trim");
int32_t contLen = 0; int32_t contLen = 0;
void *pReq = mndBuildTimerMsg(&contLen); void *pReq = mndBuildTimerMsg(&contLen);
@ -319,6 +327,7 @@ static int32_t minCronTime() {
int32_t min = INT32_MAX; int32_t min = INT32_MAX;
min = TMIN(min, tsTtlPushIntervalSec); min = TMIN(min, tsTtlPushIntervalSec);
min = TMIN(min, tsTrimVDbIntervalSec); min = TMIN(min, tsTrimVDbIntervalSec);
min = TMIN(min, tsS3MigrateIntervalSec);
min = TMIN(min, tsTransPullupInterval); min = TMIN(min, tsTransPullupInterval);
min = TMIN(min, tsCompactPullupInterval); min = TMIN(min, tsCompactPullupInterval);
min = TMIN(min, tsMqRebalanceInterval); min = TMIN(min, tsMqRebalanceInterval);
@ -342,6 +351,10 @@ void mndDoTimerPullupTask(SMnode *pMnode, int64_t sec) {
mndPullupTrimDb(pMnode); mndPullupTrimDb(pMnode);
} }
if (tsS3MigrateEnabled && sec % tsS3MigrateIntervalSec == 0) {
mndPullupS3MigrateDb(pMnode);
}
if (sec % tsTransPullupInterval == 0) { if (sec % tsTransPullupInterval == 0) {
mndPullupTrans(pMnode); mndPullupTrans(pMnode);
} }
@ -806,6 +819,7 @@ _OVER:
pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER || pMsg->msgType == TDMT_MND_COMPACT_TIMER || pMsg->msgType == TDMT_MND_NODECHECK_TIMER ||
pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE || pMsg->msgType == TDMT_MND_GRANT_HB_TIMER || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE ||
pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT || pMsg->msgType == TDMT_MND_STREAM_CHECKPOINT_TIMER || pMsg->msgType == TDMT_MND_STREAM_REQ_CHKPT ||
pMsg->msgType == TDMT_MND_S3MIGRATE_DB_TIMER ||
pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) { pMsg->msgType == TDMT_MND_ARB_HEARTBEAT_TIMER || pMsg->msgType == TDMT_MND_ARB_CHECK_SYNC_TIMER) {
mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored, mTrace("timer not process since mnode restored:%d stopped:%d, sync restored:%d role:%s ", pMnode->restored,
pMnode->stopped, state.restored, syncStr(state.state)); pMnode->stopped, state.restored, syncStr(state.state));
@ -950,7 +964,7 @@ int32_t mndGetMonitorInfo(SMnode *pMnode, SMonClusterInfo *pClusterInfo, SMonVgr
if (pObj->id == pMnode->selfDnodeId) { if (pObj->id == pMnode->selfDnodeId) {
pClusterInfo->first_ep_dnode_id = pObj->id; pClusterInfo->first_ep_dnode_id = pObj->id;
tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep)); tstrncpy(pClusterInfo->first_ep, pObj->pDnode->ep, sizeof(pClusterInfo->first_ep));
//pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f; // pClusterInfo->master_uptime = (float)mndGetClusterUpTime(pMnode) / 86400.0f;
pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode); pClusterInfo->master_uptime = mndGetClusterUpTime(pMnode);
// pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f); // pClusterInfo->master_uptime = (ms - pObj->stateStartTime) / (86400000.0f);
tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role)); tstrncpy(desc.role, syncStr(TAOS_SYNC_STATE_LEADER), sizeof(desc.role));

View File

@ -25,6 +25,7 @@
#include "mndStb.h" #include "mndStb.h"
#include "mndUser.h" #include "mndUser.h"
#include "mndView.h" #include "mndView.h"
#include "mndSma.h"
#include "tglobal.h" #include "tglobal.h"
#include "tversion.h" #include "tversion.h"
@ -606,6 +607,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
break; break;
} }
#endif #endif
case HEARTBEAT_KEY_TSMA: {
void * rspMsg = NULL;
int32_t rspLen = 0;
mndValidateTSMAInfo(pMnode, kv->value, kv->valueLen / sizeof(STSMAVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_TSMA, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv);
}
break;
}
default: default:
mError("invalid kv key:%d", kv->key); mError("invalid kv key:%d", kv->key);
hbRsp.status = TSDB_CODE_APP_ERROR; hbRsp.status = TSDB_CODE_APP_ERROR;

View File

@ -109,7 +109,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
mDebug("mndSetSinkTaskInfo to sma or table, taskId:%s", pTask->id.idStr); mDebug("mndSetSinkTaskInfo to sma or table, taskId:%s", pTask->id.idStr);
if (pStream->smaId != 0) { if (pStream->smaId != 0 && pStream->subTableWithoutMd5 != 1) {
pInfo->type = TASK_OUTPUT__SMA; pInfo->type = TASK_OUTPUT__SMA;
pInfo->smaSink.smaId = pStream->smaId; pInfo->smaSink.smaId = pStream->smaId;
} else { } else {

View File

@ -131,6 +131,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_GRANTS_LOGS; type = TSDB_MGMT_TABLE_GRANTS_LOGS;
} else if (strncasecmp(name, TSDB_INS_TABLE_MACHINES, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_MACHINES, len) == 0) {
type = TSDB_MGMT_TABLE_MACHINES; type = TSDB_MGMT_TABLE_MACHINES;
} else if (strncasecmp(name, TSDB_INS_TABLE_TSMAS, len) == 0) {
type = TSDB_MGMT_TABLE_TSMAS;
} else { } else {
mError("invalid show name:%s len:%d", name, len); mError("invalid show name:%s len:%d", name, len);
} }

File diff suppressed because it is too large Load Diff

View File

@ -33,7 +33,7 @@
#include "mndVgroup.h" #include "mndVgroup.h"
#include "tname.h" #include "tname.h"
#define STB_VER_NUMBER 1 #define STB_VER_NUMBER 2
#define STB_RESERVE_SIZE 64 #define STB_RESERVE_SIZE 64
static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw);
@ -42,6 +42,8 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb);
static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew);
static int32_t mndProcessTtlTimer(SRpcMsg *pReq); static int32_t mndProcessTtlTimer(SRpcMsg *pReq);
static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq); static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq);
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pReq);
static int32_t mndProcessCreateStbReq(SRpcMsg *pReq); static int32_t mndProcessCreateStbReq(SRpcMsg *pReq);
static int32_t mndProcessAlterStbReq(SRpcMsg *pReq); static int32_t mndProcessAlterStbReq(SRpcMsg *pReq);
static int32_t mndProcessDropStbReq(SRpcMsg *pReq); static int32_t mndProcessDropStbReq(SRpcMsg *pReq);
@ -60,6 +62,10 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDb
static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq); static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq);
static int32_t mndProcessDropIndexReq(SRpcMsg *pReq); static int32_t mndProcessDropIndexReq(SRpcMsg *pReq);
static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessDropTbWithTsma(SRpcMsg* pReq);
static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pReq);
int32_t mndInitStb(SMnode *pMnode) { int32_t mndInitStb(SMnode *pMnode) {
SSdbTable table = { SSdbTable table = {
.sdbType = SDB_STB, .sdbType = SDB_STB,
@ -82,7 +88,14 @@ int32_t mndInitStb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq);
mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer); mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB_TIMER, mndProcessTrimDbTimer); mndSetMsgHandle(pMnode, TDMT_MND_TRIM_DB_TIMER, mndProcessTrimDbTimer);
mndSetMsgHandle(pMnode, TDMT_VND_S3MIGRATE_RSP, mndProcessS3MigrateDbRsp);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB_TIMER, mndProcessS3MigrateDbTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq);
mndSetMsgHandle(pMnode, TDMT_MND_STB_DROP, mndProcessDropStbReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_MND_STB_DROP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_TB_WITH_TSMA, mndProcessDropTbWithTsma);
mndSetMsgHandle(pMnode, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mndProcessFetchTtlExpiredTbs);
mndSetMsgHandle(pMnode, TDMT_VND_DROP_TABLE_RSP, mndTransProcessRsp);
// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); // mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq);
// mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq); // mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq);
@ -105,7 +118,8 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + pStb->commentLen + int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + pStb->commentLen +
pStb->ast1Len + pStb->ast2Len + STB_RESERVE_SIZE + taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN; pStb->ast1Len + pStb->ast2Len + pStb->numOfColumns * sizeof(SColCmpr) + STB_RESERVE_SIZE +
taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN;
SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) goto _OVER;
@ -167,6 +181,13 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
SDB_SET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) SDB_SET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER)
} }
if (pStb->pCmpr != NULL) {
for (int i = 0; i < pStb->numOfColumns; i++) {
SColCmpr *p = &pStb->pCmpr[i];
SDB_SET_INT16(pRaw, dataPos, p->id, _OVER)
SDB_SET_INT32(pRaw, dataPos, p->alg, _OVER)
}
}
SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) SDB_SET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -191,7 +212,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
if (sver != STB_VER_NUMBER) { if (sver > STB_VER_NUMBER) {
terrno = TSDB_CODE_SDB_INVALID_DATA_VER; terrno = TSDB_CODE_SDB_INVALID_DATA_VER;
goto _OVER; goto _OVER;
} }
@ -273,6 +294,25 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
if (pStb->pAst2 == NULL) goto _OVER; if (pStb->pAst2 == NULL) goto _OVER;
SDB_GET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER) SDB_GET_BINARY(pRaw, dataPos, pStb->pAst2, pStb->ast2Len, _OVER)
} }
pStb->pCmpr = taosMemoryCalloc(pStb->numOfColumns, sizeof(SColCmpr));
if (sver < STB_VER_NUMBER) {
// compatible with old data, setup default compress value
// impl later
for (int i = 0; i < pStb->numOfColumns; i++) {
SSchema *pSchema = &pStb->pColumns[i];
SColCmpr *pCmpr = &pStb->pCmpr[i];
pCmpr->id = pSchema->colId;
pCmpr->alg = createDefaultColCmprByType(pSchema->type);
}
} else {
for (int i = 0; i < pStb->numOfColumns; i++) {
SColCmpr *pCmpr = &pStb->pCmpr[i];
SDB_GET_INT16(pRaw, dataPos, &pCmpr->id, _OVER)
SDB_GET_INT32(pRaw, dataPos, (int32_t *)&pCmpr->alg, _OVER) // compatiable
}
}
SDB_GET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER) SDB_GET_RESERVE(pRaw, dataPos, STB_RESERVE_SIZE, _OVER)
terrno = 0; terrno = 0;
@ -284,6 +324,7 @@ _OVER:
taosMemoryFreeClear(pStb->pColumns); taosMemoryFreeClear(pStb->pColumns);
taosMemoryFreeClear(pStb->pTags); taosMemoryFreeClear(pStb->pTags);
taosMemoryFreeClear(pStb->comment); taosMemoryFreeClear(pStb->comment);
taosMemoryFree(pStb->pCmpr);
} }
taosMemoryFreeClear(pRow); taosMemoryFreeClear(pRow);
return NULL; return NULL;
@ -300,6 +341,7 @@ void mndFreeStb(SStbObj *pStb) {
taosMemoryFreeClear(pStb->comment); taosMemoryFreeClear(pStb->comment);
taosMemoryFreeClear(pStb->pAst1); taosMemoryFreeClear(pStb->pAst1);
taosMemoryFreeClear(pStb->pAst2); taosMemoryFreeClear(pStb->pAst2);
taosMemoryFreeClear(pStb->pCmpr);
} }
static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) { static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb) {
@ -317,7 +359,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); mTrace("stb:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew);
taosWLockLatch(&pOld->lock); taosWLockLatch(&pOld->lock);
int32_t numOfColumns = pOld->numOfColumns;
if (pOld->numOfColumns < pNew->numOfColumns) { if (pOld->numOfColumns < pNew->numOfColumns) {
void *pColumns = taosMemoryMalloc(pNew->numOfColumns * sizeof(SSchema)); void *pColumns = taosMemoryMalloc(pNew->numOfColumns * sizeof(SSchema));
if (pColumns != NULL) { if (pColumns != NULL) {
@ -405,6 +447,14 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
memcpy(pOld->pAst2, pNew->pAst2, pNew->ast2Len); memcpy(pOld->pAst2, pNew->pAst2, pNew->ast2Len);
pOld->ast2Len = pNew->ast2Len; pOld->ast2Len = pNew->ast2Len;
} }
if (numOfColumns < pNew->numOfColumns) {
taosMemoryFree(pOld->pCmpr);
pOld->pCmpr = taosMemoryCalloc(pNew->numOfColumns, sizeof(SColCmpr));
memcpy(pOld->pCmpr, pNew->pCmpr, pNew->numOfColumns * sizeof(SColCmpr));
} else {
memcpy(pOld->pCmpr, pNew->pCmpr, pNew->numOfColumns * sizeof(SColCmpr));
}
taosWUnLockLatch(&pOld->lock); taosWUnLockLatch(&pOld->lock);
return 0; return 0;
} }
@ -467,6 +517,18 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3
req.schemaTag.version = pStb->tagVer; req.schemaTag.version = pStb->tagVer;
req.schemaTag.pSchema = pStb->pTags; req.schemaTag.pSchema = pStb->pTags;
req.colCmpred = 1;
SColCmprWrapper *pCmpr = &req.colCmpr;
pCmpr->version = pStb->colVer;
pCmpr->nCols = pStb->numOfColumns;
req.colCmpr.pColCmpr = taosMemoryCalloc(pCmpr->nCols, sizeof(SColCmpr));
for (int32_t i = 0; i < pStb->numOfColumns; i++) {
SColCmpr *p = &pCmpr->pColCmpr[i];
p->alg = pStb->pCmpr[i].alg;
p->id = pStb->pCmpr[i].id;
}
if (req.rollup) { if (req.rollup) {
req.rsmaParam.maxdelay[0] = pStb->maxdelay[0]; req.rsmaParam.maxdelay[0] = pStb->maxdelay[0];
req.rsmaParam.maxdelay[1] = pStb->maxdelay[1]; req.rsmaParam.maxdelay[1] = pStb->maxdelay[1];
@ -517,10 +579,12 @@ void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int3
*pContLen = contLen; *pContLen = contLen;
taosMemoryFreeClear(req.rsmaParam.qmsg[0]); taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
taosMemoryFreeClear(req.rsmaParam.qmsg[1]); taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
taosMemoryFreeClear(req.colCmpr.pColCmpr);
return pHead; return pHead;
_err: _err:
taosMemoryFreeClear(req.rsmaParam.qmsg[0]); taosMemoryFreeClear(req.rsmaParam.qmsg[0]);
taosMemoryFreeClear(req.rsmaParam.qmsg[1]); taosMemoryFreeClear(req.rsmaParam.qmsg[1]);
taosMemoryFreeClear(req.colCmpr.pColCmpr);
return NULL; return NULL;
} }
@ -583,7 +647,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) {
} }
for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) {
SField *pField1 = taosArrayGet(pCreate->pColumns, i); SFieldWithOptions *pField1 = taosArrayGet(pCreate->pColumns, i);
if (pField1->type >= TSDB_DATA_TYPE_MAX) { if (pField1->type >= TSDB_DATA_TYPE_MAX) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION; terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1; return -1;
@ -762,9 +826,9 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN); memcpy(pDst->db, pDb->name, TSDB_DB_FNAME_LEN);
pDst->createdTime = taosGetTimestampMs(); pDst->createdTime = taosGetTimestampMs();
pDst->updateTime = pDst->createdTime; pDst->updateTime = pDst->createdTime;
pDst->uid = pDst->uid = (pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX)
(pCreate->source == TD_REQ_FROM_TAOX_OLD || pCreate->source == TD_REQ_FROM_TAOX) ? pCreate->suid
? pCreate->suid : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN); : mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
pDst->dbUid = pDb->uid; pDst->dbUid = pDb->uid;
pDst->tagVer = 1; pDst->tagVer = 1;
pDst->colVer = 1; pDst->colVer = 1;
@ -825,8 +889,8 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
} }
for (int32_t i = 0; i < pDst->numOfColumns; ++i) { for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
SField *pField = taosArrayGet(pCreate->pColumns, i); SFieldWithOptions *pField = taosArrayGet(pCreate->pColumns, i);
SSchema *pSchema = &pDst->pColumns[i]; SSchema *pSchema = &pDst->pColumns[i];
pSchema->type = pField->type; pSchema->type = pField->type;
pSchema->bytes = pField->bytes; pSchema->bytes = pField->bytes;
pSchema->flags = pField->flags; pSchema->flags = pField->flags;
@ -847,6 +911,16 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
pSchema->colId = pDst->nextColId; pSchema->colId = pDst->nextColId;
pDst->nextColId++; pDst->nextColId++;
} }
// set col compress
pDst->pCmpr = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SCmprObj));
for (int32_t i = 0; i < pDst->numOfColumns; i++) {
SFieldWithOptions *pField = taosArrayGet(pCreate->pColumns, i);
SSchema *pSchema = &pDst->pColumns[i];
SColCmpr *pColCmpr = &pDst->pCmpr[i];
pColCmpr->id = pSchema->colId;
pColCmpr->alg = pField->compress;
}
return 0; return 0;
} }
static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) { static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) {
@ -932,7 +1006,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
pHead->vgId = htonl(pVgroup->vgId); pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq); tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen, .info = pReq->info}; SRpcMsg rpcMsg = {.msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS, .pCont = pHead, .contLen = contLen, .info = pReq->info};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg); int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) { if (code != 0) {
@ -983,6 +1057,43 @@ static int32_t mndProcessTrimDbTimer(SRpcMsg *pReq) {
return 0; return 0;
} }
static int32_t mndProcessS3MigrateDbTimer(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
SVgObj *pVgroup = NULL;
void *pIter = NULL;
SVS3MigrateDbReq s3migrateReq = {.timestamp = taosGetTimestampSec()};
int32_t reqLen = tSerializeSVS3MigrateDbReq(NULL, 0, &s3migrateReq);
int32_t contLen = reqLen + sizeof(SMsgHead);
while (1) {
pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup);
if (pIter == NULL) break;
SMsgHead *pHead = rpcMallocCont(contLen);
if (pHead == NULL) {
sdbCancelFetch(pSdb, pVgroup);
sdbRelease(pSdb, pVgroup);
continue;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVS3MigrateDbReq((char *)pHead + sizeof(SMsgHead), reqLen, &s3migrateReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_S3MIGRATE, .pCont = pHead, .contLen = contLen};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
mError("vgId:%d, timer failed to send vnode-s3migrate request to vnode since 0x%x", pVgroup->vgId, code);
} else {
mInfo("vgId:%d, timer send vnode-s3migrate request to vnode, time:%d", pVgroup->vgId, s3migrateReq.timestamp);
}
sdbRelease(pSdb, pVgroup);
}
return 0;
}
static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) { static int32_t mndFindSuperTableTagIndex(const SStbObj *pStb, const char *tagName) {
for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { for (int32_t tag = 0; tag < pStb->numOfTags; tag++) {
if (strcmp(pStb->pTags[tag].name, tagName) == 0) { if (strcmp(pStb->pTags[tag].name, tagName) == 0) {
@ -1028,7 +1139,9 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
pDst->numOfTags = createReq->numOfTags; pDst->numOfTags = createReq->numOfTags;
pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema)); pDst->pColumns = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SSchema));
pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema)); pDst->pTags = taosMemoryCalloc(1, pDst->numOfTags * sizeof(SSchema));
if (pDst->pColumns == NULL || pDst->pTags == NULL) { pDst->pCmpr = taosMemoryCalloc(1, pDst->numOfColumns * sizeof(SColCmpr));
if (pDst->pColumns == NULL || pDst->pTags == NULL || pDst->pCmpr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -1039,8 +1152,8 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
} }
for (int32_t i = 0; i < pDst->numOfColumns; ++i) { for (int32_t i = 0; i < pDst->numOfColumns; ++i) {
SField *pField = taosArrayGet(createReq->pColumns, i); SFieldWithOptions *pField = taosArrayGet(createReq->pColumns, i);
SSchema *pSchema = &pDst->pColumns[i]; SSchema *pSchema = &pDst->pColumns[i];
pSchema->type = pField->type; pSchema->type = pField->type;
pSchema->bytes = pField->bytes; pSchema->bytes = pField->bytes;
pSchema->flags = pField->flags; pSchema->flags = pField->flags;
@ -1066,6 +1179,17 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
pSchema->colId = pDst->nextColId++; pSchema->colId = pDst->nextColId++;
} }
} }
for (int32_t i = 0; i < pDst->numOfColumns; i++) {
SColCmpr *p = pDst->pCmpr + i;
SFieldWithOptions *pField = taosArrayGet(createReq->pColumns, i);
SSchema *pSchema = &pDst->pColumns[i];
p->id = pSchema->colId;
if (pField->compress == 0) {
p->alg = createDefaultColCmprByType(pSchema->type);
} else {
p->alg = pField->compress;
}
}
pDst->tagVer = createReq->tagVer; pDst->tagVer = createReq->tagVer;
pDst->colVer = createReq->colVer; pDst->colVer = createReq->colVer;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1110,10 +1234,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name); mInfo("stb:%s, schema version is not incremented and nothing needs to be done", createReq.name);
code = 0; code = 0;
goto _OVER; goto _OVER;
} else if ((tagDelta == 1 && colDelta == 0) || } else if ((tagDelta == 1 && colDelta == 0) || (tagDelta == 0 && colDelta == 1) ||
(tagDelta == 0 && colDelta == 1) || (pStb->colVer == 1 && createReq.colVer > 1) || (pStb->tagVer == 1 && createReq.tagVer > 1)) {
(pStb->colVer == 1 && createReq.colVer > 1) ||
(pStb->tagVer == 1 && createReq.tagVer > 1)) {
isAlter = true; isAlter = true;
mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name); mInfo("stb:%s, schema version is only increased by 1 number, do alter operation", createReq.name);
} else { } else {
@ -1133,7 +1255,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
} }
} else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) {
goto _OVER; goto _OVER;
} else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) && (createReq.tagVer != 1 || createReq.colVer != 1)) { } else if ((createReq.source == TD_REQ_FROM_TAOX_OLD || createReq.source == TD_REQ_FROM_TAOX) &&
(createReq.tagVer != 1 || createReq.colVer != 1)) {
mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name); mInfo("stb:%s, alter table does not need to be done, because table is deleted", createReq.name);
code = 0; code = 0;
goto _OVER; goto _OVER;
@ -1170,12 +1293,14 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
if (mndBuildStbFromAlter(pStb, &pDst, &createReq) != 0) { if (mndBuildStbFromAlter(pStb, &pDst, &createReq) != 0) {
taosMemoryFreeClear(pDst.pTags); taosMemoryFreeClear(pDst.pTags);
taosMemoryFreeClear(pDst.pColumns); taosMemoryFreeClear(pDst.pColumns);
taosMemoryFreeClear(pDst.pCmpr);
goto _OVER; goto _OVER;
} }
code = mndAlterStbImp(pMnode, pReq, pDb, &pDst, needRsp, NULL, 0); code = mndAlterStbImp(pMnode, pReq, pDb, &pDst, needRsp, NULL, 0);
taosMemoryFreeClear(pDst.pTags); taosMemoryFreeClear(pDst.pTags);
taosMemoryFreeClear(pDst.pColumns); taosMemoryFreeClear(pDst.pColumns);
taosMemoryFreeClear(pDst.pCmpr);
} else { } else {
code = mndCreateStb(pMnode, pReq, &createReq, pDb); code = mndCreateStb(pMnode, pReq, &createReq, pDb);
} }
@ -1184,14 +1309,13 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
SName name = {0}; SName name = {0};
tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if(createReq.sql == NULL && createReq.sqlLen == 0){ if (createReq.sql == NULL && createReq.sqlLen == 0) {
char detail[1000] = {0}; char detail[1000] = {0};
sprintf(detail, "dbname:%s, stable name:%s", name.dbname, name.tname); sprintf(detail, "dbname:%s, stable name:%s", name.dbname, name.tname);
auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, detail, strlen(detail)); auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, detail, strlen(detail));
} } else {
else{
auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, createReq.sql, createReq.sqlLen); auditRecord(pReq, pMnode->clusterId, "createStb", name.dbname, name.tname, createReq.sql, createReq.sqlLen);
} }
_OVER: _OVER:
@ -1229,13 +1353,16 @@ static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) {
pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema)); pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema));
pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema)); pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema));
if (pNew->pTags == NULL || pNew->pColumns == NULL) { pNew->pCmpr = taosMemoryCalloc(pNew->numOfColumns, sizeof(SColCmpr));
if (pNew->pTags == NULL || pNew->pColumns == NULL || pNew->pCmpr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
memcpy(pNew->pColumns, pOld->pColumns, sizeof(SSchema) * pOld->numOfColumns); memcpy(pNew->pColumns, pOld->pColumns, sizeof(SSchema) * pOld->numOfColumns);
memcpy(pNew->pTags, pOld->pTags, sizeof(SSchema) * pOld->numOfTags); memcpy(pNew->pTags, pOld->pTags, sizeof(SSchema) * pOld->numOfTags);
memcpy(pNew->pCmpr, pOld->pCmpr, sizeof(SColCmpr) * pOld->numOfColumns);
return 0; return 0;
} }
@ -1603,6 +1730,53 @@ static int32_t mndAlterStbTagBytes(SMnode *pMnode, const SStbObj *pOld, SStbObj
return 0; return 0;
} }
static int32_t mndUpdateSuperTableColumnCompress(SMnode *pMnode, const SStbObj *pOld, SStbObj *pNew, SArray *pField,
int32_t nCols) {
// if (pColCmpr == NULL || colName == NULL) return -1;
ASSERT(taosArrayGetSize(pField) == nCols);
TAOS_FIELD *p = taosArrayGet(pField, 0);
int32_t code = 0;
int32_t idx = mndFindSuperTableColumnIndex(pOld, p->name);
if (idx == -1) {
terrno = TSDB_CODE_MND_COLUMN_NOT_EXIST;
return -1;
}
SSchema *pTarget = &pOld->pColumns[idx];
col_id_t colId = pTarget->colId;
if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) {
return -1;
}
if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1;
}
if (!validColCmprByType(pTarget->type, p->bytes)) {
terrno = TSDB_CODE_TSC_ENCODE_PARAM_ERROR;
return -1;
}
int8_t updated = 0;
for (int i = 0; i < pNew->numOfColumns; i++) {
SColCmpr *pCmpr = &pNew->pCmpr[i];
if (pCmpr->id == colId) {
uint32_t dst = 0;
updated = tUpdateCompress(pCmpr->alg, p->bytes, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED,
TSDB_COLVAL_LEVEL_MEDIUM, &dst);
if (updated) pCmpr->alg = dst;
break;
}
}
if (updated == 0) {
terrno = TSDB_CODE_MND_COLUMN_COMPRESS_ALREADY_EXIST;
return -1;
}
pNew->colVer++;
return 0;
}
static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols) { static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray *pFields, int32_t ncols) {
if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) { if (pOld->numOfColumns + ncols + pOld->numOfTags > TSDB_MAX_COLUMNS) {
terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS; terrno = TSDB_CODE_MND_TOO_MANY_COLUMNS;
@ -1619,6 +1793,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
} }
pNew->numOfColumns = pNew->numOfColumns + ncols; pNew->numOfColumns = pNew->numOfColumns + ncols;
if (mndAllocStbSchemas(pOld, pNew) != 0) { if (mndAllocStbSchemas(pOld, pNew) != 0) {
return -1; return -1;
} }
@ -1647,6 +1822,10 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray
pSchema->colId = pNew->nextColId; pSchema->colId = pNew->nextColId;
pNew->nextColId++; pNew->nextColId++;
SColCmpr *pCmpr = &pNew->pCmpr[pOld->numOfColumns + i];
pCmpr->id = pSchema->colId;
pCmpr->alg = createDefaultColCmprByType(pSchema->type);
mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name); mInfo("stb:%s, start to add column %s", pNew->name, pSchema->name);
} }
@ -1680,7 +1859,9 @@ static int32_t mndDropSuperTableColumn(SMnode *pMnode, const SStbObj *pOld, SStb
return -1; return -1;
} }
memmove(pNew->pColumns + col, pNew->pColumns + col + 1, sizeof(SSchema) * (pNew->numOfColumns - col - 1)); int32_t sz = pNew->numOfColumns - col - 1;
memmove(pNew->pColumns + col, pNew->pColumns + col + 1, sizeof(SSchema) * sz);
memmove(pNew->pCmpr + col, pNew->pCmpr + col + 1, sizeof(SColCmpr) * sz);
pNew->numOfColumns--; pNew->numOfColumns--;
pNew->colVer++; pNew->colVer++;
@ -1843,6 +2024,12 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
pRsp->pSchemaExt = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchemaExt));
if (pRsp->pSchemaExt == NULL) {
taosRUnLockLatch(&pStb->lock);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName));
tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName)); tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName));
@ -1876,6 +2063,12 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa
pSchema->colId = pSrcSchema->colId; pSchema->colId = pSrcSchema->colId;
pSchema->bytes = pSrcSchema->bytes; pSchema->bytes = pSrcSchema->bytes;
} }
for (int32_t i = 0; i < pStb->numOfColumns; i++) {
SColCmpr *pCmpr = &pStb->pCmpr[i];
SSchemaExt *pSchEx = &pRsp->pSchemaExt[i];
pSchEx->colId = pCmpr->id;
pSchEx->compress = pCmpr->alg;
}
taosRUnLockLatch(&pStb->lock); taosRUnLockLatch(&pStb->lock);
return 0; return 0;
@ -1932,11 +2125,20 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
pRsp->pFuncs = taosArrayDup(pStb->pFuncs, NULL); pRsp->pFuncs = taosArrayDup(pStb->pFuncs, NULL);
} }
pRsp->pSchemaExt = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchemaExt));
for (int32_t i = 0; i < pStb->numOfColumns; i++) {
SColCmpr *pCmpr = &pStb->pCmpr[i];
SSchemaExt *pSchExt = &pRsp->pSchemaExt[i];
pSchExt->colId = pCmpr->id;
pSchExt->compress = pCmpr->alg;
}
taosRUnLockLatch(&pStb->lock); taosRUnLockLatch(&pStb->lock);
return 0; return 0;
} }
static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bool* schema, bool* sma) { static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion *pStbVer, bool *schema, bool *sma) {
char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName); snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName);
@ -2234,6 +2436,8 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
taosRUnLockLatch(&pOld->lock); taosRUnLockLatch(&pOld->lock);
stbObj.pColumns = NULL; stbObj.pColumns = NULL;
stbObj.pTags = NULL; stbObj.pTags = NULL;
stbObj.pFuncs = NULL;
stbObj.pCmpr = NULL;
stbObj.updateTime = taosGetTimestampMs(); stbObj.updateTime = taosGetTimestampMs();
stbObj.lock = 0; stbObj.lock = 0;
bool updateTagIndex = false; bool updateTagIndex = false;
@ -2269,6 +2473,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
needRsp = false; needRsp = false;
code = mndUpdateStbCommentAndTTL(pOld, &stbObj, pAlter->comment, pAlter->commentLen, pAlter->ttl); code = mndUpdateStbCommentAndTTL(pOld, &stbObj, pAlter->comment, pAlter->commentLen, pAlter->ttl);
break; break;
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS:
code = mndUpdateSuperTableColumnCompress(pMnode, pOld, &stbObj, pAlter->pFields, pAlter->numOfFields);
break;
default: default:
needRsp = false; needRsp = false;
terrno = TSDB_CODE_OPS_NOT_SUPPORT; terrno = TSDB_CODE_OPS_NOT_SUPPORT;
@ -2285,6 +2492,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p
_OVER: _OVER:
taosMemoryFreeClear(stbObj.pTags); taosMemoryFreeClear(stbObj.pTags);
taosMemoryFreeClear(stbObj.pColumns); taosMemoryFreeClear(stbObj.pColumns);
taosMemoryFreeClear(stbObj.pCmpr);
if (pAlter->commentLen > 0) { if (pAlter->commentLen > 0) {
taosMemoryFreeClear(stbObj.comment); taosMemoryFreeClear(stbObj.comment);
} }
@ -2491,11 +2699,6 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break; if (pIter == NULL) break;
if (pStream->smaId != 0) {
sdbRelease(pSdb, pStream);
continue;
}
if (pStream->targetStbUid == suid) { if (pStream->targetStbUid == suid) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
@ -2537,6 +2740,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessDropTtltbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessTrimDbRsp(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessTrimDbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessS3MigrateDbRsp(SRpcMsg *pRsp) { return 0; }
static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
@ -2750,8 +2954,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
pStbVersion->tversion = ntohl(pStbVersion->tversion); pStbVersion->tversion = ntohl(pStbVersion->tversion);
pStbVersion->smaVer = ntohl(pStbVersion->smaVer); pStbVersion->smaVer = ntohl(pStbVersion->smaVer);
bool schema = false; bool schema = false;
bool sma = false; bool sma = false;
int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma); int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
STableMetaRsp metaRsp = {0}; STableMetaRsp metaRsp = {0};
@ -3185,6 +3389,11 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
continue; continue;
} }
if (isTsmaResSTb(pStb->name)) {
sdbRelease(pSdb, pStb);
continue;
}
cols = 0; cols = 0;
SName name = {0}; SName name = {0};
@ -3651,3 +3860,376 @@ static int32_t mndProcessDropIndexReq(SRpcMsg *pReq) {
_OVER: _OVER:
return code; return code;
}*/ }*/
static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessDropStbReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}
typedef struct SVDropTbVgReqs {
SVDropTbBatchReq req;
SVgroupInfo info;
} SVDropTbVgReqs;
typedef struct SMDropTbDbInfo {
SArray *dbVgInfos;
int32_t hashPrefix;
int32_t hashSuffix;
int32_t hashMethod;
} SMDropTbDbInfo;
typedef struct SMDropTbTsmaInfo {
char tsmaResTbDbFName[TSDB_DB_FNAME_LEN];
char tsmaResTbNamePrefix[TSDB_TABLE_NAME_LEN];
int32_t suid;
SMDropTbDbInfo dbInfo; // reference to DbInfo in pDbMap
} SMDropTbTsmaInfo;
typedef struct SMDropTbTsmaInfos {
SArray* pTsmaInfos; // SMDropTbTsmaInfo
} SMDropTbTsmaInfos;
typedef struct SMndDropTbsWithTsmaCtx {
SHashObj* pTsmaMap; // <suid, SMDropTbTsmaInfos>
SHashObj* pDbMap; // <dbuid, SMDropTbDbInfo>
SHashObj* pVgMap; // <vgId, SVDropTbVgReqs>
SArray* pResTbNames; // SArray<char*>
} SMndDropTbsWithTsmaCtx;
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode* pMnode, SMndDropTbsWithTsmaCtx* pCtx, SArray* pTbs, int32_t vgId);
static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx* p) {
if (!p) return;
if (p->pDbMap) {
void* pIter = taosHashIterate(p->pDbMap, NULL);
while (pIter) {
SMDropTbDbInfo* pInfo = pIter;
taosArrayDestroy(pInfo->dbVgInfos);
pIter = taosHashIterate(p->pDbMap, pIter);
}
taosHashCleanup(p->pDbMap);
}
if (p->pResTbNames) {
taosArrayDestroyP(p->pResTbNames, taosMemoryFree);
}
if (p->pTsmaMap) {
void* pIter = taosHashIterate(p->pTsmaMap, NULL);
while (pIter) {
SMDropTbTsmaInfos* pInfos = pIter;
taosArrayDestroy(pInfos->pTsmaInfos);
pIter = taosHashIterate(p->pTsmaMap, pIter);
}
taosHashCleanup(p->pTsmaMap);
}
if (p->pVgMap) {
void* pIter = taosHashIterate(p->pVgMap, NULL);
while (pIter) {
SVDropTbVgReqs *pReqs = pIter;
taosArrayDestroy(pReqs->req.pArray);
pIter = taosHashIterate(p->pVgMap, pIter);
}
taosHashCleanup(p->pVgMap);
}
taosMemoryFree(p);
}
static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx** ppCtx) {
int32_t code = 0;
SMndDropTbsWithTsmaCtx* pCtx = taosMemoryCalloc(1, sizeof(SMndDropTbsWithTsmaCtx));
if (!pCtx) return TSDB_CODE_OUT_OF_MEMORY;
pCtx->pTsmaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (!pCtx->pTsmaMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
pCtx->pDbMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (!pCtx->pDbMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
pCtx->pResTbNames = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
pCtx->pVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (!pCtx->pVgMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
*ppCtx = pCtx;
_end:
if (code) mndDestroyDropTbsWithTsmaCtx(pCtx);
return code;
}
static void* mndBuildVDropTbsReq(SMnode* pMnode, const SVgroupInfo* pVgInfo, const SVDropTbBatchReq* pReq, int32_t *len) {
int32_t contLen = 0;
int32_t ret = 0;
SMsgHead *pHead = NULL;
SEncoder encoder = {0};
tEncodeSize(tEncodeSVDropTbBatchReq, pReq, contLen, ret);
if (ret < 0) return NULL;
contLen += sizeof(SMsgHead);
pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgInfo->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
tEncodeSVDropTbBatchReq(&encoder, pReq);
tEncoderClear(&encoder);
*len = contLen;
return pHead;
}
static int32_t mndSetDropTbsRedoActions(SMnode* pMnode, STrans* pTrans, const SVDropTbVgReqs* pVgReqs, void* pCont, int32_t contLen) {
STransAction action = {0};
action.epSet = pVgReqs->info.epSet;
action.pCont = pCont;
action.contLen = contLen;
action.msgType = TDMT_VND_DROP_TABLE;
action.acceptableCode = TSDB_CODE_TDB_TABLE_NOT_EXIST;
return mndTransAppendRedoAction(pTrans, &action);
}
static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg* pRsp, SMndDropTbsWithTsmaCtx* pCtx) {
SMnode *pMnode = pRsp->info.node;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pRsp, "drop-tbs");
mndTransSetChangeless(pTrans);
if (pTrans == NULL) goto _OVER;
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
void* pIter = taosHashIterate(pCtx->pVgMap, NULL);
while (pIter) {
const SVDropTbVgReqs* pVgReqs = pIter;
int32_t len = 0;
void* p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len);
if (!p || mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len) != 0) {
taosHashCancelIterate(pCtx->pVgMap, pIter);
goto _OVER;
}
pIter = taosHashIterate(pCtx->pVgMap, pIter);
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
_OVER:
mndTransDrop(pTrans);
return terrno;
}
static int32_t mndProcessDropTbWithTsma(SRpcMsg* pReq) {
int32_t code = -1;
SMnode *pMnode = pReq->info.node;
SDbObj *pDb = NULL;
SStbObj *pStb = NULL;
SMDropTbsReq dropReq = {0};
bool locked = false;
if (tDeserializeSMDropTbsReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
SMndDropTbsWithTsmaCtx* pCtx = NULL;
terrno = mndInitDropTbsWithTsmaCtx(&pCtx);
if (terrno) goto _OVER;
for (int32_t i = 0; i < dropReq.pVgReqs->size; ++i) {
SMDropTbReqsOnSingleVg* pReq = taosArrayGet(dropReq.pVgReqs, i);
terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId);
if (terrno) goto _OVER;
}
if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0)
code = 0;
_OVER:
tFreeSMDropTbsReq(&dropReq);
if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx);
return code;
}
static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid,
bool ignoreNotExists) {
SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists};
SVDropTbVgReqs * pReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
SVDropTbVgReqs reqs = {0};
if (pReqs == NULL) {
reqs.info = *pVgInfo;
reqs.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq));
taosArrayPush(reqs.req.pArray, &req);
taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &reqs, sizeof(reqs));
} else {
taosArrayPush(pReqs->req.pArray, &req);
}
return 0;
}
static int32_t mndGetDbVgInfoForTsma(SMnode* pMnode, const char* dbname, SMDropTbTsmaInfo* pInfo) {
int32_t code = 0;
SDbObj* pDb = mndAcquireDb(pMnode, dbname);
if (!pDb) {
code = TSDB_CODE_MND_DB_NOT_EXIST;
goto _end;
}
pInfo->dbInfo.dbVgInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if ( !pInfo->dbInfo.dbVgInfos) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos);
pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix;
pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix;
pInfo->dbInfo.hashMethod = pDb->cfg.hashMethod;
_end:
if (pDb) mndReleaseDb(pMnode, pDb);
if (code && pInfo->dbInfo.dbVgInfos) {
taosArrayDestroy(pInfo->dbInfo.dbVgInfos);
pInfo->dbInfo.dbVgInfos = NULL;
}
return code;
}
int32_t vgHashValCmp(const void* lp, const void* rp) {
uint32_t* key = (uint32_t*)lp;
SVgroupInfo* pVg = (SVgroupInfo*)rp;
if (*key < pVg->hashBegin) {
return -1;
} else if (*key > pVg->hashEnd) {
return 1;
}
return 0;
}
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode* pMnode, SMndDropTbsWithTsmaCtx* pCtx, SArray* pTbs, int32_t vgId) {
int32_t code = 0;
SVgObj* pVgObj = mndAcquireVgroup(pMnode, vgId);
if (!pVgObj) {
code = 0;
goto _end;
}
SVgroupInfo vgInfo = {.hashBegin = pVgObj->hashBegin, .hashEnd = pVgObj->hashEnd, .numOfTable = pVgObj->numOfTables, .vgId = pVgObj->vgId};
vgInfo.epSet = mndGetVgroupEpset(pMnode, pVgObj);
mndReleaseVgroup(pMnode, pVgObj);
// get all stb uids
for (int32_t i = 0; i < pTbs->size; ++i) {
const SVDropTbReq* pTb = taosArrayGet(pTbs, i);
if (taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid))) {
} else {
SMDropTbTsmaInfos infos = {0};
infos.pTsmaInfos = taosArrayInit(2, sizeof(SMDropTbTsmaInfo));
if (!infos.pTsmaInfos) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
taosHashPut(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid), &infos, sizeof(infos));
}
}
void *pIter = NULL;
SSmaObj *pSma = NULL;
char buf[TSDB_TABLE_FNAME_LEN] = {0};
// get used tsmas and it's dbs
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma);
if (!pIter) break;
SMDropTbTsmaInfos* pInfos = taosHashGet(pCtx->pTsmaMap, &pSma->stbUid, sizeof(pSma->stbUid));
if (pInfos) {
SMDropTbTsmaInfo info = {0};
int32_t len = sprintf(buf, "%s", pSma->name);
len = taosCreateMD5Hash(buf, len);
sprintf(info.tsmaResTbDbFName, "%s", pSma->db);
snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_NAME_LEN, "%s", buf);
SMDropTbDbInfo* pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN);
info.suid = pSma->dstTbUid;
if (!pDbInfo) {
code = mndGetDbVgInfoForTsma(pMnode, pSma->db, &info);
if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pSma);
goto _end;
}
taosHashPut(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN, &info.dbInfo, sizeof(SMDropTbDbInfo));
} else {
info.dbInfo = *pDbInfo;
}
taosArrayPush(pInfos->pTsmaInfos, &info);
}
sdbRelease(pMnode->pSdb, pSma);
}
// generate vg req map
for (int32_t i = 0; i < pTbs->size; ++i) {
SVDropTbReq* pTb = taosArrayGet(pTbs, i);
mndDropTbAdd(pMnode, pCtx->pVgMap, &vgInfo, pTb->name, pTb->suid, pTb->igNotExists);
SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid));
SArray *pVgInfos = NULL;
char buf[TSDB_TABLE_FNAME_LEN];
for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) {
SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j);
int32_t len = sprintf(buf, "%s.%s_%s", pInfo->tsmaResTbDbFName, pInfo->tsmaResTbNamePrefix, pTb->name);
uint32_t hashVal =
taosGetTbHashVal(buf, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix, pInfo->dbInfo.hashSuffix);
const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ);
void* p = taosStrdup(buf + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN);
taosArrayPush(pCtx->pResTbNames, &p);
mndDropTbAdd(pMnode, pCtx->pVgMap, pVgInfo, p, pInfo->suid, true);
}
}
_end:
return code;
}
static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) {
int32_t code = -1;
SDecoder decoder = {0};
SMnode *pMnode = pRsp->info.node;
SVFetchTtlExpiredTbsRsp rsp = {0};
SMndDropTbsWithTsmaCtx *pCtx = NULL;
if (pRsp->code != TSDB_CODE_SUCCESS) goto _end;
if (pRsp->contLen == 0) {
code = 0;
goto _end;
}
tDecoderInit(&decoder, pRsp->pCont, pRsp->contLen);
terrno = tDecodeVFetchTtlExpiredTbsRsp(&decoder, &rsp);
if (terrno) goto _end;
terrno = mndInitDropTbsWithTsmaCtx(&pCtx);
if (terrno) goto _end;
terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId);
if (terrno) goto _end;
if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0)
code = 0;
_end:
if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx);
tDecoderClear(&decoder);
tFreeFetchTtlExpiredTbsRsp(&rsp);
return code;
}

View File

@ -41,6 +41,10 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream); static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream);
static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq);
@ -102,6 +106,13 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp);
// for msgs inside mnode
// TODO change the name
mndSetMsgHandle(pMnode, TDMT_STREAM_CREATE, mndProcessCreateStreamReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_STREAM_CREATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_DROP, mndProcessDropStreamReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_STREAM_DROP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
@ -335,7 +346,10 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
pObj->createTime = taosGetTimestampMs(); pObj->createTime = taosGetTimestampMs();
pObj->updateTime = pObj->createTime; pObj->updateTime = pObj->createTime;
pObj->version = 1; pObj->version = 1;
pObj->smaId = 0; if (pCreate->smaId > 0) {
pObj->subTableWithoutMd5 = 1;
}
pObj->smaId = pCreate->smaId;
pObj->indexForMultiAggBalance = -1; pObj->indexForMultiAggBalance = -1;
pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name)); pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name));
@ -555,15 +569,16 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN);
createReq.numOfColumns = pStream->outputSchema.nCols; createReq.numOfColumns = pStream->outputSchema.nCols;
createReq.numOfTags = 1; // group id createReq.numOfTags = 1; // group id
createReq.pColumns = taosArrayInit_s(sizeof(SField), createReq.numOfColumns); createReq.pColumns = taosArrayInit_s(sizeof(SFieldWithOptions), createReq.numOfColumns);
// build fields // build fields
for (int32_t i = 0; i < createReq.numOfColumns; i++) { for (int32_t i = 0; i < createReq.numOfColumns; i++) {
SField *pField = taosArrayGet(createReq.pColumns, i); SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i);
tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN); tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN);
pField->flags = pStream->outputSchema.pSchema[i].flags; pField->flags = pStream->outputSchema.pSchema[i].flags;
pField->type = pStream->outputSchema.pSchema[i].type; pField->type = pStream->outputSchema.pSchema[i].type;
pField->bytes = pStream->outputSchema.pSchema[i].bytes; pField->bytes = pStream->outputSchema.pSchema[i].bytes;
pField->compress = createDefaultColCmprByType(pField->type);
} }
if (pStream->tagSchema.nCols == 0) { if (pStream->tagSchema.nCols == 0) {
@ -642,7 +657,9 @@ _OVER:
return -1; return -1;
} }
static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { // check for number of existed tasks // 1. stream number check
// 2. target stable can not be target table of other existed streams.
static int32_t doStreamCheck(SMnode *pMnode, SStreamObj *pStreamObj) {
int32_t numOfStream = 0; int32_t numOfStream = 0;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -655,14 +672,16 @@ static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { /
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
if (numOfStream > MND_STREAM_MAX_NUM) { if (numOfStream > MND_STREAM_MAX_NUM) {
mError("too many streams, no more than %d for each database", MND_STREAM_MAX_NUM); mError("too many streams, no more than %d for each database, failed to create stream:%s", MND_STREAM_MAX_NUM,
pStreamObj->name);
sdbCancelFetch(pMnode->pSdb, pIter); sdbCancelFetch(pMnode->pSdb, pIter);
terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; terrno = TSDB_CODE_MND_TOO_MANY_STREAMS;
return terrno; return terrno;
} }
if (pStream->targetStbUid == pStreamObj->targetStbUid) { if (pStream->targetStbUid == pStreamObj->targetStbUid) {
mError("Cannot write the same stable as other stream:%s", pStream->name); mError("Cannot write the same stable as other stream:%s, failed to create stream:%s", pStream->name,
pStreamObj->name);
sdbCancelFetch(pMnode->pSdb, pIter); sdbCancelFetch(pMnode->pSdb, pIter);
terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE; terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE;
return terrno; return terrno;
@ -727,11 +746,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER; goto _OVER;
} }
if (checkForNumOfStreams(pMnode, &streamObj) < 0) { if (doStreamCheck(pMnode, &streamObj) < 0) {
goto _OVER; goto _OVER;
} }
STrans *pTrans = doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, "create stream tasks on dnodes"); STrans *pTrans =
doCreateTrans(pMnode, &streamObj, pReq, TRN_CONFLICT_DB, MND_STREAM_CREATE_NAME, "create stream tasks on dnodes");
if (pTrans == NULL) { if (pTrans == NULL) {
goto _OVER; goto _OVER;
} }
@ -817,23 +837,57 @@ _OVER:
return terrno; return terrno;
} }
int64_t mndStreamGenChkpId(SMnode *pMnode) { int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int64_t maxChkpId = 0; int64_t maxChkptId = 0;
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break; if (pIter == NULL) break;
maxChkpId = TMAX(maxChkpId, pStream->checkpointId); maxChkptId = TMAX(maxChkptId, pStream->checkpointId);
mDebug("stream:%p, %s id:%" PRIx64 "checkpoint %" PRId64 "", pStream, pStream->name, pStream->uid, mDebug("stream:%p, %s id:%" PRIx64 "checkpoint %" PRId64 "", pStream, pStream->name, pStream->uid,
pStream->checkpointId); pStream->checkpointId);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
} }
mDebug("generated checkpoint %" PRId64 "", maxChkpId + 1); { // check the max checkpoint id from all vnodes.
return maxChkpId + 1; int64_t maxCheckpointId = -1;
if (lock) {
taosThreadMutexLock(&execInfo.lock);
}
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
if (pEntry == NULL) {
continue;
}
if (pEntry->checkpointInfo.failed) {
continue;
}
if (maxCheckpointId < pEntry->checkpointInfo.latestId) {
maxCheckpointId = pEntry->checkpointInfo.latestId;
}
}
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);
}
if (maxCheckpointId > maxChkptId) {
mDebug("max checkpointId in mnode:%" PRId64 ", smaller than max checkpointId in vnode:%" PRId64, maxChkptId,
maxCheckpointId);
maxChkptId = maxCheckpointId;
}
}
mDebug("generated checkpoint %" PRId64 "", maxChkptId + 1);
return maxChkptId + 1;
} }
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) { static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
@ -844,7 +898,7 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
} }
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
pMsg->checkpointId = mndStreamGenChkpId(pMnode); pMsg->checkpointId = mndStreamGenChkptId(pMnode, true);
int32_t size = sizeof(SMStreamDoCheckpointMsg); int32_t size = sizeof(SMStreamDoCheckpointMsg);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size}; SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@ -928,7 +982,6 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
int32_t code = -1; int32_t code = -1;
int64_t ts = taosGetTimestampMs(); int64_t ts = taosGetTimestampMs();
if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) { if (mndTrigger == 1 && (ts - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000)) {
// mWarn("checkpoint interval less than the threshold, ignore it");
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -940,7 +993,8 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME, "gen checkpoint for stream"); STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHECKPOINT_NAME,
"gen checkpoint for stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId, mError("failed to checkpoint of stream name%s, checkpointId: %" PRId64 ", reason:%s", pStream->name, checkpointId,
tstrerror(TSDB_CODE_MND_TRANS_CONFLICT)); tstrerror(TSDB_CODE_MND_TRANS_CONFLICT));
@ -1179,6 +1233,24 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
} }
} }
if (pStream->smaId != 0) {
void *pIter = NULL;
SSmaObj *pSma = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma);
while (pIter) {
if (pSma && pSma->uid == pStream->smaId) {
sdbRelease(pMnode->pSdb, pSma);
sdbRelease(pMnode->pSdb, pStream);
sdbCancelFetch(pMnode->pSdb, pIter);
tFreeMDropStreamReq(&dropReq);
terrno = TSDB_CODE_TSMA_MUST_BE_DROPPED;
return -1;
}
if (pSma) sdbRelease(pMnode->pSdb, pSma);
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma);
}
}
if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq); tFreeMDropStreamReq(&dropReq);
@ -1327,6 +1399,15 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams)
return 0; return 0;
} }
static void int64ToHexStr(int64_t id, char *pBuf, int32_t bufLen) {
memset(pBuf, 0, bufLen);
pBuf[2] = '0';
pBuf[3] = 'x';
int32_t len = tintToHex(id, &pBuf[4]);
varDataSetLen(pBuf, len + 2);
}
static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -1345,9 +1426,26 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false); colDataSetVal(pColInfo, numOfRows, (const char *)streamName, false);
// create time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->createTime, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pStream->createTime, false);
// stream id
char buf[128] = {0};
int64ToHexStr(pStream->uid, buf, tListLen(buf));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, buf, false);
// related fill-history stream id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (pStream->hTaskUid != 0) {
int64ToHexStr(pStream->hTaskUid, buf, tListLen(buf));
colDataSetVal(pColInfo, numOfRows, buf, false);
} else {
colDataSetVal(pColInfo, numOfRows, buf, true);
}
// related fill-history stream id
char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql)); STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
@ -1440,11 +1538,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
// task id // task id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
char idstr[128] = {0}; char idstr[128] = {0};
int32_t len = tintToHex(pTask->id.taskId, &idstr[4]); int64ToHexStr(pTask->id.taskId, idstr, tListLen(idstr));
idstr[2] = '0';
idstr[3] = 'x';
varDataSetLen(idstr, len + 2);
colDataSetVal(pColInfo, numOfRows, idstr, false); colDataSetVal(pColInfo, numOfRows, idstr, false);
// node type // node type
@ -1510,13 +1605,14 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
// colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false); // colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false);
// info
if (pTask->info.taskLevel == TASK_LEVEL__SINK) { if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
const char *sinkStr = "%.2fMiB"; const char *sinkStr = "%.2fMiB";
sprintf(buf, sinkStr, pe->sinkDataSize); sprintf(buf, sinkStr, pe->sinkDataSize);
} else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
// offset info // offset info
const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]"; const char *offsetStr = "%" PRId64 " [%" PRId64 ", %" PRId64 "]";
sprintf(buf, offsetStr, pe->processedVer, pe->verStart, pe->verEnd); sprintf(buf, offsetStr, pe->processedVer, pe->verRange.minVer, pe->verRange.maxVer);
} }
STR_TO_VARSTR(vbuf, buf); STR_TO_VARSTR(vbuf, buf);
@ -1524,6 +1620,51 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false);
// start_time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startTime, false);
// start id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointId, false);
// start ver
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointVer, false);
// checkpoint time
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (pe->checkpointInfo.latestTime != 0) {
colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestTime, false);
} else {
colDataSetVal(pColInfo, numOfRows, 0, true);
}
// checkpoint_id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestId, false);
// checkpoint info
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestVer, false);
// ds_err_info
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, 0, true);
// history_task_id
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
if (pe->hTaskId != 0) {
int64ToHexStr(pe->hTaskId, idstr, tListLen(idstr));
colDataSetVal(pColInfo, numOfRows, idstr, false);
} else {
colDataSetVal(pColInfo, numOfRows, 0, true);
}
// history_task_status
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, 0, true);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1598,7 +1739,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
} }
} }
mInfo("stream:%s,%"PRId64 " start to pause stream", pauseReq.name, pStream->uid); mInfo("stream:%s,%" PRId64 " start to pause stream", pauseReq.name, pStream->uid);
if (pStream->status == STREAM_STATUS__PAUSE) { if (pStream->status == STREAM_STATUS__PAUSE) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1623,7 +1764,8 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream"); STrans *pTrans =
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_PAUSE_NAME, "pause the stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr()); mError("stream:%s failed to pause stream since %s", pauseReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1712,7 +1854,8 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream"); STrans *pTrans =
doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_RESUME_NAME, "resume the stream");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr()); mError("stream:%s, failed to resume stream since %s", resumeReq.name, terrstr());
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
@ -1845,7 +1988,8 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
// here create only one trans // here create only one trans
if (pTrans == NULL) { if (pTrans == NULL) {
pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets"); pTrans =
doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_UPDATE_NAME, "update task epsets");
if (pTrans == NULL) { if (pTrans == NULL) {
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
@ -1885,7 +2029,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
} }
} }
// no need to build the trans to handle the vgroup upddate // no need to build the trans to handle the vgroup update
if (pTrans == NULL) { if (pTrans == NULL) {
return 0; return 0;
} }
@ -2197,12 +2341,13 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) { if (pStream == NULL) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf", req.streamId); mWarn("failed to find the stream:0x%" PRIx64 ", not handle the checkpoint req, try to acquire in buf",
req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer // not in meta-store yet, try to acquire the task in exec buffer
// the checkpoint req arrives too soon before the completion of the create stream trans. // the checkpoint req arrives too soon before the completion of the create stream trans.
STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
void* p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
if (p == NULL) { if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId); mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint req", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
@ -2214,7 +2359,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
} }
} }
int32_t numOfTasks = (pStream == NULL)? 0: mndGetNumOfStreamTasks(pStream); int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId)); SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pTransferStateStreams, &req.streamId, sizeof(req.streamId));
if (pReqTaskList == NULL) { if (pReqTaskList == NULL) {
@ -2229,10 +2374,10 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
int32_t total = taosArrayGetSize(*pReqTaskList); int32_t total = taosArrayGetSize(*pReqTaskList);
if (total == numOfTasks) { // all tasks has send the reqs if (total == numOfTasks) { // all tasks has send the reqs
int64_t checkpointId = mndStreamGenChkpId(pMnode); int64_t checkpointId = mndStreamGenChkptId(pMnode, false);
mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId); mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId);
if (pStream != NULL) { // TODO:handle error if (pStream != NULL) { // TODO:handle error
int32_t code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId, 0, false); int32_t code = mndProcessStreamCheckpointTrans(pMnode, pStream, checkpointId, 0, false);
} else { } else {
// todo: wait for the create stream trans completed, and launch the checkpoint trans // todo: wait for the create stream trans completed, and launch the checkpoint trans
@ -2256,13 +2401,35 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
{ {
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)}; SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)};
rsp.pCont = rpcMallocCont(rsp.contLen); rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead* pHead = rsp.pCont; SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.nodeId); pHead->vgId = htonl(req.nodeId);
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp pReq->info.handle = NULL; // disable auto rsp
} }
return 0; return 0;
} }
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessCreateStreamReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}
static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessDropStreamReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}

View File

@ -258,7 +258,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
if (numOfUpdated > 0) { if (numOfUpdated > 0) {
mDebug("%d stream node(s) need updated from report of hbMsg(vgId:%d)", numOfUpdated, req.vgId); mDebug("%d stream node(s) need updated from hbMsg(vgId:%d)", numOfUpdated, req.vgId);
setNodeEpsetExpiredFlag(req.pUpdateNodes); setNodeEpsetExpiredFlag(req.pUpdateNodes);
} }
@ -294,12 +294,14 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
} }
streamTaskStatusCopy(pTaskEntry, p); streamTaskStatusCopy(pTaskEntry, p);
if ((p->checkpointId != 0) && p->checkpointFailed) {
STaskCkptInfo *pChkInfo = &p->checkpointInfo;
if ((pChkInfo->activeId != 0) && pChkInfo->failed) {
mError("stream task:0x%" PRIx64 " checkpointId:%" PRIx64 " transId:%d failed, kill it", p->id.taskId, mError("stream task:0x%" PRIx64 " checkpointId:%" PRIx64 " transId:%d failed, kill it", p->id.taskId,
p->checkpointId, p->chkpointTransId); pChkInfo->activeId, pChkInfo->activeTransId);
SFailedCheckpointInfo info = { SFailedCheckpointInfo info = {
.transId = p->chkpointTransId, .checkpointId = p->checkpointId, .streamUid = p->id.streamId}; .transId = pChkInfo->activeTransId, .checkpointId = pChkInfo->activeId, .streamUid = p->id.streamId};
addIntoCheckpointList(pFailedTasks, &info); addIntoCheckpointList(pFailedTasks, &info);
} }
} }

View File

@ -32,10 +32,10 @@
#define VGROUP_VER_NUMBER 1 #define VGROUP_VER_NUMBER 1
#define VGROUP_RESERVE_SIZE 64 #define VGROUP_RESERVE_SIZE 64
static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup); static int32_t mndVgroupActionDelete(SSdb *pSdb, SVgObj *pVgroup);
static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew); static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew);
static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw); static int32_t mndNewVgActionValidate(SMnode *pMnode, STrans *pTrans, SSdbRaw *pRaw);
static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter);
@ -276,6 +276,9 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg
createReq.daysToKeep1 = pDb->cfg.daysToKeep1; createReq.daysToKeep1 = pDb->cfg.daysToKeep1;
createReq.daysToKeep2 = pDb->cfg.daysToKeep2; createReq.daysToKeep2 = pDb->cfg.daysToKeep2;
createReq.keepTimeOffset = pDb->cfg.keepTimeOffset; createReq.keepTimeOffset = pDb->cfg.keepTimeOffset;
createReq.s3ChunkSize = pDb->cfg.s3ChunkSize;
createReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
createReq.s3Compact = pDb->cfg.s3Compact;
createReq.minRows = pDb->cfg.minRows; createReq.minRows = pDb->cfg.minRows;
createReq.maxRows = pDb->cfg.maxRows; createReq.maxRows = pDb->cfg.maxRows;
createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod; createReq.walFsyncPeriod = pDb->cfg.walFsyncPeriod;
@ -399,6 +402,8 @@ static void *mndBuildAlterVnodeConfigReq(SMnode *pMnode, SDbObj *pDb, SVgObj *pV
alterReq.minRows = pDb->cfg.minRows; alterReq.minRows = pDb->cfg.minRows;
alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod; alterReq.walRetentionPeriod = pDb->cfg.walRetentionPeriod;
alterReq.walRetentionSize = pDb->cfg.walRetentionSize; alterReq.walRetentionSize = pDb->cfg.walRetentionSize;
alterReq.s3KeepLocal = pDb->cfg.s3KeepLocal;
alterReq.s3Compact = pDb->cfg.s3Compact;
mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId); mInfo("vgId:%d, build alter vnode config req", pVgroup->vgId);
int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq); int32_t contLen = tSerializeSAlterVnodeConfigReq(NULL, 0, &alterReq);

View File

@ -62,8 +62,8 @@ SRpcMsg buildHbReq() {
entry.id.taskId = 5; entry.id.taskId = 5;
entry.id.streamId = defStreamId; entry.id.streamId = defStreamId;
entry.checkpointId = 1; entry.checkpointInfo.activeId = 1;
entry.checkpointFailed = true; entry.checkpointInfo.failed = true;
taosArrayPush(msg.pTaskStatus, &entry); taosArrayPush(msg.pTaskStatus, &entry);
} }

View File

@ -352,6 +352,11 @@ void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2
*/ */
int32_t sdbGetSize(SSdb *pSdb, ESdbType type); int32_t sdbGetSize(SSdb *pSdb, ESdbType type);
/**
* @brief get valid number of rows, removed rows are ignored
*/
int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type);
/** /**
* @brief Get the max id of the table, keyType of table should be INT32 * @brief Get the max id of the table, keyType of table should be INT32
* *

View File

@ -510,3 +510,15 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type) {
return pSdb->tableVer[type]; return pSdb->tableVer[type];
} }
bool countValid(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
int32_t* pInt = p1;
(*pInt) += 1;
return true;
}
int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type) {
int32_t num = 0;
sdbTraverse(pSdb, type, countValid, &num, 0, 0);
return num;
}

View File

@ -31,7 +31,6 @@ extern "C" {
#endif #endif
struct SSnode { struct SSnode {
char* path;
SStreamMeta* pMeta; SStreamMeta* pMeta;
SMsgCb msgCb; SMsgCb msgCb;
}; };

View File

@ -32,6 +32,7 @@ static STaskId replaceStreamTaskId(SStreamTask *pTask) {
pTask->id.taskId = pTask->streamTaskId.taskId; pTask->id.taskId = pTask->streamTaskId.taskId;
return id; return id;
} }
static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) { static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) {
ASSERT(pTask->info.fillHistory); ASSERT(pTask->info.fillHistory);
pTask->id.taskId = pId->taskId; pTask->id.taskId = pId->taskId;
@ -48,46 +49,23 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
streamTaskOpenAllUpstreamInput(pTask); streamTaskOpenAllUpstreamInput(pTask);
STaskId taskId = {0}; code = tqExpandStreamTask(pTask, pSnode->pMeta, NULL);
if (pTask->info.fillHistory) { if (code != TSDB_CODE_SUCCESS) {
taskId = replaceStreamTaskId(pTask); return code;
} }
pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1);
if (pTask->pState == NULL) {
sndError("s-task:%s failed to open state for task", pTask->id.idStr);
return -1;
} else {
sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
}
if (pTask->info.fillHistory) {
restoreStreamTaskId(pTask, &taskId);
}
int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList);
SReadHandle handle = {
.checkpointId = pTask->chkInfo.checkpointId,
.vnode = NULL,
.numOfVgroups = numOfVgroups,
.pStateBackend = pTask->pState,
.fillHistory = pTask->info.fillHistory,
.winRange = pTask->dataRange.window,
};
initStreamStateAPI(&handle.api);
pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, SNODE_HANDLE, pTask->id.taskId);
ASSERT(pTask->exec.pExecutor);
qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId);
streamTaskResetUpstreamStageInfo(pTask); streamTaskResetUpstreamStageInfo(pTask);
streamSetupScheduleTrigger(pTask); streamSetupScheduleTrigger(pTask);
SCheckpointInfo *pChkInfo = &pTask->chkInfo; SCheckpointInfo *pChkInfo = &pTask->chkInfo;
// checkpoint ver is the kept version, handled data should be the next version. // checkpoint ver is the kept version, handled data should be the next version.
if (pChkInfo->checkpointId != 0) { if (pChkInfo->checkpointId != 0) {
pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1;
pChkInfo->processedVer = pChkInfo->checkpointVer; pChkInfo->processedVer = pChkInfo->checkpointVer;
pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer;
pTask->execInfo.startCheckpointId = pChkInfo->checkpointId;
sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64,
pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
} }
@ -117,11 +95,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pSnode->path = taosStrdup(path);
if (pSnode->path == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL;
}
pSnode->msgCb = pOption->msgCb; pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback);
@ -140,7 +113,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
return pSnode; return pSnode;
FAIL: FAIL:
taosMemoryFree(pSnode->path);
taosMemoryFree(pSnode); taosMemoryFree(pSnode);
return NULL; return NULL;
} }
@ -156,7 +128,6 @@ void sndClose(SSnode *pSnode) {
streamMetaNotifyClose(pSnode->pMeta); streamMetaNotifyClose(pSnode->pMeta);
streamMetaCommit(pSnode->pMeta); streamMetaCommit(pSnode->pMeta);
streamMetaClose(pSnode->pMeta); streamMetaClose(pSnode->pMeta);
taosMemoryFree(pSnode->path);
taosMemoryFree(pSnode); taosMemoryFree(pSnode);
} }

View File

@ -133,7 +133,7 @@ else()
PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode"
) )
if (${TD_LINUX}) if (${TD_LINUX})
target_include_directories( target_include_directories(
vnode vnode
PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static" PUBLIC "${TD_SOURCE_DIR}/deps/${TD_DEPS_DIR}/rocksdb_static"
) )
@ -144,6 +144,11 @@ else()
endif() endif()
endif() endif()
target_link_directories(
vnode
PUBLIC "${CMAKE_BINARY_DIR}/build/lib"
)
target_link_libraries( target_link_libraries(
vnode vnode
PUBLIC os PUBLIC os
@ -160,6 +165,7 @@ target_link_libraries(
# PUBLIC bdb # PUBLIC bdb
# PUBLIC scalar # PUBLIC scalar
#PUBLIC zstd
PUBLIC rocksdb PUBLIC rocksdb
PUBLIC transport PUBLIC transport
PUBLIC stream PUBLIC stream

View File

@ -57,7 +57,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod
int32_t diskPrimary, STfs *pTfs); int32_t diskPrimary, STfs *pTfs);
int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId, int32_t vnodeRestoreVgroupId(const char *srcPath, const char *dstPath, int32_t srcVgId, int32_t dstVgId,
int32_t diskPrimary, STfs *pTfs); int32_t diskPrimary, STfs *pTfs);
void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs); void vnodeDestroy(int32_t vgId, const char *path, STfs *pTfs, int32_t nodeId);
SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force); SVnode *vnodeOpen(const char *path, int32_t diskPrimary, STfs *pTfs, SMsgCb msgCb, bool force);
void vnodePreClose(SVnode *pVnode); void vnodePreClose(SVnode *pVnode);
void vnodePostClose(SVnode *pVnode); void vnodePostClose(SVnode *pVnode);
@ -69,7 +69,7 @@ int32_t vnodeBegin(SVnode *pVnode);
int32_t vnodeStart(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode);
void vnodeStop(SVnode *pVnode); void vnodeStop(SVnode *pVnode);
int64_t vnodeGetSyncHandle(SVnode *pVnode); int64_t vnodeGetSyncHandle(SVnode *pVnode);
int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); int32_t vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables); void vnodeGetInfo(void *pVnode, const char **dbname, int32_t *vgId, int64_t *numOfTables, int64_t *numOfNormalTables);
int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen);
int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList); int32_t vnodeGetTableList(void *pVnode, int8_t type, SArray *pList);
@ -137,8 +137,8 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList); int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload, int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen); int32_t payloadLen);
bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type); bool metaTbInFilterCache(SMeta *pMeta, const void *key, int8_t type);
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type); int32_t metaPutTbToFilterCache(SMeta *pMeta, const void *key, int8_t type);
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type); int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type);
int32_t metaInitTbFilterCache(SMeta *pMeta); int32_t metaInitTbFilterCache(SMeta *pMeta);
@ -175,13 +175,13 @@ void *tsdbGetIvtIdx2(SMeta *pMeta);
uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader); uint64_t tsdbGetReaderMaxVersion2(STsdbReader *pReader);
void tsdbReaderSetCloseFlag(STsdbReader *pReader); void tsdbReaderSetCloseFlag(STsdbReader *pReader);
int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr); int64_t tsdbGetLastTimestamp2(SVnode *pVnode, void *pTableList, int32_t numOfTables, const char *pIdStr);
void tsdbSetFilesetDelimited(STsdbReader* pReader); void tsdbSetFilesetDelimited(STsdbReader *pReader);
void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param); void tsdbReaderSetNotifyCb(STsdbReader *pReader, TsdReaderNotifyCbFn notifyFn, void *param);
int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables); int32_t tsdbReuseCacherowsReader(void *pReader, void *pTableIdList, int32_t numOfTables);
int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr, SArray *pCidList, int32_t *pSlotIds, uint64_t suid, void **pReader, const char *idstr,
SArray* pFuncTypeList, SColumnInfo* pkCol, int32_t numOfPks); SArray *pFuncTypeList, SColumnInfo* pkCol, int32_t numOfPks);
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds, int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids); SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader); void *tsdbCacherowsReaderClose(void *pReader);
@ -241,7 +241,7 @@ int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, i
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished); int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished);
// sma // sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);
@ -319,12 +319,19 @@ struct SVnodeCfg {
int16_t hashPrefix; int16_t hashPrefix;
int16_t hashSuffix; int16_t hashSuffix;
int32_t tsdbPageSize; int32_t tsdbPageSize;
int32_t s3ChunkSize;
int32_t s3KeepLocal;
int8_t s3Compact;
}; };
#define TABLE_ROLLUP_ON ((int8_t)0x1) #define TABLE_ROLLUP_ON ((int8_t)0x1)
#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0)
#define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON)
#define TABLE_COL_COMPRESSED ((int8_t)0x2)
#define TABLE_IS_COL_COMPRESSED(FLG) (((FLG) & (TABLE_COL_COMPRESSED)) != 0)
#define TABLE_SET_COL_COMPRESSED(FLG) ((FLG) |= TABLE_COL_COMPRESSED)
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -160,11 +160,12 @@ int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_
STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey);
// TODO, refactor later // TODO, refactor later
int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *param, SArray *results); int32_t metaFilterTableIds(void* pVnode, SMetaFltParam* param, SArray* results);
int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *parm, SArray *pUids); int32_t metaFilterCreateTime(void* pVnode, SMetaFltParam* parm, SArray* pUids);
int32_t metaFilterTableName(void *pVnode, SMetaFltParam *param, SArray *pUids); int32_t metaFilterTableName(void* pVnode, SMetaFltParam* param, SArray* pUids);
int32_t metaFilterTtl(void *pVnode, SMetaFltParam *param, SArray *pUids); int32_t metaFilterTtl(void* pVnode, SMetaFltParam* param, SArray* pUids);
int32_t metaGetColCmpr(SMeta* pMeta, tb_uid_t uid, SHashObj** colCmprObj);
#ifndef META_REFACT #ifndef META_REFACT
// SMetaDB // SMetaDB
int metaOpenDB(SMeta* pMeta); int metaOpenDB(SMeta* pMeta);

View File

@ -94,7 +94,7 @@ typedef struct STsdbRowKey STsdbRowKey;
#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM)) #define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM))
#define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \ #define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \
((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE)) ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE))
#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE) * PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE)) #define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE))
#define PAGE_OFFSET(PGNO, PAGE) (((PGNO)-1) * (PAGE)) #define PAGE_OFFSET(PGNO, PAGE) (((PGNO)-1) * (PAGE))
#define OFFSET_PGNO(OFFSET, PAGE) ((OFFSET) / (PAGE) + 1) #define OFFSET_PGNO(OFFSET, PAGE) ((OFFSET) / (PAGE) + 1)
@ -126,7 +126,7 @@ int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2);
int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2); int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2);
void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key); void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key);
void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key); void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// STSDBRowIter // STSDBRowIter
int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);
@ -146,8 +146,8 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2);
#define MIN_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) < 0) ? (KEY1) : (KEY2)) #define MIN_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) < 0) ? (KEY1) : (KEY2))
#define MAX_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) > 0) ? (KEY1) : (KEY2)) #define MAX_TSDBKEY(KEY1, KEY2) ((tsdbKeyCmprFn(&(KEY1), &(KEY2)) > 0) ? (KEY1) : (KEY2))
// SBlockCol // SBlockCol
int32_t tPutBlockCol(SBuffer *buffer, const SBlockCol *pBlockCol); int32_t tPutBlockCol(SBuffer *buffer, const SBlockCol *pBlockCol, int32_t ver, uint32_t cmprAlg);
int32_t tGetBlockCol(SBufferReader *br, SBlockCol *pBlockCol); int32_t tGetBlockCol(SBufferReader *br, SBlockCol *pBlockCol, int32_t ver, uint32_t cmprAlg);
int32_t tBlockColCmprFn(const void *p1, const void *p2); int32_t tBlockColCmprFn(const void *p1, const void *p2);
// SDataBlk // SDataBlk
void tDataBlkReset(SDataBlk *pBlock); void tDataBlkReset(SDataBlk *pBlock);
@ -179,7 +179,7 @@ int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid); int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid);
int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
void tBlockDataClear(SBlockData *pBlockData); void tBlockDataClear(SBlockData *pBlockData);
int32_t tBlockDataCompress(SBlockData *bData, int8_t cmprAlg, SBuffer *buffers, SBuffer *assist); int32_t tBlockDataCompress(SBlockData *bData, void *pCmprInfo, SBuffer *buffers, SBuffer *assist);
int32_t tBlockDataDecompress(SBufferReader *br, SBlockData *blockData, SBuffer *assist); int32_t tBlockDataDecompress(SBufferReader *br, SBlockData *blockData, SBuffer *assist);
int32_t tBlockDataDecompressKeyPart(const SDiskDataHdr *hdr, SBufferReader *br, SBlockData *blockData, SBuffer *assist); int32_t tBlockDataDecompressKeyPart(const SDiskDataHdr *hdr, SBufferReader *br, SBlockData *blockData, SBuffer *assist);
int32_t tBlockDataDecompressColData(const SDiskDataHdr *hdr, const SBlockCol *blockCol, SBufferReader *br, int32_t tBlockDataDecompressColData(const SDiskDataHdr *hdr, const SBlockCol *blockCol, SBufferReader *br,
@ -225,10 +225,10 @@ void tsdbMemTableDestroy(SMemTable *pMemTable, bool proactive);
STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid); STbData *tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid);
int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode); int32_t tsdbRefMemTable(SMemTable *pMemTable, SQueryNode *pQNode);
int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive); int32_t tsdbUnrefMemTable(SMemTable *pMemTable, SQueryNode *pNode, bool proactive);
SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable); SArray * tsdbMemTableGetTbDataArray(SMemTable *pMemTable);
// STbDataIter // STbDataIter
int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter **ppIter); int32_t tsdbTbDataIterCreate(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter **ppIter);
void *tsdbTbDataIterDestroy(STbDataIter *pIter); void * tsdbTbDataIterDestroy(STbDataIter *pIter);
void tsdbTbDataIterOpen(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, STsdbRowKey *pFrom, int8_t backward, STbDataIter *pIter);
bool tsdbTbDataIterNext(STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter);
void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum); void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum);
@ -281,7 +281,7 @@ int32_t tsdbGetTableSchema(SMeta *pMeta, int64_t uid, STSchema **pSchema, int64_
// tsdbMerge.c ============================================================================================== // tsdbMerge.c ==============================================================================================
typedef struct { typedef struct {
STsdb *tsdb; STsdb * tsdb;
int32_t fid; int32_t fid;
} SMergeArg; } SMergeArg;
@ -312,22 +312,22 @@ int32_t tsdbDataIterNext2(STsdbDataIter2 *pIter, STsdbFilterInfo *pFilterInfo);
// structs ======================= // structs =======================
struct STsdbFS { struct STsdbFS {
SDelFile *pDelFile; SDelFile *pDelFile;
SArray *aDFileSet; // SArray<SDFileSet> SArray * aDFileSet; // SArray<SDFileSet>
}; };
typedef struct { typedef struct {
rocksdb_t *db; rocksdb_t * db;
rocksdb_comparator_t *my_comparator; rocksdb_comparator_t * my_comparator;
rocksdb_cache_t *blockcache; rocksdb_cache_t * blockcache;
rocksdb_block_based_table_options_t *tableoptions; rocksdb_block_based_table_options_t *tableoptions;
rocksdb_options_t *options; rocksdb_options_t * options;
rocksdb_flushoptions_t *flushoptions; rocksdb_flushoptions_t * flushoptions;
rocksdb_writeoptions_t *writeoptions; rocksdb_writeoptions_t * writeoptions;
rocksdb_readoptions_t *readoptions; rocksdb_readoptions_t * readoptions;
rocksdb_writebatch_t *writebatch; rocksdb_writebatch_t * writebatch;
rocksdb_writebatch_t *rwritebatch; rocksdb_writebatch_t * rwritebatch;
TdThreadMutex rMutex; TdThreadMutex rMutex;
STSchema *pTSchema; STSchema * pTSchema;
} SRocksCache; } SRocksCache;
typedef struct { typedef struct {
@ -336,22 +336,22 @@ typedef struct {
} SCacheFlushState; } SCacheFlushState;
struct STsdb { struct STsdb {
char *path; char * path;
SVnode *pVnode; SVnode * pVnode;
STsdbKeepCfg keepCfg; STsdbKeepCfg keepCfg;
TdThreadMutex mutex; TdThreadMutex mutex;
bool bgTaskDisabled; bool bgTaskDisabled;
SMemTable *mem; SMemTable * mem;
SMemTable *imem; SMemTable * imem;
STsdbFS fs; // old STsdbFS fs; // old
SLRUCache *lruCache; SLRUCache * lruCache;
SCacheFlushState flushState; SCacheFlushState flushState;
TdThreadMutex lruMutex; TdThreadMutex lruMutex;
SLRUCache *biCache; SLRUCache * biCache;
TdThreadMutex biMutex; TdThreadMutex biMutex;
SLRUCache *bCache; SLRUCache * bCache;
TdThreadMutex bMutex; TdThreadMutex bMutex;
SLRUCache *pgCache; SLRUCache * pgCache;
TdThreadMutex pgMutex; TdThreadMutex pgMutex;
struct STFileSystem *pFS; // new struct STFileSystem *pFS; // new
SRocksCache rCache; SRocksCache rCache;
@ -380,17 +380,17 @@ struct STbData {
TSKEY minKey; TSKEY minKey;
TSKEY maxKey; TSKEY maxKey;
SRWLatch lock; SRWLatch lock;
SDelData *pHead; SDelData * pHead;
SDelData *pTail; SDelData * pTail;
SMemSkipList sl; SMemSkipList sl;
STbData *next; STbData * next;
SRBTreeNode rbtn[1]; SRBTreeNode rbtn[1];
}; };
struct SMemTable { struct SMemTable {
SRWLatch latch; SRWLatch latch;
STsdb *pTsdb; STsdb * pTsdb;
SVBufPool *pPool; SVBufPool * pPool;
volatile int32_t nRef; volatile int32_t nRef;
int64_t minVer; int64_t minVer;
int64_t maxVer; int64_t maxVer;
@ -400,7 +400,7 @@ struct SMemTable {
int64_t nDel; int64_t nDel;
int32_t nTbData; int32_t nTbData;
int32_t nBucket; int32_t nBucket;
STbData **aBucket; STbData ** aBucket;
SRBTree tbDataTree[1]; SRBTree tbDataTree[1];
}; };
@ -409,7 +409,7 @@ struct TSDBROW {
union { union {
struct { struct {
int64_t version; int64_t version;
SRow *pTSRow; SRow * pTSRow;
}; };
struct { struct {
SBlockData *pBlockData; SBlockData *pBlockData;
@ -444,15 +444,16 @@ struct SMapData {
}; };
struct SBlockCol { struct SBlockCol {
int16_t cid; int16_t cid;
int8_t type; int8_t type;
int8_t cflag; int8_t cflag;
int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE
int32_t szOrigin; // original column value size (only save for variant data type) int32_t szOrigin; // original column value size (only save for variant data type)
int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL
int32_t szOffset; // offset size, 0 only for non-variant-length type int32_t szOffset; // offset size, 0 only for non-variant-length type
int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE) int32_t szValue; // value size, 0 when flag == (HAS_NULL | HAS_NONE)
int32_t offset; int32_t offset;
uint32_t alg;
}; };
struct SBlockInfo { struct SBlockInfo {
@ -509,9 +510,9 @@ struct SBlockData {
int64_t suid; // 0 means normal table block data, otherwise child table block data int64_t suid; // 0 means normal table block data, otherwise child table block data
int64_t uid; // 0 means block data in .last file, otherwise in .data file int64_t uid; // 0 means block data in .last file, otherwise in .data file
int32_t nRow; // number of rows int32_t nRow; // number of rows
int64_t *aUid; // uids of each row, only exist in block data in .last file (uid == 0) int64_t * aUid; // uids of each row, only exist in block data in .last file (uid == 0)
int64_t *aVersion; // versions of each row int64_t * aVersion; // versions of each row
TSKEY *aTSKEY; // timestamp of each row TSKEY * aTSKEY; // timestamp of each row
int32_t nColData; int32_t nColData;
SColData *aColData; SColData *aColData;
}; };
@ -522,10 +523,10 @@ struct TABLEID {
}; };
struct STbDataIter { struct STbDataIter {
STbData *pTbData; STbData * pTbData;
int8_t backward; int8_t backward;
SMemSkipListNode *pNode; SMemSkipListNode *pNode;
TSDBROW *pRow; TSDBROW * pRow;
TSDBROW row; TSDBROW row;
}; };
@ -553,7 +554,7 @@ struct SDiskDataHdr {
int32_t szKey; int32_t szKey;
int32_t szBlkCol; int32_t szBlkCol;
int32_t nRow; int32_t nRow;
int8_t cmprAlg; uint32_t cmprAlg;
// fmtVer == 1 // fmtVer == 1
int8_t numOfPKs; int8_t numOfPKs;
@ -603,9 +604,9 @@ struct SDFileSet {
int32_t fid; int32_t fid;
SHeadFile *pHeadF; SHeadFile *pHeadF;
SDataFile *pDataF; SDataFile *pDataF;
SSmaFile *pSmaF; SSmaFile * pSmaF;
uint8_t nSttF; uint8_t nSttF;
SSttFile *aSttF[TSDB_STT_TRIGGER_ARRAY_SIZE]; SSttFile * aSttF[TSDB_STT_TRIGGER_ARRAY_SIZE];
}; };
struct STSDBRowIter { struct STSDBRowIter {
@ -621,27 +622,28 @@ struct STSDBRowIter {
struct SRowMerger { struct SRowMerger {
STSchema *pTSchema; STSchema *pTSchema;
int64_t version; int64_t version;
SArray *pArray; // SArray<SColVal> SArray * pArray; // SArray<SColVal>
}; };
typedef struct { typedef struct {
char *path; char * path;
int32_t szPage; int32_t szPage;
int32_t flag; int32_t flag;
TdFilePtr pFD; TdFilePtr pFD;
int64_t pgno; int64_t pgno;
uint8_t *pBuf; uint8_t * pBuf;
int64_t szFile; int64_t szFile;
STsdb *pTsdb; STsdb * pTsdb;
const char *objName; const char *objName;
uint8_t s3File; uint8_t s3File;
int32_t lcn;
int32_t fid; int32_t fid;
int64_t cid; int64_t cid;
int64_t blkno; int64_t blkno;
} STsdbFD; } STsdbFD;
struct SDelFWriter { struct SDelFWriter {
STsdb *pTsdb; STsdb * pTsdb;
SDelFile fDel; SDelFile fDel;
STsdbFD *pWriteH; STsdbFD *pWriteH;
uint8_t *aBuf[1]; uint8_t *aBuf[1];
@ -701,15 +703,15 @@ int32_t tDeserializeTsdbRepOpts(void *buf, int32_t bufLen, STsdbRepOpts *pInfo);
// snap read // snap read
struct STsdbReadSnap { struct STsdbReadSnap {
SMemTable *pMem; SMemTable * pMem;
SQueryNode *pNode; SQueryNode * pNode;
SMemTable *pIMem; SMemTable * pIMem;
SQueryNode *pINode; SQueryNode * pINode;
TFileSetArray *pfSetArray; TFileSetArray *pfSetArray;
}; };
struct SDataFWriter { struct SDataFWriter {
STsdb *pTsdb; STsdb * pTsdb;
SDFileSet wSet; SDFileSet wSet;
STsdbFD *pHeadFD; STsdbFD *pHeadFD;
@ -726,13 +728,13 @@ struct SDataFWriter {
}; };
struct SDataFReader { struct SDataFReader {
STsdb *pTsdb; STsdb * pTsdb;
SDFileSet *pSet; SDFileSet *pSet;
STsdbFD *pHeadFD; STsdbFD * pHeadFD;
STsdbFD *pDataFD; STsdbFD * pDataFD;
STsdbFD *pSmaFD; STsdbFD * pSmaFD;
STsdbFD *aSttFD[TSDB_STT_TRIGGER_ARRAY_SIZE]; STsdbFD * aSttFD[TSDB_STT_TRIGGER_ARRAY_SIZE];
uint8_t *aBuf[3]; uint8_t * aBuf[3];
}; };
// NOTE: do NOT change the order of the fields // NOTE: do NOT change the order of the fields
@ -767,10 +769,10 @@ typedef struct {
typedef struct SSttBlockLoadInfo { typedef struct SSttBlockLoadInfo {
SBlockDataInfo blockData[2]; // buffered block data SBlockDataInfo blockData[2]; // buffered block data
SArray *aSttBlk; SArray * aSttBlk;
int32_t currentLoadBlockIndex; int32_t currentLoadBlockIndex;
STSchema *pSchema; STSchema * pSchema;
int16_t *colIds; int16_t * colIds;
int32_t numOfCols; int32_t numOfCols;
bool checkRemainingRow; // todo: no assign value? bool checkRemainingRow; // todo: no assign value?
bool isLast; bool isLast;
@ -807,7 +809,7 @@ struct SDiskData {
const uint8_t *pUid; const uint8_t *pUid;
const uint8_t *pVer; const uint8_t *pVer;
const uint8_t *pKey; const uint8_t *pKey;
SArray *aDiskCol; // SArray<SDiskCol> SArray * aDiskCol; // SArray<SDiskCol>
}; };
struct SDiskDataBuilder { struct SDiskDataBuilder {
@ -820,15 +822,15 @@ struct SDiskDataBuilder {
SCompressor *pVerC; SCompressor *pVerC;
SCompressor *pKeyC; SCompressor *pKeyC;
int32_t nBuilder; int32_t nBuilder;
SArray *aBuilder; // SArray<SDiskColBuilder> SArray * aBuilder; // SArray<SDiskColBuilder>
uint8_t *aBuf[2]; uint8_t * aBuf[2];
SDiskData dd; SDiskData dd;
SBlkInfo bi; SBlkInfo bi;
}; };
struct SLDataIter { struct SLDataIter {
SRBTreeNode node; SRBTreeNode node;
SSttBlk *pSttBlk; SSttBlk * pSttBlk;
int64_t cid; // for debug purpose int64_t cid; // for debug purpose
int8_t backward; int8_t backward;
int32_t iSttBlk; int32_t iSttBlk;
@ -837,8 +839,8 @@ struct SLDataIter {
uint64_t uid; uint64_t uid;
STimeWindow timeWindow; STimeWindow timeWindow;
SVersionRange verRange; SVersionRange verRange;
SSttBlockLoadInfo *pBlockLoadInfo; SSttBlockLoadInfo * pBlockLoadInfo;
SRowKey* pStartRowKey; // current row key SRowKey * pStartRowKey; // current row key
bool ignoreEarlierTs; bool ignoreEarlierTs;
struct SSttFileReader *pReader; struct SSttFileReader *pReader;
}; };
@ -851,21 +853,21 @@ typedef int32_t (*_load_tomb_fn)(STsdbReader *pReader, struct SSttFileReader *pS
typedef struct SMergeTreeConf { typedef struct SMergeTreeConf {
int8_t backward; int8_t backward;
STsdb *pTsdb; STsdb * pTsdb;
uint64_t suid; uint64_t suid;
uint64_t uid; uint64_t uid;
STimeWindow timewindow; STimeWindow timewindow;
SVersionRange verRange; SVersionRange verRange;
bool strictTimeRange; bool strictTimeRange;
SArray *pSttFileBlockIterArray; SArray * pSttFileBlockIterArray;
void *pCurrentFileset; void * pCurrentFileset;
STSchema *pSchema; STSchema * pSchema;
int16_t *pCols; int16_t * pCols;
int32_t numOfCols; int32_t numOfCols;
SRowKey *pCurRowKey; SRowKey * pCurRowKey;
_load_tomb_fn loadTombFn; _load_tomb_fn loadTombFn;
void *pReader; void * pReader;
void *idstr; void * idstr;
bool rspRows; // response the rows in stt-file, if possible bool rspRows; // response the rows in stt-file, if possible
} SMergeTreeConf; } SMergeTreeConf;
@ -883,8 +885,8 @@ bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree);
void tMergeTreeClose(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree);
SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols); SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols);
void *destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); void * destroySttBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo);
void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost); void * destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoadCost);
// tsdbCache ============================================================================================== // tsdbCache ==============================================================================================
typedef enum { typedef enum {
@ -973,7 +975,7 @@ struct STsdbDataIter2 {
// TSDB_DATA_FILE_DATA_ITER // TSDB_DATA_FILE_DATA_ITER
struct { struct {
SDataFReader *pReader; SDataFReader *pReader;
SArray *aBlockIdx; // SArray<SBlockIdx> SArray * aBlockIdx; // SArray<SBlockIdx>
SMapData mDataBlk; SMapData mDataBlk;
SBlockData bData; SBlockData bData;
int32_t iBlockIdx; int32_t iBlockIdx;
@ -985,7 +987,7 @@ struct STsdbDataIter2 {
struct { struct {
SDataFReader *pReader; SDataFReader *pReader;
int32_t iStt; int32_t iStt;
SArray *aSttBlk; SArray * aSttBlk;
SBlockData bData; SBlockData bData;
int32_t iSttBlk; int32_t iSttBlk;
int32_t iRow; int32_t iRow;
@ -993,8 +995,8 @@ struct STsdbDataIter2 {
// TSDB_TOMB_FILE_DATA_ITER // TSDB_TOMB_FILE_DATA_ITER
struct { struct {
SDelFReader *pReader; SDelFReader *pReader;
SArray *aDelIdx; SArray * aDelIdx;
SArray *aDelData; SArray * aDelData;
int32_t iDelIdx; int32_t iDelIdx;
int32_t iDelData; int32_t iDelData;
} tIter; } tIter;

View File

@ -120,6 +120,7 @@ void vnodeQueryClose(SVnode* pVnode);
int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct); int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct); int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg);
int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
// vnodeCommit.c // vnodeCommit.c
int32_t vnodeBegin(SVnode* pVnode); int32_t vnodeBegin(SVnode* pVnode);
@ -133,6 +134,8 @@ int32_t vnodeAsyncCommit(SVnode* pVnode);
bool vnodeShouldRollback(SVnode* pVnode); bool vnodeShouldRollback(SVnode* pVnode);
// vnodeSync.c // vnodeSync.c
int64_t vnodeClusterId(SVnode* pVnode);
int32_t vnodeNodeId(SVnode* pVnode);
int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion); int32_t vnodeSyncOpen(SVnode* pVnode, char* path, int32_t vnodeVersion);
int32_t vnodeSyncStart(SVnode* pVnode); int32_t vnodeSyncStart(SVnode* pVnode);
void vnodeSyncPreClose(SVnode* pVnode); void vnodeSyncPreClose(SVnode* pVnode);

View File

@ -108,17 +108,17 @@ typedef struct SQueryNode SQueryNode;
#define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count" #define VNODE_METRIC_SQL_COUNT "taosd_sql_req:count"
#define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type" #define VNODE_METRIC_TAG_NAME_SQL_TYPE "sql_type"
#define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id" #define VNODE_METRIC_TAG_NAME_CLUSTER_ID "cluster_id"
#define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id" #define VNODE_METRIC_TAG_NAME_DNODE_ID "dnode_id"
#define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep" #define VNODE_METRIC_TAG_NAME_DNODE_EP "dnode_ep"
#define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id" #define VNODE_METRIC_TAG_NAME_VGROUP_ID "vgroup_id"
#define VNODE_METRIC_TAG_NAME_USERNAME "username" #define VNODE_METRIC_TAG_NAME_USERNAME "username"
#define VNODE_METRIC_TAG_NAME_RESULT "result" #define VNODE_METRIC_TAG_NAME_RESULT "result"
#define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows" #define VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS "inserted_rows"
//#define VNODE_METRIC_TAG_VALUE_INSERT "insert" // #define VNODE_METRIC_TAG_VALUE_INSERT "insert"
//#define VNODE_METRIC_TAG_VALUE_DELETE "delete" // #define VNODE_METRIC_TAG_VALUE_DELETE "delete"
// vnd.h // vnd.h
typedef int32_t (*_query_reseek_func_t)(void* pQHandle); typedef int32_t (*_query_reseek_func_t)(void* pQHandle);
@ -239,6 +239,7 @@ int32_t tsdbCacheNewNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, int8_t
int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey); int32_t tsdbCacheDropNTableColumn(STsdb* pTsdb, int64_t uid, int16_t cid, bool hasPrimayKey);
int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo);
int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync);
int32_t tsdbS3Migrate(STsdb* tsdb, int64_t now, int32_t sync);
int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg);
int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp);
int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows);
@ -293,6 +294,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg);
int32_t tqStreamProgressRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
// sma // sma
int32_t smaInit(); int32_t smaInit();
@ -461,12 +463,12 @@ typedef struct SVCommitSched {
int64_t maxWaitMs; int64_t maxWaitMs;
} SVCommitSched; } SVCommitSched;
typedef struct SVMonitorObj{ typedef struct SVMonitorObj {
char strClusterId[TSDB_CLUSTER_ID_LEN]; char strClusterId[TSDB_CLUSTER_ID_LEN];
char strDnodeId[TSDB_NODE_ID_LEN]; char strDnodeId[TSDB_NODE_ID_LEN];
char strVgId[TSDB_VGROUP_ID_LEN]; char strVgId[TSDB_VGROUP_ID_LEN];
taos_counter_t *insertCounter; taos_counter_t* insertCounter;
}SVMonitorObj; } SVMonitorObj;
struct SVnode { struct SVnode {
char* path; char* path;

View File

@ -15,6 +15,50 @@
#include "meta.h" #include "meta.h"
int meteEncodeColCmprEntry(SEncoder *pCoder, const SMetaEntry *pME) {
const SColCmprWrapper *pw = &pME->colCmpr;
if (tEncodeI32v(pCoder, pw->nCols) < 0) return -1;
if (tEncodeI32v(pCoder, pw->version) < 0) return -1;
uDebug("encode cols:%d", pw->nCols);
for (int32_t i = 0; i < pw->nCols; i++) {
SColCmpr *p = &pw->pColCmpr[i];
if (tEncodeI16v(pCoder, p->id) < 0) return -1;
if (tEncodeU32(pCoder, p->alg) < 0) return -1;
}
return 0;
}
int meteDecodeColCmprEntry(SDecoder *pDecoder, SMetaEntry *pME) {
SColCmprWrapper *pWrapper = &pME->colCmpr;
if (tDecodeI32v(pDecoder, &pWrapper->nCols) < 0) return -1;
if (tDecodeI32v(pDecoder, &pWrapper->version) < 0) return -1;
uDebug("dencode cols:%d", pWrapper->nCols);
pWrapper->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pWrapper->nCols * sizeof(SColCmpr));
if (pWrapper->pColCmpr == NULL) return -1;
for (int i = 0; i < pWrapper->nCols; i++) {
SColCmpr *p = &pWrapper->pColCmpr[i];
if (tDecodeI16v(pDecoder, &p->id) < 0) goto END;
if (tDecodeU32(pDecoder, &p->alg) < 0) goto END;
}
return 0;
END:
// taosMemoryFree(pWrapper->pColCmpr);
return -1;
}
static FORCE_INLINE void metatInitDefaultSColCmprWrapper(SDecoder *pDecoder, SColCmprWrapper *pCmpr,
SSchemaWrapper *pSchema) {
pCmpr->nCols = pSchema->nCols;
pCmpr->pColCmpr = (SColCmpr *)tDecoderMalloc(pDecoder, pCmpr->nCols * sizeof(SColCmpr));
for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr *pColCmpr = &pCmpr->pColCmpr[i];
SSchema *pColSchema = &pSchema->pSchema[i];
pColCmpr->id = pColSchema->colId;
pColCmpr->alg = createDefaultColCmprByType(pColSchema->type);
}
}
int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
if (tStartEncode(pCoder) < 0) return -1; if (tStartEncode(pCoder) < 0) return -1;
@ -55,6 +99,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) {
return -1; return -1;
} }
if (meteEncodeColCmprEntry(pCoder, pME) < 0) return -1;
tEndEncode(pCoder); tEndEncode(pCoder);
return 0; return 0;
@ -102,9 +147,20 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) {
if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1; if (tDecodeTSma(pCoder, pME->smaEntry.tsma, true) < 0) return -1;
} else { } else {
metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type); metaError("meta/entry: invalide table type: %" PRId8 " decode failed.", pME->type);
return -1; return -1;
} }
if (!tDecodeIsEnd(pCoder)) {
uDebug("set type: %d, tableName:%s", pME->type, pME->name);
if (meteDecodeColCmprEntry(pCoder, pME) < 0) return -1;
TABLE_SET_COL_COMPRESSED(pME->flags);
} else {
uDebug("set default type: %d, tableName:%s", pME->type, pME->name);
if (pME->type == TSDB_SUPER_TABLE) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->stbEntry.schemaRow);
} else if (pME->type == TSDB_NORMAL_TABLE) {
metatInitDefaultSColCmprWrapper(pCoder, &pME->colCmpr, &pME->ntbEntry.schemaRow);
}
}
tEndDecode(pCoder); tEndDecode(pCoder);
return 0; return 0;

View File

@ -63,6 +63,7 @@ int metaGetTableEntryByVersion(SMetaReader *pReader, int64_t version, tb_uid_t u
if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) {
goto _err; goto _err;
} }
// taosMemoryFreeClear(pReader->me.colCmpr.pColCmpr);
return 0; return 0;
@ -363,6 +364,7 @@ _query:
version = ((SUidIdxVal *)pData)[0].version; version = ((SUidIdxVal *)pData)[0].version;
tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData); tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData);
SMetaEntry me = {0}; SMetaEntry me = {0};
tDecoderInit(&dc, pData, nData); tDecoderInit(&dc, pData, nData);
metaDecodeEntry(&dc, &me); metaDecodeEntry(&dc, &me);
@ -410,9 +412,8 @@ _err:
return NULL; return NULL;
} }
SMCtbCursor *metaOpenCtbCursor(void *pVnode, tb_uid_t uid, int lock) {
SMCtbCursor *metaOpenCtbCursor(void* pVnode, tb_uid_t uid, int lock) { SMeta *pMeta = ((SVnode *)pVnode)->pMeta;
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
SMCtbCursor *pCtbCur = NULL; SMCtbCursor *pCtbCur = NULL;
SCtbIdxKey ctbIdxKey; SCtbIdxKey ctbIdxKey;
int ret = 0; int ret = 0;
@ -449,9 +450,9 @@ void metaCloseCtbCursor(SMCtbCursor *pCtbCur) {
taosMemoryFree(pCtbCur); taosMemoryFree(pCtbCur);
} }
void metaPauseCtbCursor(SMCtbCursor* pCtbCur) { void metaPauseCtbCursor(SMCtbCursor *pCtbCur) {
if (!pCtbCur->paused) { if (!pCtbCur->paused) {
tdbTbcClose((TBC*)pCtbCur->pCur); tdbTbcClose((TBC *)pCtbCur->pCur);
if (pCtbCur->lock) { if (pCtbCur->lock) {
metaULock(pCtbCur->pMeta); metaULock(pCtbCur->pMeta);
} }
@ -459,7 +460,7 @@ void metaPauseCtbCursor(SMCtbCursor* pCtbCur) {
} }
} }
int32_t metaResumeCtbCursor(SMCtbCursor* pCtbCur, int8_t first) { int32_t metaResumeCtbCursor(SMCtbCursor *pCtbCur, int8_t first) {
if (pCtbCur->paused) { if (pCtbCur->paused) {
pCtbCur->paused = 0; pCtbCur->paused = 0;
@ -467,14 +468,14 @@ int32_t metaResumeCtbCursor(SMCtbCursor* pCtbCur, int8_t first) {
metaRLock(pCtbCur->pMeta); metaRLock(pCtbCur->pMeta);
} }
int ret = 0; int ret = 0;
ret = tdbTbcOpen(pCtbCur->pMeta->pCtbIdx, (TBC**)&pCtbCur->pCur, NULL); ret = tdbTbcOpen(pCtbCur->pMeta->pCtbIdx, (TBC **)&pCtbCur->pCur, NULL);
if (ret < 0) { if (ret < 0) {
metaCloseCtbCursor(pCtbCur); metaCloseCtbCursor(pCtbCur);
return -1; return -1;
} }
if (first) { if (first) {
SCtbIdxKey ctbIdxKey; SCtbIdxKey ctbIdxKey;
// move to the suid // move to the suid
ctbIdxKey.suid = pCtbCur->suid; ctbIdxKey.suid = pCtbCur->suid;
ctbIdxKey.uid = INT64_MIN; ctbIdxKey.uid = INT64_MIN;
@ -1478,23 +1479,23 @@ int32_t metaGetInfo(SMeta *pMeta, int64_t uid, SMetaInfo *pInfo, SMetaReader *pR
lock = 1; lock = 1;
} }
if(!lock) metaRLock(pMeta); if (!lock) metaRLock(pMeta);
// search cache // search cache
if (metaCacheGet(pMeta, uid, pInfo) == 0) { if (metaCacheGet(pMeta, uid, pInfo) == 0) {
if(!lock) metaULock(pMeta); if (!lock) metaULock(pMeta);
goto _exit; goto _exit;
} }
// search TDB // search TDB
if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) { if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) {
// not found // not found
if(!lock) metaULock(pMeta); if (!lock) metaULock(pMeta);
code = TSDB_CODE_NOT_FOUND; code = TSDB_CODE_NOT_FOUND;
goto _exit; goto _exit;
} }
if(!lock) metaULock(pMeta); if (!lock) metaULock(pMeta);
pInfo->uid = uid; pInfo->uid = uid;
pInfo->suid = ((SUidIdxVal *)pData)->suid; pInfo->suid = ((SUidIdxVal *)pData)->suid;

View File

@ -528,6 +528,7 @@ int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLe
req.schemaTag = me.stbEntry.schemaTag; req.schemaTag = me.stbEntry.schemaTag;
req.schemaRow.version = 1; req.schemaRow.version = 1;
req.schemaTag.version = 1; req.schemaTag.version = 1;
req.colCmpr = me.colCmpr;
ret = buildSuperTableInfo(&req, pBuf, contLen); ret = buildSuperTableInfo(&req, pBuf, contLen);
*type = TDMT_VND_CREATE_STB; *type = TDMT_VND_CREATE_STB;
@ -603,6 +604,7 @@ int32_t getTableInfoFromSnapshot(SSnapContext* ctx, void** pBuf, int32_t* contLe
req.uid = me.uid; req.uid = me.uid;
req.commentLen = -1; req.commentLen = -1;
req.ntb.schemaRow = me.ntbEntry.schemaRow; req.ntb.schemaRow = me.ntbEntry.schemaRow;
req.colCmpr = me.colCmpr;
ret = buildNormalChildTableInfo(&req, pBuf, contLen); ret = buildNormalChildTableInfo(&req, pBuf, contLen);
*type = TDMT_VND_CREATE_TABLE; *type = TDMT_VND_CREATE_TABLE;
} else { } else {

View File

@ -36,6 +36,36 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME);
int8_t updataTableColCmpr(SColCmprWrapper *pWp, SSchema *pSchema, int8_t add) {
int32_t nCols = pWp->nCols;
int32_t ver = pWp->version;
if (add) {
SColCmpr *p = taosMemoryCalloc(1, sizeof(SColCmpr) * (nCols + 1));
memcpy(p, pWp->pColCmpr, sizeof(SColCmpr) * nCols);
SColCmpr *pCol = p + nCols;
pCol->id = pSchema->colId;
pCol->alg = createDefaultColCmprByType(pSchema->type);
pWp->nCols = nCols + 1;
pWp->version = ver;
pWp->pColCmpr = p;
} else {
for (int32_t i = 0; i < nCols; i++) {
SColCmpr *pOCmpr = &pWp->pColCmpr[i];
if (pOCmpr->id == pSchema->colId) {
int32_t left = (nCols - i - 1) * sizeof(SColCmpr);
if (left) {
memmove(pWp->pColCmpr + i, pWp->pColCmpr + i + 1, left);
}
nCols--;
break;
}
}
pWp->nCols = nCols;
pWp->version = ver;
}
return 1;
}
static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) {
pInfo->uid = pEntry->uid; pInfo->uid = pEntry->uid;
pInfo->version = pEntry->version; pInfo->version = pEntry->version;
@ -55,11 +85,14 @@ static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) {
static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { static int metaUpdateMetaRsp(tb_uid_t uid, char *tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) {
pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema));
if (NULL == pMetaRsp->pSchemas) { if (NULL == pMetaRsp->pSchemas) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
pMetaRsp->pSchemaExt = taosMemoryMalloc(pSchema->nCols * sizeof(SSchemaExt));
tstrncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN); tstrncpy(pMetaRsp->tbName, tbName, TSDB_TABLE_NAME_LEN);
pMetaRsp->numOfColumns = pSchema->nCols; pMetaRsp->numOfColumns = pSchema->nCols;
pMetaRsp->tableType = TSDB_NORMAL_TABLE; pMetaRsp->tableType = TSDB_NORMAL_TABLE;
@ -76,7 +109,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
return -1; return -1;
} }
void *data = pCtbEntry->ctbEntry.pTags; void * data = pCtbEntry->ctbEntry.pTags;
const char *tagName = pSchema->name; const char *tagName = pSchema->name;
tb_uid_t suid = pCtbEntry->ctbEntry.suid; tb_uid_t suid = pCtbEntry->ctbEntry.suid;
@ -95,7 +128,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type; char type = pTagVal->type;
char *key = pTagVal->pKey; char * key = pTagVal->pKey;
int32_t nKey = strlen(key); int32_t nKey = strlen(key);
SIndexTerm *term = NULL; SIndexTerm *term = NULL;
@ -103,7 +136,7 @@ static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const
term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) { if (pTagVal->nData > 0) {
char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
type = TSDB_DATA_TYPE_VARCHAR; type = TSDB_DATA_TYPE_VARCHAR;
@ -137,7 +170,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) { if (pMeta->pTagIvtIdx == NULL || pCtbEntry == NULL) {
return -1; return -1;
} }
void *data = pCtbEntry->ctbEntry.pTags; void * data = pCtbEntry->ctbEntry.pTags;
const char *tagName = pSchema->name; const char *tagName = pSchema->name;
tb_uid_t suid = pCtbEntry->ctbEntry.suid; tb_uid_t suid = pCtbEntry->ctbEntry.suid;
@ -156,7 +189,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i);
char type = pTagVal->type; char type = pTagVal->type;
char *key = pTagVal->pKey; char * key = pTagVal->pKey;
int32_t nKey = strlen(key); int32_t nKey = strlen(key);
SIndexTerm *term = NULL; SIndexTerm *term = NULL;
@ -164,7 +197,7 @@ int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSche
term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0); term = indexTermCreate(suid, DEL_VALUE, TSDB_DATA_TYPE_VARCHAR, key, nKey, NULL, 0);
} else if (type == TSDB_DATA_TYPE_NCHAR) { } else if (type == TSDB_DATA_TYPE_NCHAR) {
if (pTagVal->nData > 0) { if (pTagVal->nData > 0) {
char *val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE); char * val = taosMemoryCalloc(1, pTagVal->nData + VARSTR_HEADER_SIZE);
int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE); int32_t len = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, val + VARSTR_HEADER_SIZE);
memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE); memcpy(val, (uint16_t *)&len, VARSTR_HEADER_SIZE);
type = TSDB_DATA_TYPE_VARCHAR; type = TSDB_DATA_TYPE_VARCHAR;
@ -211,9 +244,9 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
int vLen = 0; int vLen = 0;
const void *pKey = NULL; const void *pKey = NULL;
const void *pVal = NULL; const void *pVal = NULL;
void *pBuf = NULL; void * pBuf = NULL;
int32_t szBuf = 0; int32_t szBuf = 0;
void *p = NULL; void * p = NULL;
// validate req // validate req
void *pData = NULL; void *pData = NULL;
@ -241,10 +274,20 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
me.name = pReq->name; me.name = pReq->name;
me.stbEntry.schemaRow = pReq->schemaRow; me.stbEntry.schemaRow = pReq->schemaRow;
me.stbEntry.schemaTag = pReq->schemaTag; me.stbEntry.schemaTag = pReq->schemaTag;
// me.stbEntry.colCmpr = pReq->colCmpr;
// me.stbEntry.colCmpr = pReq->
if (pReq->rollup) { if (pReq->rollup) {
TABLE_SET_ROLLUP(me.flags); TABLE_SET_ROLLUP(me.flags);
me.stbEntry.rsmaParam = pReq->rsmaParam; me.stbEntry.rsmaParam = pReq->rsmaParam;
} }
if (pReq->colCmpred) {
TABLE_SET_COL_COMPRESSED(me.flags);
me.colCmpr = pReq->colCmpr;
} else {
TABLE_SET_COL_COMPRESSED(me.flags);
// TODO(yihao)
// SETUP default compress algr
}
if (metaHandleEntry(pMeta, &me) < 0) goto _err; if (metaHandleEntry(pMeta, &me) < 0) goto _err;
@ -342,7 +385,7 @@ static void metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) {
int c = 0; int c = 0;
void *pKey = NULL; void *pKey = NULL;
int nKey = 0; int nKey = 0;
TBC *pCtbIdxc = NULL; TBC * pCtbIdxc = NULL;
tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL);
int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c);
@ -373,8 +416,8 @@ static void metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) {
int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
SMetaEntry oStbEntry = {0}; SMetaEntry oStbEntry = {0};
SMetaEntry nStbEntry = {0}; SMetaEntry nStbEntry = {0};
TBC *pUidIdxc = NULL; TBC * pUidIdxc = NULL;
TBC *pTbDbc = NULL; TBC * pTbDbc = NULL;
const void *pData; const void *pData;
int nData; int nData;
int64_t oversion; int64_t oversion;
@ -432,6 +475,8 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
nStbEntry.name = pReq->name; nStbEntry.name = pReq->name;
nStbEntry.stbEntry.schemaRow = pReq->schemaRow; nStbEntry.stbEntry.schemaRow = pReq->schemaRow;
nStbEntry.stbEntry.schemaTag = pReq->schemaTag; nStbEntry.stbEntry.schemaTag = pReq->schemaTag;
nStbEntry.colCmpr = pReq->colCmpr;
TABLE_SET_COL_COMPRESSED(nStbEntry.flags);
int nCols = pReq->schemaRow.nCols; int nCols = pReq->schemaRow.nCols;
int onCols = oStbEntry.stbEntry.schemaRow.nCols; int onCols = oStbEntry.stbEntry.schemaRow.nCols;
@ -439,7 +484,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1); bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1);
if (!TSDB_CACHE_NO(pMeta->pVnode->config)) { if (!TSDB_CACHE_NO(pMeta->pVnode->config)) {
STsdb *pTsdb = pMeta->pVnode->pTsdb; STsdb * pTsdb = pMeta->pVnode->pTsdb;
SArray *uids = taosArrayInit(8, sizeof(int64_t)); SArray *uids = taosArrayInit(8, sizeof(int64_t));
if (deltaCol == 1) { if (deltaCol == 1) {
int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId; int16_t cid = pReq->schemaRow.pSchema[nCols - 1].colId;
@ -509,9 +554,9 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
STbDbKey tbDbKey = {0}; STbDbKey tbDbKey = {0};
TBC *pUidIdxc = NULL; TBC * pUidIdxc = NULL;
TBC *pTbDbc = NULL; TBC * pTbDbc = NULL;
void *pData = NULL; void * pData = NULL;
int nData = 0; int nData = 0;
int64_t oversion; int64_t oversion;
SDecoder dc = {0}; SDecoder dc = {0};
@ -638,6 +683,7 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
nStbEntry.name = pReq->name; nStbEntry.name = pReq->name;
nStbEntry.stbEntry.schemaRow = pReq->schemaRow; nStbEntry.stbEntry.schemaRow = pReq->schemaRow;
nStbEntry.stbEntry.schemaTag = pReq->schemaTag; nStbEntry.stbEntry.schemaTag = pReq->schemaTag;
nStbEntry.colCmpr = pReq->colCmpr;
metaWLock(pMeta); metaWLock(pMeta);
// update table.db // update table.db
@ -664,11 +710,11 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
SMetaEntry nStbEntry = {0}; SMetaEntry nStbEntry = {0};
STbDbKey tbDbKey = {0}; STbDbKey tbDbKey = {0};
TBC *pUidIdxc = NULL; TBC * pUidIdxc = NULL;
TBC *pTbDbc = NULL; TBC * pTbDbc = NULL;
int ret = 0; int ret = 0;
int c = -2; int c = -2;
void *pData = NULL; void * pData = NULL;
int nData = 0; int nData = 0;
int64_t oversion; int64_t oversion;
SDecoder dc = {0}; SDecoder dc = {0};
@ -771,12 +817,16 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
nStbEntry.uid = oStbEntry.uid; nStbEntry.uid = oStbEntry.uid;
nStbEntry.name = oStbEntry.name; nStbEntry.name = oStbEntry.name;
SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); SSchemaWrapper * row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow);
SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); SSchemaWrapper * tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag);
SColCmprWrapper *cmpr = tCloneSColCmprWrapper(&oStbEntry.colCmpr);
nStbEntry.stbEntry.schemaRow = *row; nStbEntry.stbEntry.schemaRow = *row;
nStbEntry.stbEntry.schemaTag = *tag; nStbEntry.stbEntry.schemaTag = *tag;
nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam;
nStbEntry.colCmpr = *cmpr;
nStbEntry.colCmpr = oStbEntry.colCmpr;
metaWLock(pMeta); metaWLock(pMeta);
// update table.db // update table.db
@ -787,6 +837,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq)
tDeleteSchemaWrapper(tag); tDeleteSchemaWrapper(tag);
tDeleteSchemaWrapper(row); tDeleteSchemaWrapper(row);
tDeleteSColCmprWrapper(cmpr);
if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf);
tDecoderClear(&dc); tDecoderClear(&dc);
@ -901,6 +952,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
me.ntbEntry.comment = pReq->comment; me.ntbEntry.comment = pReq->comment;
me.ntbEntry.schemaRow = pReq->ntb.schemaRow; me.ntbEntry.schemaRow = pReq->ntb.schemaRow;
me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1; me.ntbEntry.ncid = me.ntbEntry.schemaRow.pSchema[me.ntbEntry.schemaRow.nCols - 1].colId + 1;
me.colCmpr = pReq->colCmpr;
TABLE_SET_COL_COMPRESSED(me.flags);
++pStats->numOfNTables; ++pStats->numOfNTables;
pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1; pStats->numOfNTimeSeries += me.ntbEntry.schemaRow.nCols - 1;
@ -925,6 +978,11 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
strcpy((*pMetaRsp)->tbName, pReq->name); strcpy((*pMetaRsp)->tbName, pReq->name);
} else { } else {
metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp); metaUpdateMetaRsp(pReq->uid, pReq->name, &pReq->ntb.schemaRow, *pMetaRsp);
for (int32_t i = 0; i < pReq->colCmpr.nCols; i++) {
SColCmpr *p = &pReq->colCmpr.pColCmpr[i];
(*pMetaRsp)->pSchemaExt[i].colId = p->id;
(*pMetaRsp)->pSchemaExt[i].compress = p->alg;
}
} }
} }
} }
@ -941,7 +999,7 @@ _err:
} }
int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) {
void *pData = NULL; void * pData = NULL;
int nData = 0; int nData = 0;
int rc = 0; int rc = 0;
tb_uid_t uid = 0; tb_uid_t uid = 0;
@ -1020,10 +1078,10 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) {
metaULock(pMeta); metaULock(pMeta);
// update timeseries // update timeseries
void *pCtbDropped = NULL; void * pCtbDropped = NULL;
int32_t iter = 0; int32_t iter = 0;
while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) { while ((pCtbDropped = tSimpleHashIterate(suidHash, pCtbDropped, &iter))) {
tb_uid_t *pSuid = tSimpleHashGetKey(pCtbDropped, NULL); tb_uid_t * pSuid = tSimpleHashGetKey(pCtbDropped, NULL);
int32_t nCols = 0; int32_t nCols = 0;
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats; SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) { if (metaGetStbStats(pMeta->pVnode, *pSuid, NULL, &nCols) == 0) {
@ -1163,7 +1221,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
} }
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) { static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) {
void *pData = NULL; void * pData = NULL;
int nData = 0; int nData = 0;
int rc = 0; int rc = 0;
SMetaEntry e = {0}; SMetaEntry e = {0};
@ -1202,7 +1260,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0; if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0;
SSchema *pTagColumn = NULL; SSchema * pTagColumn = NULL;
SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) {
pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0];
@ -1326,18 +1384,18 @@ int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME) {
} }
static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) {
void *pVal = NULL; void * pVal = NULL;
int nVal = 0; int nVal = 0;
const void *pData = NULL; const void * pData = NULL;
int nData = 0; int nData = 0;
int ret = 0; int ret = 0;
tb_uid_t uid; tb_uid_t uid;
int64_t oversion; int64_t oversion;
SSchema *pColumn = NULL; SSchema * pColumn = NULL;
SMetaEntry entry = {0}; SMetaEntry entry = {0};
SSchemaWrapper *pSchema; SSchemaWrapper *pSchema;
int c; int c;
bool freeColCmpr = false;
if (pAlterTbReq->colName == NULL) { if (pAlterTbReq->colName == NULL) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
metaError("meta/table: null pAlterTbReq->colName"); metaError("meta/table: null pAlterTbReq->colName");
@ -1434,6 +1492,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
entry.version = version; entry.version = version;
int tlen; int tlen;
SSchema *pNewSchema = NULL; SSchema *pNewSchema = NULL;
SSchema tScheam;
switch (pAlterTbReq->action) { switch (pAlterTbReq->action) {
case TSDB_ALTER_TABLE_ADD_COLUMN: case TSDB_ALTER_TABLE_ADD_COLUMN:
if (pColumn) { if (pColumn) {
@ -1466,6 +1525,10 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type; int8_t col_type = pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1].type;
(void)tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type); (void)tsdbCacheNewNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, col_type);
} }
SSchema *pCol = &pSchema->pSchema[entry.ntbEntry.schemaRow.nCols - 1];
updataTableColCmpr(&entry.colCmpr, pCol, 1);
freeColCmpr = true;
ASSERT(entry.colCmpr.nCols == pSchema->nCols);
break; break;
case TSDB_ALTER_TABLE_DROP_COLUMN: case TSDB_ALTER_TABLE_DROP_COLUMN:
if (pColumn == NULL) { if (pColumn == NULL) {
@ -1485,6 +1548,7 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false; hasPrimayKey = pSchema->pSchema[1].flags & COL_IS_KEY ? true : false;
} }
memcpy(&tScheam, pColumn, sizeof(SSchema));
pSchema->version++; pSchema->version++;
tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema); tlen = (pSchema->nCols - iCol - 1) * sizeof(SSchema);
if (tlen) { if (tlen) {
@ -1499,6 +1563,9 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
(void)tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey); (void)tsdbCacheDropNTableColumn(pMeta->pVnode->pTsdb, entry.uid, cid, hasPrimayKey);
} }
updataTableColCmpr(&entry.colCmpr, &tScheam, 0);
ASSERT(entry.colCmpr.nCols == pSchema->nCols);
break; break;
case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:
if (pColumn == NULL) { if (pColumn == NULL) {
@ -1557,9 +1624,16 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl
metaULock(pMeta); metaULock(pMeta);
metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp); metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp);
for (int32_t i = 0; i < entry.colCmpr.nCols; i++) {
SColCmpr *p = &entry.colCmpr.pColCmpr[i];
pMetaRsp->pSchemaExt[i].colId = p->id;
pMetaRsp->pSchemaExt[i].compress = p->alg;
}
if (entry.pBuf) taosMemoryFree(entry.pBuf); if (entry.pBuf) taosMemoryFree(entry.pBuf);
if (pNewSchema) taosMemoryFree(pNewSchema); if (pNewSchema) taosMemoryFree(pNewSchema);
if (freeColCmpr) taosMemoryFree(entry.colCmpr.pColCmpr);
tdbTbcClose(pTbDbc); tdbTbcClose(pTbDbc);
tdbTbcClose(pUidIdxc); tdbTbcClose(pUidIdxc);
tDecoderClear(&dc); tDecoderClear(&dc);
@ -1578,7 +1652,7 @@ _err:
static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
SMetaEntry ctbEntry = {0}; SMetaEntry ctbEntry = {0};
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
void *pVal = NULL; void * pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
@ -1619,7 +1693,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
oversion = ((SUidIdxVal *)pData)[0].version; oversion = ((SUidIdxVal *)pData)[0].version;
// search table.db // search table.db
TBC *pTbDbc = NULL; TBC * pTbDbc = NULL;
SDecoder dc1 = {0}; SDecoder dc1 = {0};
SDecoder dc2 = {0}; SDecoder dc2 = {0};
@ -1655,7 +1729,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
metaDecodeEntry(&dc2, &stbEntry); metaDecodeEntry(&dc2, &stbEntry);
SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
SSchema *pColumn = NULL; SSchema * pColumn = NULL;
int32_t iCol = 0; int32_t iCol = 0;
for (;;) { for (;;) {
pColumn = NULL; pColumn = NULL;
@ -1682,8 +1756,8 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
} else { } else {
const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags;
STag *pNewTag = NULL; STag * pNewTag = NULL;
SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); SArray * pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal));
if (!pTagArray) { if (!pTagArray) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err; goto _err;
@ -1765,7 +1839,7 @@ _err:
} }
static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
void *pVal = NULL; void * pVal = NULL;
int nVal = 0; int nVal = 0;
const void *pData = NULL; const void *pData = NULL;
int nData = 0; int nData = 0;
@ -1869,7 +1943,7 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p
static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
void *pVal = NULL; void * pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
@ -2002,7 +2076,7 @@ typedef struct SMetaPair {
static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) {
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
void *pVal = NULL; void * pVal = NULL;
int nVal = 0; int nVal = 0;
int ret; int ret;
int c; int c;
@ -2109,6 +2183,91 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT
_err: _err:
return -1; return -1;
} }
int32_t metaUpdateTableColCompress(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) {
// impl later
SMetaEntry tbEntry = {0};
void * pVal = NULL;
int nVal = 0;
int ret;
int c;
tb_uid_t suid;
int64_t oversion;
const void *pData = NULL;
int nData = 0;
SDecoder dc = {0};
ret = tdbTbGet(pMeta->pNameIdx, pReq->tbName, strlen(pReq->tbName) + 1, &pVal, &nVal);
if (ret < 0) {
terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST;
return -1;
}
suid = *(tb_uid_t *)pVal;
tdbFree(pVal);
pVal = NULL;
if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) {
terrno = TSDB_CODE_INVALID_MSG;
ret = -1;
goto _err;
}
STbDbKey tbDbKey = {0};
tbDbKey.uid = suid;
tbDbKey.version = ((SUidIdxVal *)pVal)[0].version;
if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal) < 0) {
terrno = TSDB_CODE_INVALID_MSG;
tdbFree(pVal);
goto _err;
}
tDecoderInit(&dc, pVal, nVal);
ret = metaDecodeEntry(&dc, &tbEntry);
if (ret < 0) {
terrno = TSDB_CODE_INVALID_MSG;
tdbFree(pVal);
tDecoderClear(&dc);
goto _err;
}
if (tbEntry.type != TSDB_NORMAL_TABLE && tbEntry.type != TSDB_SUPER_TABLE) {
terrno = TSDB_CODE_INVALID_MSG;
tdbFree(pVal);
tDecoderClear(&dc);
goto _err;
}
int8_t updated = 0;
SColCmprWrapper *wp = &tbEntry.colCmpr;
for (int32_t i = 0; i < wp->nCols; i++) {
SColCmpr *p = &wp->pColCmpr[i];
if (p->id == pReq->colId) {
uint32_t dst = 0;
updated = tUpdateCompress(p->alg, pReq->compress, TSDB_COLVAL_COMPRESS_DISABLED, TSDB_COLVAL_LEVEL_DISABLED,
TSDB_COLVAL_LEVEL_MEDIUM, &dst);
if (updated) {
p->alg = dst;
}
}
}
if (updated == 0) {
tdbFree(pVal);
tDecoderClear(&dc);
terrno = TSDB_CODE_VND_COLUMN_COMPRESS_ALREADY_EXIST;
goto _err;
}
tbEntry.version = version;
metaWLock(pMeta);
metaSaveToTbDb(pMeta, &tbEntry);
metaUpdateUidIdx(pMeta, &tbEntry);
metaUpdateChangeTime(pMeta, suid, pReq->ctimeMs);
metaULock(pMeta);
tdbFree(pVal);
tDecoderClear(&dc);
return 0;
_err:
return -1;
}
int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) {
pMeta->changed = true; pMeta->changed = true;
@ -2126,6 +2285,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta
return metaAddTagIndex(pMeta, version, pReq); return metaAddTagIndex(pMeta, version, pReq);
case TSDB_ALTER_TABLE_DROP_TAG_INDEX: case TSDB_ALTER_TABLE_DROP_TAG_INDEX:
return metaDropTagIndex(pMeta, version, pReq); return metaDropTagIndex(pMeta, version, pReq);
case TSDB_ALTER_TABLE_UPDATE_COLUMN_COMPRESS:
return metaUpdateTableColCompress(pMeta, version, pReq);
default: default:
terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION;
return -1; return -1;
@ -2135,8 +2296,8 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta
static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) { static int metaSaveToTbDb(SMeta *pMeta, const SMetaEntry *pME) {
STbDbKey tbDbKey; STbDbKey tbDbKey;
void *pKey = NULL; void * pKey = NULL;
void *pVal = NULL; void * pVal = NULL;
int kLen = 0; int kLen = 0;
int vLen = 0; int vLen = 0;
SEncoder coder = {0}; SEncoder coder = {0};
@ -2287,14 +2448,14 @@ static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey) {
} }
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) {
void *pData = NULL; void * pData = NULL;
int nData = 0; int nData = 0;
STbDbKey tbDbKey = {0}; STbDbKey tbDbKey = {0};
SMetaEntry stbEntry = {0}; SMetaEntry stbEntry = {0};
STagIdxKey *pTagIdxKey = NULL; STagIdxKey * pTagIdxKey = NULL;
int32_t nTagIdxKey; int32_t nTagIdxKey;
const SSchema *pTagColumn; const SSchema *pTagColumn;
const void *pTagData = NULL; const void * pTagData = NULL;
int32_t nTagData = 0; int32_t nTagData = 0;
SDecoder dc = {0}; SDecoder dc = {0};
int32_t ret = 0; int32_t ret = 0;
@ -2363,7 +2524,7 @@ end:
static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) { static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME) {
SEncoder coder = {0}; SEncoder coder = {0};
void *pVal = NULL; void * pVal = NULL;
int vLen = 0; int vLen = 0;
int rcode = 0; int rcode = 0;
SSkmDbKey skmDbKey = {0}; SSkmDbKey skmDbKey = {0};
@ -2463,6 +2624,9 @@ int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) {
VND_CHECK_CODE(code, line, _err); VND_CHECK_CODE(code, line, _err);
} }
if (pME->type == TSDB_SUPER_TABLE || pME->type == TSDB_NORMAL_TABLE) {
}
metaULock(pMeta); metaULock(pMeta);
metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode), metaDebug("vgId:%d, handle meta entry, ver:%" PRId64 ", uid:%" PRId64 ", name:%s", TD_VID(pMeta->pVnode),
pME->version, pME->uid, pME->name); pME->version, pME->uid, pME->name);
@ -2475,6 +2639,79 @@ _err:
return -1; return -1;
} }
int32_t colCompressDebug(SHashObj *pColCmprObj) {
void *p = taosHashIterate(pColCmprObj, NULL);
while (p) {
uint32_t cmprAlg = *(uint32_t *)p;
col_id_t colId = *(col_id_t *)taosHashGetKey(p, NULL);
p = taosHashIterate(pColCmprObj, p);
uint8_t l1, l2, lvl;
tcompressDebug(cmprAlg, &l1, &l2, &lvl);
const char *l1str = columnEncodeStr(l1);
const char *l2str = columnCompressStr(l2);
const char *lvlstr = columnLevelStr(lvl);
metaInfo("colId: %d, encode:%s, compress:%s,level:%s", colId, l1str, l2str, lvlstr);
}
return 0;
}
int32_t metaGetColCmpr(SMeta *pMeta, tb_uid_t uid, SHashObj **ppColCmprObj) {
int rc = 0;
SHashObj * pColCmprObj = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
void * pData = NULL;
int nData = 0;
SMetaEntry e = {0};
SDecoder dc = {0};
metaRLock(pMeta);
rc = tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData);
if (rc < 0) {
taosHashClear(pColCmprObj);
metaULock(pMeta);
return -1;
}
int64_t version = ((SUidIdxVal *)pData)[0].version;
rc = tdbTbGet(pMeta->pTbDb, &(STbDbKey){.version = version, .uid = uid}, sizeof(STbDbKey), &pData, &nData);
if (rc < 0) {
metaULock(pMeta);
taosHashClear(pColCmprObj);
metaError("failed to get table entry");
return rc;
}
tDecoderInit(&dc, pData, nData);
rc = metaDecodeEntry(&dc, &e);
if (rc < 0) {
tDecoderClear(&dc);
tdbFree(pData);
metaULock(pMeta);
taosHashClear(pColCmprObj);
return -1;
}
if (useCompress(e.type)) {
SColCmprWrapper *p = &e.colCmpr;
for (int32_t i = 0; i < p->nCols; i++) {
SColCmpr *pCmpr = &p->pColCmpr[i];
taosHashPut(pColCmprObj, &pCmpr->id, sizeof(pCmpr->id), &pCmpr->alg, sizeof(pCmpr->alg));
}
} else {
tDecoderClear(&dc);
tdbFree(pData);
metaULock(pMeta);
taosHashClear(pColCmprObj);
return 0;
}
tDecoderClear(&dc);
tdbFree(pData);
metaULock(pMeta);
*ppColCmprObj = pColCmprObj;
colCompressDebug(pColCmprObj);
return 0;
}
// refactor later // refactor later
void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; } void *metaGetIdx(SMeta *pMeta) { return pMeta->pTagIdx; }
void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; } void *metaGetIvtIdx(SMeta *pMeta) { return pMeta->pTagIvtIdx; }

View File

@ -711,22 +711,6 @@ end:
static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); } static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); }
static STaskId replaceStreamTaskId(SStreamTask* pTask) {
ASSERT(pTask->info.fillHistory);
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
pTask->id.streamId = pTask->streamTaskId.streamId;
pTask->id.taskId = pTask->streamTaskId.taskId;
return id;
}
static void restoreStreamTaskId(SStreamTask* pTask, STaskId* pId) {
ASSERT(pTask->info.fillHistory);
pTask->id.taskId = pId->taskId;
pTask->id.streamId = pId->streamId;
}
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
tqDebug("s-task:0x%x start to expand task", pTask->id.taskId); tqDebug("s-task:0x%x start to expand task", pTask->id.taskId);
@ -736,74 +720,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
return code; return code;
} }
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { code = tqExpandStreamTask(pTask, pTq->pStreamMeta, pTq->pVnode);
STaskId taskId = {0}; if (code != TSDB_CODE_SUCCESS) {
if (pTask->info.fillHistory) { return code;
taskId = replaceStreamTaskId(pTask);
}
pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
if (pTask->pState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId);
return -1;
}
tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
if (pTask->info.fillHistory) {
restoreStreamTaskId(pTask, &taskId);
}
SReadHandle handle = {
.checkpointId = pTask->chkInfo.checkpointId,
.vnode = pTq->pVnode,
.initTqReader = 1,
.pStateBackend = pTask->pState,
.fillHistory = pTask->info.fillHistory,
.winRange = pTask->dataRange.window,
};
initStorageAPI(&handle.api);
pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId, pTask->id.taskId);
if (pTask->exec.pExecutor == NULL) {
return -1;
}
qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId);
} else if (pTask->info.taskLevel == TASK_LEVEL__AGG) {
STaskId taskId = {0};
if (pTask->info.fillHistory) {
taskId = replaceStreamTaskId(pTask);
}
pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1);
if (pTask->pState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId);
return -1;
} else {
tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
}
if (pTask->info.fillHistory) {
restoreStreamTaskId(pTask, &taskId);
}
SReadHandle handle = {
.checkpointId = pTask->chkInfo.checkpointId,
.vnode = NULL,
.numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList),
.pStateBackend = pTask->pState,
.fillHistory = pTask->info.fillHistory,
.winRange = pTask->dataRange.window,
};
initStorageAPI(&handle.api);
pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId, pTask->id.taskId);
if (pTask->exec.pExecutor == NULL) {
return -1;
}
qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId);
} }
// sink // sink
@ -839,12 +758,15 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
streamTaskResetUpstreamStageInfo(pTask); streamTaskResetUpstreamStageInfo(pTask);
streamSetupScheduleTrigger(pTask); streamSetupScheduleTrigger(pTask);
SCheckpointInfo* pChkInfo = &pTask->chkInfo; SCheckpointInfo* pChkInfo = &pTask->chkInfo;
// checkpoint ver is the kept version, handled data should be the next version. // checkpoint ver is the kept version, handled data should be the next version.
if (pChkInfo->checkpointId != 0) { if (pChkInfo->checkpointId != 0) {
pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1; pChkInfo->nextProcessVer = pChkInfo->checkpointVer + 1;
pChkInfo->processedVer = pChkInfo->checkpointVer; pChkInfo->processedVer = pChkInfo->checkpointVer;
pTask->execInfo.startCheckpointVer = pChkInfo->nextProcessVer;
pTask->execInfo.startCheckpointId = pChkInfo->checkpointId;
tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr, tqInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " currentVer:%" PRId64, pTask->id.idStr,
pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
} }
@ -890,33 +812,33 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer; int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer;
SVersionRange* pRange = &pTask->dataRange.range; SVersionRange* pStep2Range = &pTask->step2Range;
// if it's an source task, extract the last version in wal. // if it's an source task, extract the last version in wal.
bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer); bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
pTask->execInfo.step2Start = taosGetTimestampMs(); pTask->execInfo.step2Start = taosGetTimestampMs();
if (done) { if (done) {
qDebug("s-task:%s scan wal(step 2) verRange:%" PRId64 "-%" PRId64 " ended, elapsed time:%.2fs", id, pRange->minVer, qDebug("s-task:%s scan wal(step 2) verRange:%" PRId64 "-%" PRId64 " ended, elapsed time:%.2fs", id, pStep2Range->minVer,
pRange->maxVer, 0.0); pStep2Range->maxVer, 0.0);
streamTaskPutTranstateIntoInputQ(pTask); streamTaskPutTranstateIntoInputQ(pTask);
streamExecTask(pTask); // exec directly streamExecTask(pTask); // exec directly
} else { } else {
STimeWindow* pWindow = &pTask->dataRange.window; STimeWindow* pWindow = &pTask->dataRange.window;
tqDebug("s-task:%s level:%d verRange:%" PRId64 " - %" PRId64 " window:%" PRId64 "-%" PRId64 tqDebug("s-task:%s level:%d verRange:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64
", do secondary scan-history from WAL after halt the related stream task:%s", ", do secondary scan-history from WAL after halt the related stream task:%s",
id, pTask->info.taskLevel, pRange->minVer, pRange->maxVer, pWindow->skey, pWindow->ekey, id, pTask->info.taskLevel, pStep2Range->minVer, pStep2Range->maxVer, pWindow->skey, pWindow->ekey,
pStreamTask->id.idStr); pStreamTask->id.idStr);
ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING); ASSERT(pTask->status.schedStatus == TASK_SCHED_STATUS__WAITING);
streamSetParamForStreamScannerStep2(pTask, pRange, pWindow); streamSetParamForStreamScannerStep2(pTask, pStep2Range, pWindow);
int64_t dstVer = pTask->dataRange.range.minVer; int64_t dstVer =pStep2Range->minVer;
pTask->chkInfo.nextProcessVer = dstVer; pTask->chkInfo.nextProcessVer = dstVer;
walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer); walReaderSetSkipToVersion(pTask->exec.pWalReader, dstVer);
tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer, tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer,
pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE); pStep2Range->maxVer, TASK_SCHED_STATUS__INACTIVE);
/*int8_t status = */ streamTaskSetSchedStatusInactive(pTask); /*int8_t status = */ streamTaskSetSchedStatusInactive(pTask);
@ -1014,7 +936,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
atomic_store_32(&pTask->status.inScanHistorySentinel, 0); atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
if (retInfo.ret == TASK_SCANHISTORY_REXEC) { if (retInfo.ret == TASK_SCANHISTORY_REXEC) {
streamReExecScanHistoryFuture(pTask, retInfo.idleTime); streamExecScanHistoryInFuture(pTask, retInfo.idleTime);
} else { } else {
SStreamTaskState* p = streamTaskGetStatus(pTask); SStreamTaskState* p = streamTaskGetStatus(pTask);
ETaskStatus s = p->state; ETaskStatus s = p->state;
@ -1109,6 +1031,44 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) {
return 0; return 0;
} }
int32_t tqStreamProgressRetrieveReq(STQ *pTq, SRpcMsg *pMsg) {
char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamProgressReq req;
char* pRspBuf = taosMemoryCalloc(1, sizeof(SMsgHead) + sizeof(SStreamProgressRsp));
SStreamProgressRsp* pRsp = POINTER_SHIFT(pRspBuf, sizeof(SMsgHead));
if (!pRspBuf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _OVER;
}
code = tDeserializeStreamProgressReq(msgBody, msgLen, &req);
if (code == TSDB_CODE_SUCCESS) {
code = tqGetStreamExecInfo(pTq->pVnode, req.streamId, &pRsp->progressDelay, &pRsp->fillHisFinished);
}
if (code == TSDB_CODE_SUCCESS) {
pRsp->fetchIdx = req.fetchIdx;
pRsp->subFetchIdx = req.subFetchIdx;
pRsp->vgId = req.vgId;
pRsp->streamId = req.streamId;
tSerializeStreamProgressRsp(pRsp, sizeof(SStreamProgressRsp) + sizeof(SMsgHead), pRsp);
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
rsp.pCont = pRspBuf;
pRspBuf = NULL;
rsp.contLen = sizeof(SMsgHead) + sizeof(SStreamProgressRsp);
tmsgSendRsp(&rsp);
}
_OVER:
if (pRspBuf) {
taosMemoryFree(pRspBuf);
}
return code;
}
int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) { int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
@ -1142,7 +1102,9 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
} }
if (!pTq->pVnode->restored) { if (!pTq->pVnode->restored) {
tqDebug("vgId:%d checkpoint-source msg received during restoring, s-task:0x%x ignore it", vgId, req.taskId); tqDebug("vgId:%d checkpoint-source msg received during restoring, checkpointId:%" PRId64
", transId:%d s-task:0x%x ignore it",
vgId, req.checkpointId, req.transId, req.taskId);
SRpcMsg rsp = {0}; SRpcMsg rsp = {0};
buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0);
tmsgSendRsp(&rsp); // error occurs tmsgSendRsp(&rsp); // error occurs
@ -1151,7 +1113,9 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. it may have been destroyed", vgId, req.taskId); tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. checkpointId:%" PRId64
" transId:%d it may have been destroyed",
vgId, req.taskId, req.checkpointId, req.transId);
SRpcMsg rsp = {0}; SRpcMsg rsp = {0};
buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0); buildCheckpointSourceRsp(&req, &pMsg->info, &rsp, 0);
tmsgSendRsp(&rsp); // error occurs tmsgSendRsp(&rsp); // error occurs
@ -1163,7 +1127,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
pTask->chkInfo.checkpointingId = req.checkpointId; pTask->chkInfo.checkpointingId = req.checkpointId;
pTask->chkInfo.transId = req.transId; pTask->chkInfo.transId = req.transId;
tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpointId:%" PRId64
", transId:%d set it failed", ", transId:%d set it failed",
pTask->id.idStr, req.checkpointId, req.transId); pTask->id.idStr, req.checkpointId, req.transId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
@ -1180,7 +1144,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
if (req.mndTrigger == 1) { if (req.mndTrigger == 1) {
if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) { if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {
tqError("s-task:%s not ready for checkpoint, since it is halt, ignore checkpoint:%" PRId64 ", set it failure", tqError("s-task:%s not ready for checkpoint, since it is halt, ignore checkpointId:%" PRId64 ", set it failure",
pTask->id.idStr, req.checkpointId); pTask->id.idStr, req.checkpointId);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);

View File

@ -45,7 +45,7 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) {
// 1. the vnode has already been restored. // 1. the vnode has already been restored.
// 2. the vnode should be the leader. // 2. the vnode should be the leader.
// 3. the stream is not suspended yet. // 3. the stream is not suspended yet.
if ((!tsDisableStream) && (numOfTasks > 0) && (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_DELETE)) { if ((!tsDisableStream) && (numOfTasks > 0)) {
tqScanWalAsync(pTq, true); tqScanWalAsync(pTq, true);
} }

View File

@ -592,7 +592,7 @@ static int32_t buildResSDataBlock(SSDataBlock* pBlock, SSchemaWrapper* pSchema,
static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SColVal* pColVal) { static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SColVal* pColVal) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
if (IS_STR_DATA_TYPE(pColVal->value.type)) { if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
char val[65535 + 2] = {0}; char val[65535 + 2] = {0};
if (COL_VAL_IS_VALUE(pColVal)) { if (COL_VAL_IS_VALUE(pColVal)) {
if (pColVal->value.pData != NULL) { if (pColVal->value.pData != NULL) {

View File

@ -309,11 +309,10 @@ int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2*
pRec->numOfSubmit += 1; pRec->numOfSubmit += 1;
if ((pRec->numOfSubmit % 1000) == 0) { if ((pRec->numOfSubmit % 1000) == 0) {
double el = (taosGetTimestampMs() - pTask->execInfo.start) / 1000.0; double el = (taosGetTimestampMs() - pTask->execInfo.readyTs) / 1000.0;
tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) in %" PRId64 tqInfo("s-task:%s vgId:%d write %" PRId64 " blocks (%" PRId64 " rows) in %" PRId64
" submit into dst table, %.2fMiB duration:%.2f Sec.", " submit into dst table, %.2fMiB duration:%.2f Sec.",
pTask->id.idStr, vgId, pRec->numOfBlocks, pRec->numOfRows, pRec->numOfSubmit, SIZE_IN_MiB(pRec->dataSize), id, vgId, pRec->numOfBlocks, pRec->numOfRows, pRec->numOfSubmit, SIZE_IN_MiB(pRec->dataSize), el);
el);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -604,7 +603,7 @@ int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDat
dataIndex++; dataIndex++;
} else { } else {
void* colData = colDataGetData(pColData, j); void* colData = colDataGetData(pColData, j);
if (IS_STR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
// address copy, no value // address copy, no value
SValue sv = SValue sv =
(SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)}; (SValue){.type = pCol->type, .nData = varDataLen(colData), .pData = (uint8_t*)varDataVal(colData)};
@ -794,8 +793,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
return TDB_CODE_SUCCESS; return TDB_CODE_SUCCESS;
} }
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock, int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id) { SSubmitTbData* pTableData, const char* id) {
int32_t numOfRows = pDataBlock->info.rows; int32_t numOfRows = pDataBlock->info.rows;
tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, id, tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, id,

View File

@ -242,21 +242,23 @@ int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId) {
// todo handle memory error // todo handle memory error
bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) { bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) {
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int64_t maxVer = pTask->dataRange.range.maxVer; int64_t maxVer = pTask->step2Range.maxVer;
if ((pTask->info.fillHistory == 1) && ver > pTask->dataRange.range.maxVer) { if ((pTask->info.fillHistory == 1) && ver > maxVer) {
if (!pTask->status.appendTranstateBlock) { if (!pTask->status.appendTranstateBlock) {
qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the maximum ver:%" PRId64 qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the maximum ver:%" PRId64
", not scan wal anymore, add transfer-state block into inputQ", ", not scan wal anymore, add transfer-state block into inputQ",
id, ver, maxVer); id, ver, maxVer);
double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0; double el = (taosGetTimestampMs() - pTask->execInfo.step2Start) / 1000.0;
qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, el); qDebug("s-task:%s scan-history from WAL stage(step 2) ended, range:%" PRId64 "-%" PRId64 ", elapsed time:%.2fs",
id, pTask->step2Range.minVer, maxVer, el);
/*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask); /*int32_t code = */streamTaskPutTranstateIntoInputQ(pTask);
return true; return true;
} else { } else {
qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the maximum ver:%" PRId64 ", not scan wal", qWarn("s-task:%s fill-history scan WAL, nextProcessVer:%" PRId64 " out of the ver range:%" PRId64 "-%" PRId64
id, ver, maxVer); ", not scan wal",
id, ver, pTask->step2Range.minVer, maxVer);
} }
} }
@ -389,7 +391,7 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
} }
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX; int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->step2Range.maxVer : INT64_MAX;
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);

Some files were not shown because too many files have changed in this diff Show More