xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TD-30313

This commit is contained in:
54liuyao 2024-06-18 10:28:01 +08:00
parent 7620be6e28 f93127489b
commit e858dd5df0
180 changed files with 10094 additions and 8770 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmake/cmake.version
View File

@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER) IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER}) SET(TD_VER_NUMBER ${VERNUMBER})
ELSE () ELSE ()
SET(TD_VER_NUMBER "3.3.1.0.alpha") SET(TD_VER_NUMBER "3.3.2.0.alpha")
ENDIF () ENDIF ()
IF (DEFINED VERCOMPATIBLE) IF (DEFINED VERCOMPATIBLE)

2
cmake/libuv_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# libuv # libuv
ExternalProject_Add(libuv ExternalProject_Add(libuv
GIT_REPOSITORY https://github.com/libuv/libuv.git GIT_REPOSITORY https://github.com/libuv/libuv.git
GIT_TAG v1.44.2 GIT_TAG v1.48.0
SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" SOURCE_DIR "${TD_CONTRIB_DIR}/libuv"
BINARY_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv"
CONFIGURE_COMMAND "" CONFIGURE_COMMAND ""

2
cmake/taostools_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# taos-tools # taos-tools
ExternalProject_Add(taos-tools ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
GIT_TAG main GIT_TAG 3.0
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR "" BINARY_DIR ""
#BUILD_IN_SOURCE TRUE #BUILD_IN_SOURCE TRUE

2
docs/en/12-taos-sql/14-stream.md
View File

@ -47,7 +47,7 @@ window_clause: {
} }
``` ```
`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. `SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.The `_wend` of this window is the time of the last data plus `tol_val`.
`EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns. `EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns.

2
docs/en/14-reference/12-config/index.md
View File

@ -432,7 +432,7 @@ The charset that takes effect is UTF-8.
| Applicable | Server Only | | Applicable | Server Only |
| Meaning | Maximum number of threads to commit | | Meaning | Maximum number of threads to commit |
| Value Range | 0-1024 | | Value Range | 0-1024 |
| Default Value | | | Default Value | 4 |
## Log Parameters ## Log Parameters

14
docs/en/21-tdinternal/08-compress.md
View File

@ -1,12 +1,8 @@
--- ---
title: Configurable Column Compression title: Configurable Column Compression
description: Configurable column storage compression method description: Configurable column storage compression method
--- ---
# Configurable Storage Compression
Since TDengine 3.3.0.0, more advanced compression feature is introduced, you can specify compression or not, the compression method and compression level for each column. Since TDengine 3.3.0.0, more advanced compression feature is introduced, you can specify compression or not, the compression method and compression level for each column.
## Compression Terminology Definition ## Compression Terminology Definition
@ -32,16 +28,14 @@ In this article, it specifically refers to the level within the secondary compre
- Default compression algorithm list and applicable range for each data type - Default compression algorithm list and applicable range for each data type
| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level| | Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level|
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| | tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| | bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium| |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| |binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| |bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
Note: For floating point types, if configured as tsz, its precision is determined by the global configuration of taosd. If configured as tsz, but the lossy compression flag is not configured, lz4 is used for compression by default.
## SQL ## SQL
### Create Table with Compression ### Create Table with Compression
@ -76,7 +70,7 @@ ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPR
- Change the compression method of the column - Change the compression method of the column
### View Compression Dethod ### View Compression Method
```sql ```sql
DESCRIBE [dbname.]tabName DESCRIBE [dbname.]tabName

4
docs/en/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.3.1.0
<Release type="tdengine" version="3.3.1.0" />
## 3.3.0.3 ## 3.3.0.3
<Release type="tdengine" version="3.3.0.3" /> <Release type="tdengine" version="3.3.0.3" />

4
docs/zh/12-taos-sql/14-stream.md
View File

@ -54,8 +54,10 @@ window_clause: {
} }
``` ```
其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。
EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。
COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。
窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished)

2
docs/zh/14-reference/12-config/index.md
View File

@ -430,7 +430,7 @@ charset 的有效值是 UTF-8。
| 适用范围 | 仅服务端适用 | | 适用范围 | 仅服务端适用 |
| 含义 | 设置写入线程的最大数量 | | 含义 | 设置写入线程的最大数量 |
| 取值范围 | 0-1024 | | 取值范围 | 0-1024 |
| 缺省值 | | | 缺省值 | 4 |
## 日志相关 ## 日志相关

10
docs/zh/21-tdinternal/08-compress.md
View File

@ -3,8 +3,6 @@ title: 可配置压缩算法
description: 可配置压缩算法 description: 可配置压缩算法
--- ---
# 可配置存储压缩
从 TDengine 3.3.0.0 版本开始，TDengine 提供了更高级的压缩功能，用户可以在建表时针对每一列配置是否进行压缩、以及使用的压缩算法和压缩级别。 从 TDengine 3.3.0.0 版本开始，TDengine 提供了更高级的压缩功能，用户可以在建表时针对每一列配置是否进行压缩、以及使用的压缩算法和压缩级别。
## 压缩术语定义 ## 压缩术语定义
@ -30,16 +28,14 @@ description: 可配置压缩算法
- 各个数据类型的默认压缩算法列表和适用范围 - 各个数据类型的默认压缩算法列表和适用范围
| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|可选压缩算法| 压缩等级默认值| | 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值|
| :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:|
tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| | tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium|
| bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| | bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium|
|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium| |float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium|
|binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| |binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium|
|bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| |bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium|
注意: 针对浮点类型，如果配置为tsz, 其精度由taosd的全局配置决定，如果配置为tsz, 但是没有配置有损压缩标志, 则使用lz4进行压缩
## SQL 语法 ## SQL 语法
### 建表时指定压缩 ### 建表时指定压缩

4
docs/zh/28-releases/01-tdengine.md
View File

@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do
import Release from "/components/ReleaseV3"; import Release from "/components/ReleaseV3";
## 3.3.1.0
<Release type="tdengine" version="3.3.1.0" />
## 3.3.0.3 ## 3.3.0.3
<Release type="tdengine" version="3.3.0.3" /> <Release type="tdengine" version="3.3.0.3" />

7
include/client/taos.h
View File

@ -150,6 +150,12 @@ typedef struct TAOS_DB_ROUTE_INFO {
TAOS_VGROUP_HASH_INFO *vgHash; TAOS_VGROUP_HASH_INFO *vgHash;
} TAOS_DB_ROUTE_INFO; } TAOS_DB_ROUTE_INFO;
typedef struct TAOS_STMT_OPTIONS {
int64_t reqId;
bool singleStbInsert;
bool singleTableBindOnce;
} TAOS_STMT_OPTIONS;
DLL_EXPORT void taos_cleanup(void); DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT setConfRet taos_set_config(const char *config); DLL_EXPORT setConfRet taos_set_config(const char *config);
@ -162,6 +168,7 @@ DLL_EXPORT const char *taos_data_type(int type);
DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos);
DLL_EXPORT TAOS_STMT *taos_stmt_init_with_reqid(TAOS *taos, int64_t reqid); DLL_EXPORT TAOS_STMT *taos_stmt_init_with_reqid(TAOS *taos, int64_t reqid);
DLL_EXPORT TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS* options);
DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length);
DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags); DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_MULTI_BIND *tags);
DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name);

2
include/common/tcommon.h
View File

@ -240,7 +240,7 @@ typedef struct SDataBlockInfo {
} SDataBlockInfo; } SDataBlockInfo;
typedef struct SSDataBlock { typedef struct SSDataBlock {
SColumnDataAgg** pBlockAgg; SColumnDataAgg* pBlockAgg;
SArray* pDataBlock; // SArray<SColumnInfoData> SArray* pDataBlock; // SArray<SColumnInfoData>
SDataBlockInfo info; SDataBlockInfo info;
} SSDataBlock; } SSDataBlock;

4
include/common/tdatablock.h
View File

@ -102,7 +102,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u
return false; return false;
} }
if (pColAgg != NULL) { if (pColAgg != NULL && pColAgg->colId != -1) {
if (pColAgg->numOfNull == totalRows) { if (pColAgg->numOfNull == totalRows) {
ASSERT(pColumnInfoData->nullbitmap == NULL); ASSERT(pColumnInfoData->nullbitmap == NULL);
return true; return true;
@ -282,6 +282,8 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* p
void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList); void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList);
void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

8
include/common/tdataformat.h
View File

@ -370,6 +370,14 @@ int32_t tDecompressData(void *input, // input
int32_t tCompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist); int32_t tCompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist);
int32_t tDecompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist); int32_t tDecompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist);
typedef struct {
int32_t columnId;
int32_t type;
TAOS_MULTI_BIND *bind;
} SBindInfo;
int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema,
SArray *rowArray);
#endif #endif
#ifdef __cplusplus #ifdef __cplusplus

1
include/common/tglobal.h
View File

@ -86,6 +86,7 @@ extern int32_t tsNumOfQnodeFetchThreads;
extern int32_t tsNumOfSnodeStreamThreads; extern int32_t tsNumOfSnodeStreamThreads;
extern int32_t tsNumOfSnodeWriteThreads; extern int32_t tsNumOfSnodeWriteThreads;
extern int64_t tsRpcQueueMemoryAllowed; extern int64_t tsRpcQueueMemoryAllowed;
extern int32_t tsRetentionSpeedLimitMB;
// sync raft // sync raft
extern int32_t tsElectInterval; extern int32_t tsElectInterval;

40
include/common/tmsg.h
View File

@ -37,6 +37,7 @@ extern "C" {
#define TD_MSG_NUMBER_ #define TD_MSG_NUMBER_
#undef TD_MSG_DICT_ #undef TD_MSG_DICT_
#undef TD_MSG_INFO_ #undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_ #undef TD_MSG_RANGE_CODE_
#undef TD_MSG_SEG_CODE_ #undef TD_MSG_SEG_CODE_
#include "tmsgdef.h" #include "tmsgdef.h"
@ -44,6 +45,7 @@ extern "C" {
#undef TD_MSG_NUMBER_ #undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_ #undef TD_MSG_DICT_
#undef TD_MSG_INFO_ #undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_ #undef TD_MSG_RANGE_CODE_
#define TD_MSG_SEG_CODE_ #define TD_MSG_SEG_CODE_
#include "tmsgdef.h" #include "tmsgdef.h"
@ -51,6 +53,7 @@ extern "C" {
#undef TD_MSG_NUMBER_ #undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_ #undef TD_MSG_DICT_
#undef TD_MSG_INFO_ #undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_SEG_CODE_ #undef TD_MSG_SEG_CODE_
#undef TD_MSG_RANGE_CODE_ #undef TD_MSG_RANGE_CODE_
#include "tmsgdef.h" #include "tmsgdef.h"
@ -190,6 +193,7 @@ typedef enum _mgmt_table {
#define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6 #define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6
#define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7 #define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7
#define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8 #define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8
#define TSDB_ALTER_USER_CREATEDB 0x9
#define TSDB_KILL_MSG_LEN 30 #define TSDB_KILL_MSG_LEN 30
@ -1044,11 +1048,18 @@ int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq
int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq); int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
typedef struct { typedef struct {
int8_t alterType; int8_t alterType;
int8_t superUser; int8_t superUser;
int8_t sysInfo; int8_t sysInfo;
int8_t enable; int8_t enable;
int8_t isView; int8_t isView;
union {
uint8_t flag;
struct {
uint8_t createdb : 1;
uint8_t reserve : 7;
};
};
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_USET_PASSWORD_LEN]; char pass[TSDB_USET_PASSWORD_LEN];
char objname[TSDB_DB_FNAME_LEN]; // db or topic char objname[TSDB_DB_FNAME_LEN]; // db or topic
@ -2117,6 +2128,7 @@ typedef struct {
int8_t precision; int8_t precision;
int8_t compressed; int8_t compressed;
int8_t streamBlockType; int8_t streamBlockType;
int32_t payloadLen;
int32_t compLen; int32_t compLen;
int32_t numOfBlocks; int32_t numOfBlocks;
int64_t numOfRows; // from int32_t change to int64_t int64_t numOfRows; // from int32_t change to int64_t
@ -2129,6 +2141,14 @@ typedef struct {
char data[]; char data[];
} SRetrieveTableRsp; } SRetrieveTableRsp;
#define PAYLOAD_PREFIX_LEN ((sizeof(int32_t)) << 1)
#define SET_PAYLOAD_LEN(_p, _compLen, _fullLen) \
do { \
((int32_t*)(_p))[0] = (_compLen); \
((int32_t*)(_p))[1] = (_fullLen); \
} while (0);
typedef struct { typedef struct {
int64_t version; int64_t version;
int64_t numOfRows; int64_t numOfRows;
@ -2145,6 +2165,7 @@ typedef struct {
int8_t compressed; int8_t compressed;
int32_t compLen; int32_t compLen;
int32_t numOfRows; int32_t numOfRows;
int32_t fullLen;
char data[]; char data[];
} SRetrieveMetaTableRsp; } SRetrieveMetaTableRsp;
@ -2500,6 +2521,7 @@ typedef struct SSubQueryMsg {
int8_t taskType; int8_t taskType;
int8_t explain; int8_t explain;
int8_t needFetch; int8_t needFetch;
int8_t compress;
uint32_t sqlLen; uint32_t sqlLen;
char* sql; char* sql;
uint32_t msgLen; uint32_t msgLen;
@ -3472,9 +3494,9 @@ typedef struct SVUpdateCheckpointInfoReq {
int64_t checkpointVer; int64_t checkpointVer;
int64_t checkpointTs; int64_t checkpointTs;
int32_t transId; int32_t transId;
int8_t dropRelHTask; int64_t hStreamId; // add encode/decode
int64_t hStreamId;
int64_t hTaskId; int64_t hTaskId;
int8_t dropRelHTask;
} SVUpdateCheckpointInfoReq; } SVUpdateCheckpointInfoReq;
typedef struct { typedef struct {
@ -3627,10 +3649,6 @@ typedef struct {
int32_t taskId; int32_t taskId;
} SVPauseStreamTaskReq, SVResetStreamTaskReq; } SVPauseStreamTaskReq, SVResetStreamTaskReq;
typedef struct {
int8_t reserved;
} SVPauseStreamTaskRsp;
typedef struct { typedef struct {
char name[TSDB_STREAM_FNAME_LEN]; char name[TSDB_STREAM_FNAME_LEN];
int8_t igNotExists; int8_t igNotExists;

23
include/common/tmsgdef.h
View File

@ -17,6 +17,7 @@
#if 0 #if 0
#undef TD_MSG_INFO_ #undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_NUMBER_ #undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_ #undef TD_MSG_DICT_
#undef TD_MSG_SEG_CODE_ #undef TD_MSG_SEG_CODE_
@ -76,6 +77,23 @@
#define TD_CLOSE_MSG_SEG(TYPE) #define TD_CLOSE_MSG_SEG(TYPE)
enum { enum {
#elif defined(TD_MSG_TYPE_INFO_)
typedef struct {
const char *name;
const char *rspName;
int32_t type;
int32_t rspType;
} SMsgTypeInfo;
#undef TD_NEW_MSG_SEG
#undef TD_DEF_MSG_TYPE
#undef TD_CLOSE_MSG_SEG
#define TD_NEW_MSG_SEG(TYPE)
#define TD_DEF_MSG_TYPE(TYPE, MSG, REQ, RSP) { #TYPE, #TYPE "_RSP", TYPE, TYPE##_RSP },
#define TD_CLOSE_MSG_SEG(TYPE)
SMsgTypeInfo tMsgTypeInfo[] = {
#else #else
#undef TD_NEW_MSG_SEG #undef TD_NEW_MSG_SEG
@ -207,9 +225,9 @@
TD_DEF_MSG_TYPE(TDMT_MND_RESTORE_DNODE, "restore-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RESTORE_DNODE, "restore-dnode", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_PAUSE_STREAM, "pause-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_PAUSE_STREAM, "pause-stream", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_UPDATE_CHKPT_EVT, "stream-update-chkpt-evt", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, "stream-checkpoint-remain", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHKPT_REPORT, "stream-chkpt-report", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL)
@ -372,6 +390,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_UNUSED, "vnd-stream-unused", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_UNUSED, "vnd-stream-unused", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)

1
include/common/trow.h
View File

@ -289,7 +289,6 @@ int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValT
*/ */
static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) { static FORCE_INLINE void tdSRowInit(SRowBuilder *pBuilder, int16_t sver) {
pBuilder->rowType = pBuilder->rowType;
pBuilder->sver = sver; pBuilder->sver = sver;
} }
int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen); int32_t tdSRowSetInfo(SRowBuilder *pBuilder, int32_t nCols, int32_t nBoundCols, int32_t flen);

751
include/common/ttokendef.h
View File

@ -16,381 +16,382 @@
#ifndef _TD_COMMON_TOKEN_H_ #ifndef _TD_COMMON_TOKEN_H_
#define _TD_COMMON_TOKEN_H_ #define _TD_COMMON_TOKEN_H_
#define TK_OR 1 #define TK_OR 1
#define TK_AND 2 #define TK_AND 2
#define TK_UNION 3 #define TK_UNION 3
#define TK_ALL 4 #define TK_ALL 4
#define TK_MINUS 5 #define TK_MINUS 5
#define TK_EXCEPT 6 #define TK_EXCEPT 6
#define TK_INTERSECT 7 #define TK_INTERSECT 7
#define TK_NK_BITAND 8 #define TK_NK_BITAND 8
#define TK_NK_BITOR 9 #define TK_NK_BITOR 9
#define TK_NK_LSHIFT 10 #define TK_NK_LSHIFT 10
#define TK_NK_RSHIFT 11 #define TK_NK_RSHIFT 11
#define TK_NK_PLUS 12 #define TK_NK_PLUS 12
#define TK_NK_MINUS 13 #define TK_NK_MINUS 13
#define TK_NK_STAR 14 #define TK_NK_STAR 14
#define TK_NK_SLASH 15 #define TK_NK_SLASH 15
#define TK_NK_REM 16 #define TK_NK_REM 16
#define TK_NK_CONCAT 17 #define TK_NK_CONCAT 17
#define TK_CREATE 18 #define TK_CREATE 18
#define TK_ACCOUNT 19 #define TK_ACCOUNT 19
#define TK_NK_ID 20 #define TK_NK_ID 20
#define TK_PASS 21 #define TK_PASS 21
#define TK_NK_STRING 22 #define TK_NK_STRING 22
#define TK_ALTER 23 #define TK_ALTER 23
#define TK_PPS 24 #define TK_PPS 24
#define TK_TSERIES 25 #define TK_TSERIES 25
#define TK_STORAGE 26 #define TK_STORAGE 26
#define TK_STREAMS 27 #define TK_STREAMS 27
#define TK_QTIME 28 #define TK_QTIME 28
#define TK_DBS 29 #define TK_DBS 29
#define TK_USERS 30 #define TK_USERS 30
#define TK_CONNS 31 #define TK_CONNS 31
#define TK_STATE 32 #define TK_STATE 32
#define TK_NK_COMMA 33 #define TK_NK_COMMA 33
#define TK_HOST 34 #define TK_HOST 34
#define TK_USER 35 #define TK_USER 35
#define TK_ENABLE 36 #define TK_ENABLE 36
#define TK_NK_INTEGER 37 #define TK_NK_INTEGER 37
#define TK_SYSINFO 38 #define TK_SYSINFO 38
#define TK_ADD 39 #define TK_CREATEDB 39
#define TK_DROP 40 #define TK_ADD 40
#define TK_GRANT 41 #define TK_DROP 41
#define TK_ON 42 #define TK_GRANT 42
#define TK_TO 43 #define TK_ON 43
#define TK_REVOKE 44 #define TK_TO 44
#define TK_FROM 45 #define TK_REVOKE 45
#define TK_SUBSCRIBE 46 #define TK_FROM 46
#define TK_READ 47 #define TK_SUBSCRIBE 47
#define TK_WRITE 48 #define TK_READ 48
#define TK_NK_DOT 49 #define TK_WRITE 49
#define TK_WITH 50 #define TK_NK_DOT 50
#define TK_ENCRYPT_KEY 51 #define TK_WITH 51
#define TK_DNODE 52 #define TK_ENCRYPT_KEY 52
#define TK_PORT 53 #define TK_DNODE 53
#define TK_DNODES 54 #define TK_PORT 54
#define TK_RESTORE 55 #define TK_DNODES 55
#define TK_NK_IPTOKEN 56 #define TK_RESTORE 56
#define TK_FORCE 57 #define TK_NK_IPTOKEN 57
#define TK_UNSAFE 58 #define TK_FORCE 58
#define TK_CLUSTER 59 #define TK_UNSAFE 59
#define TK_LOCAL 60 #define TK_CLUSTER 60
#define TK_QNODE 61 #define TK_LOCAL 61
#define TK_BNODE 62 #define TK_QNODE 62
#define TK_SNODE 63 #define TK_BNODE 63
#define TK_MNODE 64 #define TK_SNODE 64
#define TK_VNODE 65 #define TK_MNODE 65
#define TK_DATABASE 66 #define TK_VNODE 66
#define TK_USE 67 #define TK_DATABASE 67
#define TK_FLUSH 68 #define TK_USE 68
#define TK_TRIM 69 #define TK_FLUSH 69
#define TK_S3MIGRATE 70 #define TK_TRIM 70
#define TK_COMPACT 71 #define TK_S3MIGRATE 71
#define TK_IF 72 #define TK_COMPACT 72
#define TK_NOT 73 #define TK_IF 73
#define TK_EXISTS 74 #define TK_NOT 74
#define TK_BUFFER 75 #define TK_EXISTS 75
#define TK_CACHEMODEL 76 #define TK_BUFFER 76
#define TK_CACHESIZE 77 #define TK_CACHEMODEL 77
#define TK_COMP 78 #define TK_CACHESIZE 78
#define TK_DURATION 79 #define TK_COMP 79
#define TK_NK_VARIABLE 80 #define TK_DURATION 80
#define TK_MAXROWS 81 #define TK_NK_VARIABLE 81
#define TK_MINROWS 82 #define TK_MAXROWS 82
#define TK_KEEP 83 #define TK_MINROWS 83
#define TK_PAGES 84 #define TK_KEEP 84
#define TK_PAGESIZE 85 #define TK_PAGES 85
#define TK_TSDB_PAGESIZE 86 #define TK_PAGESIZE 86
#define TK_PRECISION 87 #define TK_TSDB_PAGESIZE 87
#define TK_REPLICA 88 #define TK_PRECISION 88
#define TK_VGROUPS 89 #define TK_REPLICA 89
#define TK_SINGLE_STABLE 90 #define TK_VGROUPS 90
#define TK_RETENTIONS 91 #define TK_SINGLE_STABLE 91
#define TK_SCHEMALESS 92 #define TK_RETENTIONS 92
#define TK_WAL_LEVEL 93 #define TK_SCHEMALESS 93
#define TK_WAL_FSYNC_PERIOD 94 #define TK_WAL_LEVEL 94
#define TK_WAL_RETENTION_PERIOD 95 #define TK_WAL_FSYNC_PERIOD 95
#define TK_WAL_RETENTION_SIZE 96 #define TK_WAL_RETENTION_PERIOD 96
#define TK_WAL_ROLL_PERIOD 97 #define TK_WAL_RETENTION_SIZE 97
#define TK_WAL_SEGMENT_SIZE 98 #define TK_WAL_ROLL_PERIOD 98
#define TK_STT_TRIGGER 99 #define TK_WAL_SEGMENT_SIZE 99
#define TK_TABLE_PREFIX 100 #define TK_STT_TRIGGER 100
#define TK_TABLE_SUFFIX 101 #define TK_TABLE_PREFIX 101
#define TK_S3_CHUNKSIZE 102 #define TK_TABLE_SUFFIX 102
#define TK_S3_KEEPLOCAL 103 #define TK_S3_CHUNKSIZE 103
#define TK_S3_COMPACT 104 #define TK_S3_KEEPLOCAL 104
#define TK_KEEP_TIME_OFFSET 105 #define TK_S3_COMPACT 105
#define TK_ENCRYPT_ALGORITHM 106 #define TK_KEEP_TIME_OFFSET 106
#define TK_NK_COLON 107 #define TK_ENCRYPT_ALGORITHM 107
#define TK_BWLIMIT 108 #define TK_NK_COLON 108
#define TK_START 109 #define TK_BWLIMIT 109
#define TK_TIMESTAMP 110 #define TK_START 110
#define TK_END 111 #define TK_TIMESTAMP 111
#define TK_TABLE 112 #define TK_END 112
#define TK_NK_LP 113 #define TK_TABLE 113
#define TK_NK_RP 114 #define TK_NK_LP 114
#define TK_STABLE 115 #define TK_NK_RP 115
#define TK_COLUMN 116 #define TK_STABLE 116
#define TK_MODIFY 117 #define TK_COLUMN 117
#define TK_RENAME 118 #define TK_MODIFY 118
#define TK_TAG 119 #define TK_RENAME 119
#define TK_SET 120 #define TK_TAG 120
#define TK_NK_EQ 121 #define TK_SET 121
#define TK_USING 122 #define TK_NK_EQ 122
#define TK_TAGS 123 #define TK_USING 123
#define TK_BOOL 124 #define TK_TAGS 124
#define TK_TINYINT 125 #define TK_BOOL 125
#define TK_SMALLINT 126 #define TK_TINYINT 126
#define TK_INT 127 #define TK_SMALLINT 127
#define TK_INTEGER 128 #define TK_INT 128
#define TK_BIGINT 129 #define TK_INTEGER 129
#define TK_FLOAT 130 #define TK_BIGINT 130
#define TK_DOUBLE 131 #define TK_FLOAT 131
#define TK_BINARY 132 #define TK_DOUBLE 132
#define TK_NCHAR 133 #define TK_BINARY 133
#define TK_UNSIGNED 134 #define TK_NCHAR 134
#define TK_JSON 135 #define TK_UNSIGNED 135
#define TK_VARCHAR 136 #define TK_JSON 136
#define TK_MEDIUMBLOB 137 #define TK_VARCHAR 137
#define TK_BLOB 138 #define TK_MEDIUMBLOB 138
#define TK_VARBINARY 139 #define TK_BLOB 139
#define TK_GEOMETRY 140 #define TK_VARBINARY 140
#define TK_DECIMAL 141 #define TK_GEOMETRY 141
#define TK_COMMENT 142 #define TK_DECIMAL 142
#define TK_MAX_DELAY 143 #define TK_COMMENT 143
#define TK_WATERMARK 144 #define TK_MAX_DELAY 144
#define TK_ROLLUP 145 #define TK_WATERMARK 145
#define TK_TTL 146 #define TK_ROLLUP 146
#define TK_SMA 147 #define TK_TTL 147
#define TK_DELETE_MARK 148 #define TK_SMA 148
#define TK_FIRST 149 #define TK_DELETE_MARK 149
#define TK_LAST 150 #define TK_FIRST 150
#define TK_SHOW 151 #define TK_LAST 151
#define TK_PRIVILEGES 152 #define TK_SHOW 152
#define TK_DATABASES 153 #define TK_PRIVILEGES 153
#define TK_TABLES 154 #define TK_DATABASES 154
#define TK_STABLES 155 #define TK_TABLES 155
#define TK_MNODES 156 #define TK_STABLES 156
#define TK_QNODES 157 #define TK_MNODES 157
#define TK_ARBGROUPS 158 #define TK_QNODES 158
#define TK_FUNCTIONS 159 #define TK_ARBGROUPS 159
#define TK_INDEXES 160 #define TK_FUNCTIONS 160
#define TK_ACCOUNTS 161 #define TK_INDEXES 161
#define TK_APPS 162 #define TK_ACCOUNTS 162
#define TK_CONNECTIONS 163 #define TK_APPS 163
#define TK_LICENCES 164 #define TK_CONNECTIONS 164
#define TK_GRANTS 165 #define TK_LICENCES 165
#define TK_FULL 166 #define TK_GRANTS 166
#define TK_LOGS 167 #define TK_FULL 167
#define TK_MACHINES 168 #define TK_LOGS 168
#define TK_ENCRYPTIONS 169 #define TK_MACHINES 169
#define TK_QUERIES 170 #define TK_ENCRYPTIONS 170
#define TK_SCORES 171 #define TK_QUERIES 171
#define TK_TOPICS 172 #define TK_SCORES 172
#define TK_VARIABLES 173 #define TK_TOPICS 173
#define TK_BNODES 174 #define TK_VARIABLES 174
#define TK_SNODES 175 #define TK_BNODES 175
#define TK_TRANSACTIONS 176 #define TK_SNODES 176
#define TK_DISTRIBUTED 177 #define TK_TRANSACTIONS 177
#define TK_CONSUMERS 178 #define TK_DISTRIBUTED 178
#define TK_SUBSCRIPTIONS 179 #define TK_CONSUMERS 179
#define TK_VNODES 180 #define TK_SUBSCRIPTIONS 180
#define TK_ALIVE 181 #define TK_VNODES 181
#define TK_VIEWS 182 #define TK_ALIVE 182
#define TK_VIEW 183 #define TK_VIEWS 183
#define TK_COMPACTS 184 #define TK_VIEW 184
#define TK_NORMAL 185 #define TK_COMPACTS 185
#define TK_CHILD 186 #define TK_NORMAL 186
#define TK_LIKE 187 #define TK_CHILD 187
#define TK_TBNAME 188 #define TK_LIKE 188
#define TK_QTAGS 189 #define TK_TBNAME 189
#define TK_AS 190 #define TK_QTAGS 190
#define TK_SYSTEM 191 #define TK_AS 191
#define TK_TSMA 192 #define TK_SYSTEM 192
#define TK_INTERVAL 193 #define TK_TSMA 193
#define TK_RECURSIVE 194 #define TK_INTERVAL 194
#define TK_TSMAS 195 #define TK_RECURSIVE 195
#define TK_FUNCTION 196 #define TK_TSMAS 196
#define TK_INDEX 197 #define TK_FUNCTION 197
#define TK_COUNT 198 #define TK_INDEX 198
#define TK_LAST_ROW 199 #define TK_COUNT 199
#define TK_META 200 #define TK_LAST_ROW 200
#define TK_ONLY 201 #define TK_META 201
#define TK_TOPIC 202 #define TK_ONLY 202
#define TK_CONSUMER 203 #define TK_TOPIC 203
#define TK_GROUP 204 #define TK_CONSUMER 204
#define TK_DESC 205 #define TK_GROUP 205
#define TK_DESCRIBE 206 #define TK_DESC 206
#define TK_RESET 207 #define TK_DESCRIBE 207
#define TK_QUERY 208 #define TK_RESET 208
#define TK_CACHE 209 #define TK_QUERY 209
#define TK_EXPLAIN 210 #define TK_CACHE 210
#define TK_ANALYZE 211 #define TK_EXPLAIN 211
#define TK_VERBOSE 212 #define TK_ANALYZE 212
#define TK_NK_BOOL 213 #define TK_VERBOSE 213
#define TK_RATIO 214 #define TK_NK_BOOL 214
#define TK_NK_FLOAT 215 #define TK_RATIO 215
#define TK_OUTPUTTYPE 216 #define TK_NK_FLOAT 216
#define TK_AGGREGATE 217 #define TK_OUTPUTTYPE 217
#define TK_BUFSIZE 218 #define TK_AGGREGATE 218
#define TK_LANGUAGE 219 #define TK_BUFSIZE 219
#define TK_REPLACE 220 #define TK_LANGUAGE 220
#define TK_STREAM 221 #define TK_REPLACE 221
#define TK_INTO 222 #define TK_STREAM 222
#define TK_PAUSE 223 #define TK_INTO 223
#define TK_RESUME 224 #define TK_PAUSE 224
#define TK_PRIMARY 225 #define TK_RESUME 225
#define TK_KEY 226 #define TK_PRIMARY 226
#define TK_TRIGGER 227 #define TK_KEY 227
#define TK_AT_ONCE 228 #define TK_TRIGGER 228
#define TK_WINDOW_CLOSE 229 #define TK_AT_ONCE 229
#define TK_IGNORE 230 #define TK_WINDOW_CLOSE 230
#define TK_EXPIRED 231 #define TK_IGNORE 231
#define TK_FILL_HISTORY 232 #define TK_EXPIRED 232
#define TK_UPDATE 233 #define TK_FILL_HISTORY 233
#define TK_SUBTABLE 234 #define TK_UPDATE 234
#define TK_UNTREATED 235 #define TK_SUBTABLE 235
#define TK_KILL 236 #define TK_UNTREATED 236
#define TK_CONNECTION 237 #define TK_KILL 237
#define TK_TRANSACTION 238 #define TK_CONNECTION 238
#define TK_BALANCE 239 #define TK_TRANSACTION 239
#define TK_VGROUP 240 #define TK_BALANCE 240
#define TK_LEADER 241 #define TK_VGROUP 241
#define TK_MERGE 242 #define TK_LEADER 242
#define TK_REDISTRIBUTE 243 #define TK_MERGE 243
#define TK_SPLIT 244 #define TK_REDISTRIBUTE 244
#define TK_DELETE 245 #define TK_SPLIT 245
#define TK_INSERT 246 #define TK_DELETE 246
#define TK_NK_BIN 247 #define TK_INSERT 247
#define TK_NK_HEX 248 #define TK_NK_BIN 248
#define TK_NULL 249 #define TK_NK_HEX 249
#define TK_NK_QUESTION 250 #define TK_NULL 250
#define TK_NK_ALIAS 251 #define TK_NK_QUESTION 251
#define TK_NK_ARROW 252 #define TK_NK_ALIAS 252
#define TK_ROWTS 253 #define TK_NK_ARROW 253
#define TK_QSTART 254 #define TK_ROWTS 254
#define TK_QEND 255 #define TK_QSTART 255
#define TK_QDURATION 256 #define TK_QEND 256
#define TK_WSTART 257 #define TK_QDURATION 257
#define TK_WEND 258 #define TK_WSTART 258
#define TK_WDURATION 259 #define TK_WEND 259
#define TK_IROWTS 260 #define TK_WDURATION 260
#define TK_ISFILLED 261 #define TK_IROWTS 261
#define TK_CAST 262 #define TK_ISFILLED 262
#define TK_NOW 263 #define TK_CAST 263
#define TK_TODAY 264 #define TK_NOW 264
#define TK_TIMEZONE 265 #define TK_TODAY 265
#define TK_CLIENT_VERSION 266 #define TK_TIMEZONE 266
#define TK_SERVER_VERSION 267 #define TK_CLIENT_VERSION 267
#define TK_SERVER_STATUS 268 #define TK_SERVER_VERSION 268
#define TK_CURRENT_USER 269 #define TK_SERVER_STATUS 269
#define TK_CASE 270 #define TK_CURRENT_USER 270
#define TK_WHEN 271 #define TK_CASE 271
#define TK_THEN 272 #define TK_WHEN 272
#define TK_ELSE 273 #define TK_THEN 273
#define TK_BETWEEN 274 #define TK_ELSE 274
#define TK_IS 275 #define TK_BETWEEN 275
#define TK_NK_LT 276 #define TK_IS 276
#define TK_NK_GT 277 #define TK_NK_LT 277
#define TK_NK_LE 278 #define TK_NK_GT 278
#define TK_NK_GE 279 #define TK_NK_LE 279
#define TK_NK_NE 280 #define TK_NK_GE 280
#define TK_MATCH 281 #define TK_NK_NE 281
#define TK_NMATCH 282 #define TK_MATCH 282
#define TK_CONTAINS 283 #define TK_NMATCH 283
#define TK_IN 284 #define TK_CONTAINS 284
#define TK_JOIN 285 #define TK_IN 285
#define TK_INNER 286 #define TK_JOIN 286
#define TK_LEFT 287 #define TK_INNER 287
#define TK_RIGHT 288 #define TK_LEFT 288
#define TK_OUTER 289 #define TK_RIGHT 289
#define TK_SEMI 290 #define TK_OUTER 290
#define TK_ANTI 291 #define TK_SEMI 291
#define TK_ASOF 292 #define TK_ANTI 292
#define TK_WINDOW 293 #define TK_ASOF 293
#define TK_WINDOW_OFFSET 294 #define TK_WINDOW 294
#define TK_JLIMIT 295 #define TK_WINDOW_OFFSET 295
#define TK_SELECT 296 #define TK_JLIMIT 296
#define TK_NK_HINT 297 #define TK_SELECT 297
#define TK_DISTINCT 298 #define TK_NK_HINT 298
#define TK_WHERE 299 #define TK_DISTINCT 299
#define TK_PARTITION 300 #define TK_WHERE 300
#define TK_BY 301 #define TK_PARTITION 301
#define TK_SESSION 302 #define TK_BY 302
#define TK_STATE_WINDOW 303 #define TK_SESSION 303
#define TK_EVENT_WINDOW 304 #define TK_STATE_WINDOW 304
#define TK_COUNT_WINDOW 305 #define TK_EVENT_WINDOW 305
#define TK_SLIDING 306 #define TK_COUNT_WINDOW 306
#define TK_FILL 307 #define TK_SLIDING 307
#define TK_VALUE 308 #define TK_FILL 308
#define TK_VALUE_F 309 #define TK_VALUE 309
#define TK_NONE 310 #define TK_VALUE_F 310
#define TK_PREV 311 #define TK_NONE 311
#define TK_NULL_F 312 #define TK_PREV 312
#define TK_LINEAR 313 #define TK_NULL_F 313
#define TK_NEXT 314 #define TK_LINEAR 314
#define TK_HAVING 315 #define TK_NEXT 315
#define TK_RANGE 316 #define TK_HAVING 316
#define TK_EVERY 317 #define TK_RANGE 317
#define TK_ORDER 318 #define TK_EVERY 318
#define TK_SLIMIT 319 #define TK_ORDER 319
#define TK_SOFFSET 320 #define TK_SLIMIT 320
#define TK_LIMIT 321 #define TK_SOFFSET 321
#define TK_OFFSET 322 #define TK_LIMIT 322
#define TK_ASC 323 #define TK_OFFSET 323
#define TK_NULLS 324 #define TK_ASC 324
#define TK_ABORT 325 #define TK_NULLS 325
#define TK_AFTER 326 #define TK_ABORT 326
#define TK_ATTACH 327 #define TK_AFTER 327
#define TK_BEFORE 328 #define TK_ATTACH 328
#define TK_BEGIN 329 #define TK_BEFORE 329
#define TK_BITAND 330 #define TK_BEGIN 330
#define TK_BITNOT 331 #define TK_BITAND 331
#define TK_BITOR 332 #define TK_BITNOT 332
#define TK_BLOCKS 333 #define TK_BITOR 333
#define TK_CHANGE 334 #define TK_BLOCKS 334
#define TK_COMMA 335 #define TK_CHANGE 335
#define TK_CONCAT 336 #define TK_COMMA 336
#define TK_CONFLICT 337 #define TK_CONCAT 337
#define TK_COPY 338 #define TK_CONFLICT 338
#define TK_DEFERRED 339 #define TK_COPY 339
#define TK_DELIMITERS 340 #define TK_DEFERRED 340
#define TK_DETACH 341 #define TK_DELIMITERS 341
#define TK_DIVIDE 342 #define TK_DETACH 342
#define TK_DOT 343 #define TK_DIVIDE 343
#define TK_EACH 344 #define TK_DOT 344
#define TK_FAIL 345 #define TK_EACH 345
#define TK_FILE 346 #define TK_FAIL 346
#define TK_FOR 347 #define TK_FILE 347
#define TK_GLOB 348 #define TK_FOR 348
#define TK_ID 349 #define TK_GLOB 349
#define TK_IMMEDIATE 350 #define TK_ID 350
#define TK_IMPORT 351 #define TK_IMMEDIATE 351
#define TK_INITIALLY 352 #define TK_IMPORT 352
#define TK_INSTEAD 353 #define TK_INITIALLY 353
#define TK_ISNULL 354 #define TK_INSTEAD 354
#define TK_MODULES 355 #define TK_ISNULL 355
#define TK_NK_BITNOT 356 #define TK_MODULES 356
#define TK_NK_SEMI 357 #define TK_NK_BITNOT 357
#define TK_NOTNULL 358 #define TK_NK_SEMI 358
#define TK_OF 359 #define TK_NOTNULL 359
#define TK_PLUS 360 #define TK_OF 360
#define TK_PRIVILEGE 361 #define TK_PLUS 361
#define TK_RAISE 362 #define TK_PRIVILEGE 362
#define TK_RESTRICT 363 #define TK_RAISE 363
#define TK_ROW 364 #define TK_RESTRICT 364
#define TK_STAR 365 #define TK_ROW 365
#define TK_STATEMENT 366 #define TK_STAR 366
#define TK_STRICT 367 #define TK_STATEMENT 367
#define TK_STRING 368 #define TK_STRICT 368
#define TK_TIMES 369 #define TK_STRING 369
#define TK_VALUES 370 #define TK_TIMES 370
#define TK_VARIABLE 371 #define TK_VALUES 371
#define TK_WAL 372 #define TK_VARIABLE 372
#define TK_ENCODE 373 #define TK_WAL 373
#define TK_COMPRESS 374 #define TK_ENCODE 374
#define TK_LEVEL 375 #define TK_COMPRESS 375
#define TK_LEVEL 376
#define TK_NK_SPACE 600 #define TK_NK_SPACE 600
#define TK_NK_COMMENT 601 #define TK_NK_COMMENT 601

3
include/dnode/vnode/tqCommon.h
View File

@ -28,6 +28,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe
int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen, int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen,
bool isLeader, bool restored); bool isLeader, bool restored);
@ -42,7 +43,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg);
int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode);
int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen);
int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta);
void tqSetRestoreVersionInfo(SStreamTask* pTask); void tqSetRestoreVersionInfo(SStreamTask* pTask);
int32_t tqExpandStreamTask(SStreamTask* pTask);
#endif // TDENGINE_TQ_COMMON_H #endif // TDENGINE_TQ_COMMON_H

3
include/libs/executor/dataSinkMgt.h
View File

@ -55,6 +55,7 @@ typedef struct SDataSinkStat {
} SDataSinkStat; } SDataSinkStat;
typedef struct SDataSinkMgtCfg { typedef struct SDataSinkMgtCfg {
int8_t compress;
uint32_t maxDataBlockNum; // todo: this should be numOfRows? uint32_t maxDataBlockNum; // todo: this should be numOfRows?
uint32_t maxDataBlockNumPerQuery; uint32_t maxDataBlockNumPerQuery;
} SDataSinkMgtCfg; } SDataSinkMgtCfg;
@ -104,7 +105,7 @@ void dsReset(DataSinkHandle handle);
* @param handle * @param handle
* @param pLen data length * @param pLen data length
*/ */
void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd); void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd);
/** /**
* Get data, the caller needs to allocate data memory. * Get data, the caller needs to allocate data memory.

5
include/libs/executor/executor.h
View File

@ -139,8 +139,9 @@ void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam);
* @param qId * @param qId
* @return * @return
*/ */
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pSubplan,
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model); qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, int8_t compressResult, char* sql,
EOPTR_EXEC_MODEL model);
/** /**
* *

2
include/libs/executor/storageapi.h
View File

@ -410,7 +410,7 @@ typedef struct SStateStore {
void (*streamFileStateClear)(struct SStreamFileState* pFileState); void (*streamFileStateClear)(struct SStreamFileState* pFileState);
bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); bool (*needClearDiskBuff)(struct SStreamFileState* pFileState);
SStreamState* (*streamStateOpen)(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId, bool specPath, int32_t szPage, int32_t pages);
void (*streamStateClose)(SStreamState* pState, bool remove); void (*streamStateClose)(SStreamState* pState, bool remove);
int32_t (*streamStateBegin)(SStreamState* pState); int32_t (*streamStateBegin)(SStreamState* pState);
int32_t (*streamStateCommit)(SStreamState* pState); int32_t (*streamStateCommit)(SStreamState* pState);

1
include/libs/nodes/cmdnodes.h
View File

@ -271,6 +271,7 @@ typedef struct SAlterUserStmt {
char password[TSDB_USET_PASSWORD_LEN]; char password[TSDB_USET_PASSWORD_LEN];
int8_t enable; int8_t enable;
int8_t sysinfo; int8_t sysinfo;
int8_t createdb;
int32_t numIpRanges; int32_t numIpRanges;
SIpV4Range* pIpRanges; SIpV4Range* pIpRanges;

10
include/libs/parser/parser.h
View File

@ -118,7 +118,11 @@ int32_t qExtractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** p
int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid); int32_t qSetSTableIdForRsma(SNode* pStmt, int64_t uid);
void qCleanupKeywordsTable(); void qCleanupKeywordsTable();
int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo);
int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks);
//int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t tbNum);
int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash); int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash);
int32_t qResetStmtColumns(SArray* pCols, bool deepClear);
int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf); int32_t qResetStmtDataBlock(STableDataCxt* block, bool keepBuf);
int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset); int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool reset);
int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId, int32_t qRebuildStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, uint64_t uid, uint64_t suid, int32_t vgId,
@ -129,8 +133,9 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData
int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx); int32_t qStmtBindParams(SQuery* pQuery, TAOS_MULTI_BIND* pParams, int32_t colIdx);
int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery); int32_t qStmtParseQuerySql(SParseContext* pCxt, SQuery* pQuery);
int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen); int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, STSchema** pTSchema, SBindInfo* pBindInfos);
int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen);
int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx,
int32_t rowNum); int32_t rowNum);
int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtColFields(void* pDataBlock, int32_t* fieldNum, TAOS_FIELD_E** fields);
int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields); int32_t qBuildStmtTagFields(void* pBlock, void* boundTags, int32_t* fieldNum, TAOS_FIELD_E** fields);
@ -160,6 +165,7 @@ SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap); SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
void destoryCatalogReq(SCatalogReq *pCatalogReq); void destoryCatalogReq(SCatalogReq *pCatalogReq);
bool isPrimaryKeyImpl(SNode* pExpr); bool isPrimaryKeyImpl(SNode* pExpr);
int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo);
#ifdef __cplusplus #ifdef __cplusplus
} }

49
include/libs/qcom/query.h
View File

@ -25,6 +25,7 @@ extern "C" {
#include "tarray.h" #include "tarray.h"
#include "thash.h" #include "thash.h"
#include "tlog.h" #include "tlog.h"
#include "tsimplehash.h"
#include "tmsg.h" #include "tmsg.h"
#include "tmsgcb.h" #include "tmsgcb.h"
@ -193,6 +194,27 @@ typedef struct SBoundColInfo {
int32_t numOfBound; int32_t numOfBound;
} SBoundColInfo; } SBoundColInfo;
typedef struct STableColsData {
char tbName[TSDB_TABLE_NAME_LEN];
SArray* aCol;
bool getFromHash;
} STableColsData;
typedef struct STableVgUid {
uint64_t uid;
int32_t vgid;
} STableVgUid;
typedef struct STableBufInfo {
void* pCurBuff;
SArray* pBufList;
int64_t buffUnit;
int64_t buffSize;
int64_t buffIdx;
int64_t buffOffset;
} STableBufInfo;
typedef struct STableDataCxt { typedef struct STableDataCxt {
STableMeta* pMeta; STableMeta* pMeta;
STSchema* pSchema; STSchema* pSchema;
@ -204,6 +226,33 @@ typedef struct STableDataCxt {
bool duplicateTs; bool duplicateTs;
} STableDataCxt; } STableDataCxt;
typedef struct SStbInterlaceInfo {
void* pCatalog;
void* pQuery;
int32_t acctId;
char* dbname;
void* transport;
SEpSet mgmtEpSet;
void* pRequest;
uint64_t requestId;
int64_t requestSelf;
bool tbFromHash;
SHashObj* pVgroupHash;
SArray* pVgroupList;
SSHashObj* pTableHash;
int64_t tbRemainNum;
STableBufInfo tbBuf;
char firstName[TSDB_TABLE_NAME_LEN];
STSchema *pTSchema;
STableDataCxt *pDataCtx;
void *boundTags;
bool tableColsReady;
SArray *pTableCols;
int32_t pTableColsIdx;
} SStbInterlaceInfo;
typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code); typedef int32_t (*__async_send_cb_fn_t)(void* param, SDataBuf* pMsg, int32_t code);
typedef int32_t (*__async_exec_fn_t)(void* param); typedef int32_t (*__async_exec_fn_t)(void* param);

1
include/libs/qworker/qworker.h
View File

@ -61,6 +61,7 @@ typedef struct SQWMsgInfo {
int8_t taskType; int8_t taskType;
int8_t explain; int8_t explain;
int8_t needFetch; int8_t needFetch;
int8_t compressMsg;
} SQWMsgInfo; } SQWMsgInfo;
typedef struct SQWMsg { typedef struct SQWMsg {

2
include/libs/scalar/filter.h
View File

@ -58,7 +58,7 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict
extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar);
extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo); extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo);
extern void filterFreeInfo(SFilterInfo *info); extern void filterFreeInfo(SFilterInfo *info);
extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t numOfCols, int32_t numOfRows); extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pColsAgg, int32_t numOfCols, int32_t numOfRows);
/* condition split interface */ /* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,

10
include/libs/scheduler/scheduler.h
View File

@ -48,16 +48,6 @@ typedef struct SQueryProfileSummary {
uint64_t resultSize; // generated result size in Kb. uint64_t resultSize; // generated result size in Kb.
} SQueryProfileSummary; } SQueryProfileSummary;
typedef struct STaskInfo {
SQueryNodeAddr addr;
SSubQueryMsg* msg;
} STaskInfo;
typedef struct SSchdFetchParam {
void** pData;
int32_t* code;
} SSchdFetchParam;
typedef void (*schedulerExecFp)(SExecResult* pResult, void* param, int32_t code); typedef void (*schedulerExecFp)(SExecResult* pResult, void* param, int32_t code);
typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code); typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code);
typedef bool (*schedulerChkKillFp)(void* param); typedef bool (*schedulerChkKillFp)(void* param);

3
include/libs/stream/streamState.h
View File

@ -29,7 +29,8 @@ extern "C" {
#include "storageapi.h" #include "storageapi.h"
SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId, bool specPath,
int32_t szPage, int32_t pages);
void streamStateClose(SStreamState* pState, bool remove); void streamStateClose(SStreamState* pState, bool remove);
int32_t streamStateBegin(SStreamState* pState); int32_t streamStateBegin(SStreamState* pState);
int32_t streamStateCommit(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState);

14
include/libs/stream/streammsg.h
View File

@ -190,6 +190,20 @@ typedef struct SCheckpointTriggerRsp {
int32_t rspCode; int32_t rspCode;
} SCheckpointTriggerRsp; } SCheckpointTriggerRsp;
typedef struct SCheckpointReport {
int64_t streamId;
int32_t taskId;
int32_t nodeId;
int64_t checkpointId;
int64_t checkpointVer;
int64_t checkpointTs;
int32_t transId;
int8_t dropHTask;
} SCheckpointReport;
int32_t tEncodeStreamTaskChkptReport(SEncoder* pEncoder, const SCheckpointReport* pReq);
int32_t tDecodeStreamTaskChkptReport(SDecoder* pDecoder, SCheckpointReport* pReq);
typedef struct { typedef struct {
SMsgHead head; SMsgHead head;
int64_t streamId; int64_t streamId;

38
include/libs/stream/tstream.h
View File

@ -157,7 +157,8 @@ typedef enum EStreamTaskEvent {
typedef void FTbSink(SStreamTask* pTask, void* vnode, void* data); typedef void FTbSink(SStreamTask* pTask, void* vnode, void* data);
typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data); typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data);
typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); typedef int32_t FTaskBuild(void* ahandle, SStreamTask* pTask, int64_t ver);
typedef int32_t FTaskExpand(SStreamTask* pTask);
typedef struct { typedef struct {
int8_t type; int8_t type;
@ -205,7 +206,6 @@ typedef struct {
typedef struct { typedef struct {
char stbFullName[TSDB_TABLE_FNAME_LEN]; char stbFullName[TSDB_TABLE_FNAME_LEN];
int32_t waitingRspCnt;
SUseDbRsp dbInfo; SUseDbRsp dbInfo;
} STaskDispatcherShuffle; } STaskDispatcherShuffle;
@ -312,15 +312,18 @@ typedef struct SMetaHbInfo SMetaHbInfo;
typedef struct SDispatchMsgInfo { typedef struct SDispatchMsgInfo {
SStreamDispatchReq* pData; // current dispatch data SStreamDispatchReq* pData; // current dispatch data
int8_t dispatchMsgType;
int64_t checkpointId;// checkpoint id msg int8_t dispatchMsgType;
int32_t transId; // transId for current checkpoint int64_t checkpointId; // checkpoint id msg
int16_t msgType; // dispatch msg type int32_t transId; // transId for current checkpoint
int32_t retryCount; // retry send data count int16_t msgType; // dispatch msg type
int64_t startTs; // dispatch start time, record total elapsed time for dispatch int32_t msgId;
SArray* pRetryList; // current dispatch successfully completed node of downstream int64_t startTs; // dispatch start time, record total elapsed time for dispatch
void* pRetryTmr; // used to dispatch data after a given time duration int64_t rspTs; // latest rsp time
void* pRspTmr; // used to dispatch data after a given time duration void* pRetryTmr; // used to dispatch data after a given time duration
TdThreadMutex lock;
int8_t inMonitor;
SArray* pSendInfo; // SArray<SDispatchEntry>
} SDispatchMsgInfo; } SDispatchMsgInfo;
typedef struct STaskQueue { typedef struct STaskQueue {
@ -484,7 +487,8 @@ typedef struct SStreamMeta {
SArray* pTaskList; // SArray<STaskId*> SArray* pTaskList; // SArray<STaskId*>
void* ahandle; void* ahandle;
TXN* txn; TXN* txn;
FTaskExpand* expandFunc; FTaskBuild* buildTaskFn;
FTaskExpand* expandTaskFn;
int32_t vgId; int32_t vgId;
int64_t stage; int64_t stage;
int32_t role; int32_t role;
@ -708,8 +712,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st);
// stream task meta // stream task meta
void streamMetaInit(); void streamMetaInit();
void streamMetaCleanup(); void streamMetaCleanup();
SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn,
startComplete_fn_t fn); int32_t vgId, int64_t stage, startComplete_fn_t fn);
void streamMetaClose(SStreamMeta* streamMeta); void streamMetaClose(SStreamMeta* streamMeta);
int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store
int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
@ -734,6 +738,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32
void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs); void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs);
void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId,
int64_t startTs); int64_t startTs);
void streamMetaClearUpdateTaskList(SStreamMeta* pMeta);
void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId);
void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRLock(SStreamMeta* pMeta);
void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta);
void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta);
@ -762,8 +769,7 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI
int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask); int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask);
int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg,
int32_t setCode); int32_t setCode);
int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId, int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask);
int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq); int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq);
SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo(); SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo();

1
include/libs/transport/trpc.h
View File

@ -62,6 +62,7 @@ typedef struct SRpcHandleInfo {
SRpcConnInfo conn; SRpcConnInfo conn;
int8_t forbiddenIp; int8_t forbiddenIp;
int8_t notFreeAhandle; int8_t notFreeAhandle;
int8_t compressed;
} SRpcHandleInfo; } SRpcHandleInfo;
typedef struct SRpcMsg { typedef struct SRpcMsg {

11
include/util/taoserror.h
View File

@ -24,6 +24,14 @@ extern "C" {
// clang-format off // clang-format off
typedef struct {
int32_t val;
const char* str;
const char* macro;
} STaosError;
extern STaosError errors[];
#define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code))))
#define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code))
@ -38,6 +46,7 @@ const char* terrstr();
char* taosGetErrMsgReturn(); char* taosGetErrMsgReturn();
char* taosGetErrMsg(); char* taosGetErrMsg();
int32_t* taosGetErrno(); int32_t* taosGetErrno();
int32_t taosGetErrSize();
#define terrno (*taosGetErrno()) #define terrno (*taosGetErrno())
#define terrMsg (taosGetErrMsg()) #define terrMsg (taosGetErrMsg())
@ -316,7 +325,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) //
#define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B)
#define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C) #define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C)
#define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) // #define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) // unused
#define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E) #define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E)
// #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x
// #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x

4
packaging/tools/install.sh
View File

@ -795,10 +795,10 @@ function is_version_compatible() {
if [ -f ${script_dir}/driver/vercomp.txt ]; then if [ -f ${script_dir}/driver/vercomp.txt ]; then
min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt)
else else
min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) min_compatible_version=$(${script_dir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 5)
fi fi
exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) exist_version=$(${installDir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 3)
vercomp $exist_version "3.0.0.0" vercomp $exist_version "3.0.0.0"
case $? in case $? in
2) 2)

5
source/client/inc/clientInt.h
View File

@ -124,7 +124,6 @@ struct SAppInstInfo {
typedef struct SAppInfo { typedef struct SAppInfo {
int64_t startTime; int64_t startTime;
char appName[TSDB_APP_NAME_LEN]; char appName[TSDB_APP_NAME_LEN];
char* ep;
int32_t pid; int32_t pid;
int32_t numOfThreads; int32_t numOfThreads;
SHashObj* pInstMap; SHashObj* pInstMap;
@ -197,8 +196,10 @@ typedef struct SReqResultInfo {
uint64_t current; uint64_t current;
bool localResultFetched; bool localResultFetched;
bool completed; bool completed;
int32_t precision;
bool convertUcs4; bool convertUcs4;
char* decompBuf;
int32_t decompBufSize;
int32_t precision;
int32_t payloadLen; int32_t payloadLen;
char* convertJson; char* convertJson;
} SReqResultInfo; } SReqResultInfo;

74
source/client/inc/clientStmt.h
View File

@ -40,6 +40,8 @@ typedef enum {
STMT_MAX, STMT_MAX,
} STMT_STATUS; } STMT_STATUS;
#define STMT_TABLE_COLS_NUM 1000
typedef struct SStmtTableCache { typedef struct SStmtTableCache {
STableDataCxt *pDataCtx; STableDataCxt *pDataCtx;
void *boundTags; void *boundTags;
@ -57,6 +59,7 @@ typedef struct SStmtBindInfo {
bool inExecCache; bool inExecCache;
uint64_t tbUid; uint64_t tbUid;
uint64_t tbSuid; uint64_t tbSuid;
int32_t tbVgId;
int32_t sBindRowNum; int32_t sBindRowNum;
int32_t sBindLastIdx; int32_t sBindLastIdx;
int8_t tbType; int8_t tbType;
@ -66,8 +69,15 @@ typedef struct SStmtBindInfo {
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
char stbFName[TSDB_TABLE_FNAME_LEN]; char stbFName[TSDB_TABLE_FNAME_LEN];
SName sname; SName sname;
char statbName[TSDB_TABLE_FNAME_LEN];
} SStmtBindInfo; } SStmtBindInfo;
typedef struct SStmtAsyncParam {
STableColsData *pTbData;
void* pStmt;
} SStmtAsyncParam;
typedef struct SStmtExecInfo { typedef struct SStmtExecInfo {
int32_t affectedRows; int32_t affectedRows;
SRequestObj *pRequest; SRequestObj *pRequest;
@ -77,8 +87,10 @@ typedef struct SStmtExecInfo {
} SStmtExecInfo; } SStmtExecInfo;
typedef struct SStmtSQLInfo { typedef struct SStmtSQLInfo {
bool stbInterlaceMode;
STMT_TYPE type; STMT_TYPE type;
STMT_STATUS status; STMT_STATUS status;
uint64_t suid;
uint64_t runTimes; uint64_t runTimes;
SHashObj *pTableCache; // SHash<SStmtTableCache> SHashObj *pTableCache; // SHash<SStmtTableCache>
SQuery *pQuery; SQuery *pQuery;
@ -88,21 +100,60 @@ typedef struct SStmtSQLInfo {
SStmtQueryResInfo queryRes; SStmtQueryResInfo queryRes;
bool autoCreateTbl; bool autoCreateTbl;
SHashObj *pVgHash; SHashObj *pVgHash;
SBindInfo *pBindInfo;
SStbInterlaceInfo siInfo;
} SStmtSQLInfo; } SStmtSQLInfo;
typedef struct SStmtStatInfo {
int64_t ctgGetTbMetaNum;
int64_t getCacheTbInfo;
int64_t parseSqlNum;
int64_t bindDataNum;
int64_t setTbNameUs;
int64_t bindDataUs1;
int64_t bindDataUs2;
int64_t bindDataUs3;
int64_t bindDataUs4;
int64_t addBatchUs;
int64_t execWaitUs;
int64_t execUseUs;
} SStmtStatInfo;
typedef struct SStmtQNode {
bool restoreTbCols;
STableColsData tblData;
struct SStmtQNode* next;
} SStmtQNode;
typedef struct SStmtQueue {
bool stopQueue;
SStmtQNode* head;
SStmtQNode* tail;
uint64_t qRemainNum;
} SStmtQueue;
typedef struct STscStmt { typedef struct STscStmt {
STscObj *taos; STscObj *taos;
SCatalog *pCatalog; SCatalog *pCatalog;
int32_t affectedRows; int32_t affectedRows;
uint32_t seqId; uint32_t seqId;
uint32_t seqIds[STMT_MAX]; uint32_t seqIds[STMT_MAX];
bool bindThreadInUse;
TdThread bindThread;
TAOS_STMT_OPTIONS options;
bool stbInterlaceMode;
SStmtQueue queue;
SStmtSQLInfo sql; SStmtSQLInfo sql;
SStmtExecInfo exec; SStmtExecInfo exec;
SStmtBindInfo bInfo; SStmtBindInfo bInfo;
int64_t reqid; int64_t reqid;
int32_t errCode; int32_t errCode;
SStmtStatInfo stat;
} STscStmt; } STscStmt;
extern char *gStmtStatusStr[]; extern char *gStmtStatusStr[];
@ -154,13 +205,14 @@ extern char *gStmtStatusStr[];
} while (0) } while (0)
#define STMT_FLOG(param, ...) qFatal("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__)
#define STMT_ELOG_E(param) qError("stmt:%p " param, pStmt) #define STMT_ELOG_E(param) qError("stmt:%p " param, pStmt)
#define STMT_DLOG_E(param) qDebug("stmt:%p " param, pStmt) #define STMT_DLOG_E(param) qDebug("stmt:%p " param, pStmt)
TAOS_STMT *stmtInit(STscObj *taos, int64_t reqid); TAOS_STMT *stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions);
int stmtClose(TAOS_STMT *stmt); int stmtClose(TAOS_STMT *stmt);
int stmtExec(TAOS_STMT *stmt); int stmtExec(TAOS_STMT *stmt);
const char *stmtErrstr(TAOS_STMT *stmt); const char *stmtErrstr(TAOS_STMT *stmt);

3
source/client/src/clientEnv.c
View File

@ -334,6 +334,7 @@ void *createRequest(uint64_t connId, int32_t type, int64_t reqid) {
} }
SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); SSyncQueryParam *interParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam));
if (interParam == NULL) { if (interParam == NULL) {
releaseTscObj(connId);
doDestroyRequest(pRequest); doDestroyRequest(pRequest);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
@ -374,6 +375,7 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) {
taosMemoryFreeClear(pResInfo->fields); taosMemoryFreeClear(pResInfo->fields);
taosMemoryFreeClear(pResInfo->userFields); taosMemoryFreeClear(pResInfo->userFields);
taosMemoryFreeClear(pResInfo->convertJson); taosMemoryFreeClear(pResInfo->convertJson);
taosMemoryFreeClear(pResInfo->decompBuf);
if (pResInfo->convertBuf != NULL) { if (pResInfo->convertBuf != NULL) {
for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { for (int32_t i = 0; i < pResInfo->numOfCols; ++i) {
@ -763,7 +765,6 @@ void taos_init_imp(void) {
clientConnRefPool = taosOpenRef(200, destroyTscObj); clientConnRefPool = taosOpenRef(200, destroyTscObj);
clientReqRefPool = taosOpenRef(40960, doDestroyRequest); clientReqRefPool = taosOpenRef(40960, doDestroyRequest);
// transDestroyBuffer(&conn->readBuf);
taosGetAppName(appInfo.appName, NULL); taosGetAppName(appInfo.appName, NULL);
taosThreadMutexInit(&appInfo.mutex, NULL); taosThreadMutexInit(&appInfo.mutex, NULL);

56
source/client/src/clientImpl.c
View File

@ -1711,10 +1711,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4)
} }
SReqResultInfo* pResInfo = &pRequest->body.resInfo; SReqResultInfo* pResInfo = &pRequest->body.resInfo;
SSchedulerReq req = { SSchedulerReq req = { .syncReq = true, .pFetchRes = (void**)&pResInfo->pData };
.syncReq = true,
.pFetchRes = (void**)&pResInfo->pData,
};
pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req); pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
pResultInfo->numOfRows = 0; pResultInfo->numOfRows = 0;
@ -2065,6 +2063,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
tscError("setResultDataPtr paras error"); tscError("setResultDataPtr paras error");
return TSDB_CODE_TSC_INTERNAL_ERROR; return TSDB_CODE_TSC_INTERNAL_ERROR;
} }
if (numOfRows == 0) { if (numOfRows == 0) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -2195,17 +2194,58 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR
taosMemoryFreeClear(pResultInfo->pRspMsg); taosMemoryFreeClear(pResultInfo->pRspMsg);
pResultInfo->pRspMsg = (const char*)pRsp; pResultInfo->pRspMsg = (const char*)pRsp;
pResultInfo->pData = (void*)pRsp->data;
pResultInfo->numOfRows = htobe64(pRsp->numOfRows); pResultInfo->numOfRows = htobe64(pRsp->numOfRows);
pResultInfo->current = 0; pResultInfo->current = 0;
pResultInfo->completed = (pRsp->completed == 1); pResultInfo->completed = (pRsp->completed == 1);
pResultInfo->payloadLen = htonl(pRsp->compLen);
pResultInfo->precision = pRsp->precision; pResultInfo->precision = pRsp->precision;
// decompress data if needed
int32_t payloadLen = htonl(pRsp->payloadLen);
if (pRsp->compressed) {
if (pResultInfo->decompBuf == NULL) {
pResultInfo->decompBuf = taosMemoryMalloc(payloadLen);
pResultInfo->decompBufSize = payloadLen;
} else {
if (pResultInfo->decompBufSize < payloadLen) {
char* p = taosMemoryRealloc(pResultInfo->decompBuf, payloadLen);
if (p == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tscError("failed to prepare the decompress buffer, size:%d", payloadLen);
return terrno;
}
pResultInfo->decompBuf = p;
pResultInfo->decompBufSize = payloadLen;
}
}
}
if (payloadLen > 0) {
int32_t compLen = *(int32_t*)pRsp->data;
int32_t rawLen = *(int32_t*)(pRsp->data + sizeof(int32_t));
char* pStart = (char*)pRsp->data + sizeof(int32_t) * 2;
if (pRsp->compressed && compLen < rawLen) {
int32_t len = tsDecompressString(pStart, compLen, 1, pResultInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0);
ASSERT(len == rawLen);
pResultInfo->pData = pResultInfo->decompBuf;
pResultInfo->payloadLen = rawLen;
} else {
pResultInfo->pData = pStart;
pResultInfo->payloadLen = htonl(pRsp->compLen);
ASSERT(pRsp->compLen == pRsp->payloadLen);
}
}
// TODO handle the compressed case // TODO handle the compressed case
pResultInfo->totalRows += pResultInfo->numOfRows; pResultInfo->totalRows += pResultInfo->numOfRows;
return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows,
convertUcs4); int32_t code =
setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4);
return code;
} }
TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) { TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) {

20
source/client/src/clientMain.c
View File

@ -1552,7 +1552,7 @@ TAOS_STMT *taos_stmt_init(TAOS *taos) {
return NULL; return NULL;
} }
TAOS_STMT *pStmt = stmtInit(pObj, 0); TAOS_STMT *pStmt = stmtInit(pObj, 0, NULL);
releaseTscObj(*(int64_t *)taos); releaseTscObj(*(int64_t *)taos);
@ -1567,13 +1567,29 @@ TAOS_STMT *taos_stmt_init_with_reqid(TAOS *taos, int64_t reqid) {
return NULL; return NULL;
} }
TAOS_STMT *pStmt = stmtInit(pObj, reqid); TAOS_STMT *pStmt = stmtInit(pObj, reqid, NULL);
releaseTscObj(*(int64_t *)taos); releaseTscObj(*(int64_t *)taos);
return pStmt; return pStmt;
} }
TAOS_STMT *taos_stmt_init_with_options(TAOS *taos, TAOS_STMT_OPTIONS *options) {
STscObj *pObj = acquireTscObj(*(int64_t *)taos);
if (NULL == pObj) {
tscError("invalid parameter for %s", __FUNCTION__);
terrno = TSDB_CODE_TSC_DISCONNECTED;
return NULL;
}
TAOS_STMT *pStmt = stmtInit(pObj, options->reqId, options);
releaseTscObj(*(int64_t *)taos);
return pStmt;
}
int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) { int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length) {
if (stmt == NULL || sql == NULL) { if (stmt == NULL || sql == NULL) {
tscError("NULL parameter for %s", __FUNCTION__); tscError("NULL parameter for %s", __FUNCTION__);

28
source/client/src/clientMsgHandler.c
View File

@ -25,6 +25,7 @@
#include "tglobal.h" #include "tglobal.h"
#include "tname.h" #include "tname.h"
#include "tversion.h" #include "tversion.h"
#include "command.h"
extern SClientHbMgr clientHbMgr; extern SClientHbMgr clientHbMgr;
@ -499,7 +500,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
return code; return code;
} }
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
*pRsp = taosMemoryCalloc(1, rspSize); *pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) { if (NULL == *pRsp) {
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
@ -510,14 +511,20 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) {
(*pRsp)->completed = 1; (*pRsp)->completed = 1;
(*pRsp)->precision = 0; (*pRsp)->precision = 0;
(*pRsp)->compressed = 0; (*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, SHOW_VARIABLES_RESULT_COLS);
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
if (len != rspSize - sizeof(SRetrieveTableRsp)) { SET_PAYLOAD_LEN((*pRsp)->data, len, len);
int32_t payloadLen = len + PAYLOAD_PREFIX_LEN;
(*pRsp)->payloadLen = htonl(payloadLen);
(*pRsp)->compLen = htonl(payloadLen);
if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) {
uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
(uint64_t)(rspSize - sizeof(SRetrieveTableRsp))); (uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
return TSDB_CODE_TSC_INVALID_INPUT; return TSDB_CODE_TSC_INVALID_INPUT;
@ -611,7 +618,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr
return code; return code;
} }
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
*pRsp = taosMemoryCalloc(1, rspSize); *pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) { if (NULL == *pRsp) {
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
@ -623,13 +630,20 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr
(*pRsp)->precision = 0; (*pRsp)->precision = 0;
(*pRsp)->compressed = 0; (*pRsp)->compressed = 0;
(*pRsp)->compLen = 0; (*pRsp)->compLen = 0;
(*pRsp)->payloadLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS);
int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, COMPACT_DB_RESULT_COLS);
blockDataDestroy(pBlock); blockDataDestroy(pBlock);
if (len != rspSize - sizeof(SRetrieveTableRsp)) { SET_PAYLOAD_LEN((*pRsp)->data, len, len);
int32_t payloadLen = len + PAYLOAD_PREFIX_LEN;
(*pRsp)->payloadLen = htonl(payloadLen);
(*pRsp)->compLen = htonl(payloadLen);
if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) {
uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len,
(uint64_t)(rspSize - sizeof(SRetrieveTableRsp))); (uint64_t)(rspSize - sizeof(SRetrieveTableRsp)));
return TSDB_CODE_TSC_INVALID_INPUT; return TSDB_CODE_TSC_INVALID_INPUT;

742
source/client/src/clientStmt.c
View File

File diff suppressed because it is too large Load Diff

100
source/client/test/clientTests.cpp
View File

@ -821,32 +821,21 @@ TEST(clientCase, projection_query_tables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = NULL;
// TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); // TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1");
// if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
// printf("error in create db, reason:%s\n", taos_errstr(pRes)); // printf("error in create db, reason:%s\n", taos_errstr(pRes));
// } // }
// taos_free_result(pRes); // taos_free_result(pRes);
/*
TAOS_RES* pRes = taos_query(pConn, "alter local 'fqdn 127.0.0.1'"); TAOS_RES* pRes = taos_query(pConn, "select last(ts), ts from cache_1.t1");
if (taos_errno(pRes) != 0) {
printf("failed to exec query, %s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "select last(ts), ts from cache_1.t1");
// pRes = taos_query(pConn, "select last(ts), ts from cache_1.no_pk_t1"); // pRes = taos_query(pConn, "select last(ts), ts from cache_1.no_pk_t1");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to exec query, %s\n", taos_errstr(pRes)); printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
} }
taos_free_result(pRes); taos_free_result(pRes);
// pRes = taos_query(pConn, "create stream stream_1 trigger at_once fill_history 1 ignore expired 0 into str_res1 as select _wstart as ts, count(*) from stable_1 interval(10s);");
// if (taos_errno(pRes) != 0) {
// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
// }
// taos_free_result(pRes);
pRes = taos_query(pConn, "create table tu using st2 tags(2)"); pRes = taos_query(pConn, "create table tu using st2 tags(2)");
if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
@ -876,27 +865,27 @@ TEST(clientCase, projection_query_tables) {
for(int32_t j = 0; j < 1; ++j) { for(int32_t j = 0; j < 1; ++j) {
start += 20; start += 20;
for (int32_t i = 0; i < 1; ++i) { for (int32_t i = 0; i < 1; ++i) {
createNewTable(pConn, i, 100, start, pstr); createNewTable(pConn, i, 100000, 0, pstr);
} }
} }
*/
// pRes = taos_query(pConn, "select * from abc1.st2");
// pRes = taos_query(pConn, "select * from tu"); if (taos_errno(pRes) != 0) {
// if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes);
// taos_free_result(pRes); ASSERT_TRUE(false);
// ASSERT_TRUE(false); }
// }
// TAOS_ROW pRow = NULL;
// TAOS_ROW pRow = NULL; TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// TAOS_FIELD* pFields = taos_fetch_fields(pRes); int32_t numOfFields = taos_num_fields(pRes);
// int32_t numOfFields = taos_num_fields(pRes);
// char str[512] = {0};
// char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) {
// while ((pRow = taos_fetch_row(pRes)) != NULL) { // int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); // printf("%s\n", str);
// printf("%s\n", str); }
// }
taos_free_result(pRes); taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
@ -915,25 +904,44 @@ TEST(clientCase, projection_query_stables) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use test"); TAOS_RES* pRes = taos_query(pConn, "explain select * from dbvg.st where tbname='ct1'");
taos_free_result(pRes); // taos_free_result(pRes);
pRes = taos_query(pConn, "select * from meters limit 50000000"); // pRes = taos_query(pConn, "select * from st2");
if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); // printf("failed to select from table, reason:%s\n", taos_errstr(pRes));
taos_free_result(pRes); // taos_free_result(pRes);
ASSERT_TRUE(false); // ASSERT_TRUE(false);
} // }
TAOS_ROW pRow = NULL; TAOS_ROW pRow = NULL;
TAOS_FIELD* pFields = taos_fetch_fields(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes); int32_t numOfFields = taos_num_fields(pRes);
char str[512] = {0}; int32_t numOfRows = 0;
while ((pRow = taos_fetch_row(pRes)) != NULL) { int32_t i = 0;
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); int32_t prev = 0;
// printf("%s\n", str);
char str[512] = {0};
while (1) {
pRow = taos_fetch_row(pRes);
if (pRow == NULL) {
break;
}
i += numOfRows;
if ( (i / 1000000) > prev) {
printf("%d\n", i);
prev = i/1000000;
}
//printf("%d\n", i);
} }
// while ((pRow = taos_fetch_row(pRes)) != NULL) {
// int32_t code = taos_print_row(str, pRow, pFields, numOfFields);
// if (i++ % 100000 == 0) {
// printf("%d\n", i);
// }
// }
taos_free_result(pRes); taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);

1
source/common/src/systable.c
View File

@ -264,6 +264,7 @@ static const SSysDbTableSchema userUsersSchema[] = {
{.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "super", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "createdb", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
}; };

4
source/common/src/tcol.c
View File

@ -327,7 +327,9 @@ int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressTy
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; } bool useCompress(uint8_t tableType) {
return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType;
}
int8_t validColCompressLevel(uint8_t type, uint8_t level) { int8_t validColCompressLevel(uint8_t type, uint8_t level) {
if (level == TSDB_COLVAL_LEVEL_DISABLED) return 1; if (level == TSDB_COLVAL_LEVEL_DISABLED) return 1;

6
source/common/src/tdatablock.c
View File

@ -21,7 +21,7 @@
#define MALLOC_ALIGN_BYTES 32 #define MALLOC_ALIGN_BYTES 32
static void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc);
int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) {
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
@ -848,7 +848,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
if (pBlock->pBlockAgg == NULL) { if (pBlock->pBlockAgg == NULL) {
isNull = colDataIsNull_s(pColData, j); isNull = colDataIsNull_s(pColData, j);
} else { } else {
isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); isNull = colDataIsNull(pColData, pBlock->info.rows, j, &pBlock->pBlockAgg[i]);
} }
if (isNull) { if (isNull) {
@ -1362,6 +1362,8 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
return; return;
} }
taosMemoryFreeClear(pDataBlock->pBlockAgg);
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);

73
source/common/src/tdataformat.c
View File

@ -426,6 +426,79 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
return code; return code;
} }
static int32_t tBindInfoCompare(const void *p1, const void *p2, const void *param) {
if (((SBindInfo *)p1)->columnId < ((SBindInfo *)p2)->columnId) {
return -1;
} else if (((SBindInfo *)p1)->columnId > ((SBindInfo *)p2)->columnId) {
return 1;
}
return 0;
}
/* build rows to `rowArray` from bind
* `infos` is the bind information array
* `numOfInfos` is the number of bind information
* `infoSorted` is whether the bind information is sorted by column id
* `pTSchema` is the schema of the table
* `rowArray` is the array to store the rows
*/
int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema,
SArray *rowArray) {
if (infos == NULL || numOfInfos <= 0 || numOfInfos > pTSchema->numOfCols || pTSchema == NULL || rowArray == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (!infoSorted) {
taosqsort_r(infos, numOfInfos, sizeof(SBindInfo), NULL, tBindInfoCompare);
}
int32_t code = 0;
int32_t numOfRows = infos[0].bind->num;
SArray *colValArray;
SColVal colVal;
if ((colValArray = taosArrayInit(numOfInfos, sizeof(SColVal))) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int32_t iRow = 0; iRow < numOfRows; iRow++) {
taosArrayClear(colValArray);
for (int32_t iInfo = 0; iInfo < numOfInfos; iInfo++) {
if (infos[iInfo].bind->is_null && infos[iInfo].bind->is_null[iRow]) {
colVal = COL_VAL_NULL(infos[iInfo].columnId, infos[iInfo].type);
} else {
SValue value = {
.type = infos[iInfo].type,
};
if (IS_VAR_DATA_TYPE(infos[iInfo].type)) {
value.nData = infos[iInfo].bind->length[iRow];
value.pData = (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow;
} else {
memcpy(&value.val, (uint8_t *)infos[iInfo].bind->buffer + infos[iInfo].bind->buffer_length * iRow,
infos[iInfo].bind->buffer_length);
}
colVal = COL_VAL_VALUE(infos[iInfo].columnId, value);
}
taosArrayPush(colValArray, &colVal);
}
SRow *row;
if ((code = tRowBuild(colValArray, pTSchema, &row))) {
goto _exit;
}
if ((taosArrayPush(rowArray, &row)) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
}
}
_exit:
taosArrayDestroy(colValArray);
return code;
}
int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) { int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) {
ASSERT(iCol < pTSchema->numOfCols); ASSERT(iCol < pTSchema->numOfCols);
ASSERT(pRow->sver == pTSchema->version); ASSERT(pRow->sver == pTSchema->version);

9
source/common/src/tglobal.c
View File

@ -74,6 +74,7 @@ int32_t tsNumOfSnodeStreamThreads = 4;
int32_t tsNumOfSnodeWriteThreads = 1; int32_t tsNumOfSnodeWriteThreads = 1;
int32_t tsMaxStreamBackendCache = 128; // M int32_t tsMaxStreamBackendCache = 128; // M
int32_t tsPQSortMemThreshold = 16; // M int32_t tsPQSortMemThreshold = 16; // M
int32_t tsRetentionSpeedLimitMB = 0; // unlimited
// sync raft // sync raft
int32_t tsElectInterval = 25 * 1000; int32_t tsElectInterval = 25 * 1000;
@ -667,6 +668,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1;
@ -776,7 +778,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
// min free disk space used to check if the disk is full [50MB, 1GB] // min free disk space used to check if the disk is full [50MB, 1GB]
if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
// clang-format on // clang-format on
@ -1117,6 +1119,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32;
tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32;
tsRetentionSpeedLimitMB = cfgGetItem(pCfg, "retentionSpeedLimitMB")->i32;
tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32;
tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32;
tsRatioOfVnodeStreamThreads = cfgGetItem(pCfg, "ratioOfVnodeStreamThreads")->fval; tsRatioOfVnodeStreamThreads = cfgGetItem(pCfg, "ratioOfVnodeStreamThreads")->fval;
@ -1299,8 +1302,8 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi
return 0; return 0;
} }
int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl,
const char *envFile, char *apolloUrl, SArray *pArgs) { SArray *pArgs) {
if (tsCfg == NULL) osDefaultInit(); if (tsCfg == NULL) osDefaultInit();
SConfig *pCfg = cfgInit(); SConfig *pCfg = cfgInit();

10
source/common/src/tmsg.c
View File

@ -20,11 +20,13 @@
#undef TD_MSG_DICT_ #undef TD_MSG_DICT_
#undef TD_MSG_RANGE_CODE_ #undef TD_MSG_RANGE_CODE_
#define TD_MSG_INFO_ #define TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_SEG_CODE_ #undef TD_MSG_SEG_CODE_
#include "tmsgdef.h" #include "tmsgdef.h"
#undef TD_MSG_NUMBER_ #undef TD_MSG_NUMBER_
#undef TD_MSG_INFO_ #undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_ #undef TD_MSG_RANGE_CODE_
#define TD_MSG_DICT_ #define TD_MSG_DICT_
#undef TD_MSG_SEG_CODE_ #undef TD_MSG_SEG_CODE_
@ -32,6 +34,7 @@
#undef TD_MSG_NUMBER_ #undef TD_MSG_NUMBER_
#undef TD_MSG_INFO_ #undef TD_MSG_INFO_
#undef TD_MSG_TYPE_INFO_
#undef TD_MSG_DICT_ #undef TD_MSG_DICT_
#undef TD_MSG_SEG_CODE_ #undef TD_MSG_SEG_CODE_
#define TD_MSG_RANGE_CODE_ #define TD_MSG_RANGE_CODE_
@ -1810,6 +1813,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
} }
if (tEncodeI64(&encoder, pReq->privileges) < 0) return -1; if (tEncodeI64(&encoder, pReq->privileges) < 0) return -1;
ENCODESQL(); ENCODESQL();
if (tEncodeU8(&encoder, pReq->flag) < 0) return -1;
tEndEncode(&encoder); tEndEncode(&encoder);
int32_t tlen = encoder.pos; int32_t tlen = encoder.pos;
@ -1849,6 +1853,9 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
} }
if (tDecodeI64(&decoder, &pReq->privileges) < 0) return -1; if (tDecodeI64(&decoder, &pReq->privileges) < 0) return -1;
DECODESQL(); DECODESQL();
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeU8(&decoder, &pReq->flag) < 0) return -1;
}
tEndDecode(&decoder); tEndDecode(&decoder);
tDecoderClear(&decoder); tDecoderClear(&decoder);
@ -7046,6 +7053,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) {
if (tEncodeI8(&encoder, pReq->taskType) < 0) return -1; if (tEncodeI8(&encoder, pReq->taskType) < 0) return -1;
if (tEncodeI8(&encoder, pReq->explain) < 0) return -1; if (tEncodeI8(&encoder, pReq->explain) < 0) return -1;
if (tEncodeI8(&encoder, pReq->needFetch) < 0) return -1; if (tEncodeI8(&encoder, pReq->needFetch) < 0) return -1;
if (tEncodeI8(&encoder, pReq->compress) < 0) return -1;
if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1;
if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1;
if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1; if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1;
@ -7086,6 +7094,7 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq)
if (tDecodeI8(&decoder, &pReq->taskType) < 0) return -1; if (tDecodeI8(&decoder, &pReq->taskType) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->explain) < 0) return -1; if (tDecodeI8(&decoder, &pReq->explain) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->needFetch) < 0) return -1; if (tDecodeI8(&decoder, &pReq->needFetch) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->compress) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1; if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1;
if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1; if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1;
@ -9728,6 +9737,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
for (int32_t i = 0; i < nRow; ++i) { for (int32_t i = 0; i < nRow; ++i) {
tRowDestroy(rows[i]); tRowDestroy(rows[i]);
rows[i] = NULL;
} }
taosArrayDestroy(pTbData->aRowP); taosArrayDestroy(pTbData->aRowP);
} }

6
source/common/src/tvariant.c
View File

@ -88,7 +88,7 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (val > UINT64_MAX) { if (val > (double)UINT64_MAX) {
errno = ERANGE; errno = ERANGE;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
@ -172,7 +172,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) {
} break; } break;
case TK_NK_FLOAT: { case TK_NK_FLOAT: {
double val = round(taosStr2Double(z, &endPtr)); double val = round(taosStr2Double(z, &endPtr));
if (!IS_VALID_INT64(val)) { if(val < (double)INT64_MIN || val > (double)INT64_MAX){
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {
@ -271,7 +271,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) {
} break; } break;
case TK_NK_FLOAT: { case TK_NK_FLOAT: {
double val = round(taosStr2Double(p, &endPtr)); double val = round(taosStr2Double(p, &endPtr));
if (!IS_VALID_UINT64(val)) { if (val < 0 || val > (double)UINT64_MAX) {
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;
} }
if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { if (errno == ERANGE || errno == EINVAL || endPtr - z != n) {

16
source/common/test/CMakeLists.txt
View File

@ -41,11 +41,11 @@ add_test(
) )
# tmsg test # tmsg test
# add_executable(tmsgTest "") add_executable(tmsgTest "")
# target_sources(tmsgTest target_sources(tmsgTest
# PRIVATE PRIVATE
# "tmsgTest.cpp" "tmsgTest.cpp"
# "../src/tmsg.c" "../src/tmsg.c"
# ) )
# target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/") target_include_directories(tmsgTest PUBLIC "${TD_SOURCE_DIR}/include/common/")
# target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main) target_link_libraries(tmsgTest PUBLIC os util gtest gtest_main)

15
source/common/test/tmsgTest.cpp
View File

@ -4,8 +4,23 @@
#include "tmsg.h" #include "tmsg.h"
#undef TD_MSG_NUMBER_
#undef TD_MSG_DICT_
#undef TD_MSG_INFO_
#define TD_MSG_TYPE_INFO_
#undef TD_MSG_RANGE_CODE_
#undef TD_MSG_SEG_CODE_
#include "tmsgdef.h"
TEST(td_msg_test, simple_msg_test) { TEST(td_msg_test, simple_msg_test) {
// std::cout << TMSG_INFO(TDMT_VND_DROP_TABLE) << std::endl; // std::cout << TMSG_INFO(TDMT_VND_DROP_TABLE) << std::endl;
// std::cout << TMSG_INFO(TDMT_MND_DROP_SUPER_TABLE) << std::endl; // std::cout << TMSG_INFO(TDMT_MND_DROP_SUPER_TABLE) << std::endl;
// std::cout << TMSG_INFO(TDMT_MND_CREATE_SUPER_TABLE) << std::endl; // std::cout << TMSG_INFO(TDMT_MND_CREATE_SUPER_TABLE) << std::endl;
int32_t msgSize = sizeof(tMsgTypeInfo) / sizeof(SMsgTypeInfo);
for (int32_t i = 0; i < msgSize; ++i) {
SMsgTypeInfo *pInfo = &tMsgTypeInfo[i];
std::cout << i * 2 + 1 << " " << pInfo->name << " " << pInfo->type << std::endl;
std::cout << i * 2 + 2 << " " << pInfo->rspName << " " << pInfo->rspType << std::endl;
}
} }

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -232,6 +232,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -240,6 +241,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

9
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -75,26 +75,27 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0; code = 0;
_OVER: _OVER:

1
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -967,6 +967,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

29
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -77,6 +77,7 @@ typedef enum {
MND_OPER_CREATE_VIEW, MND_OPER_CREATE_VIEW,
MND_OPER_DROP_VIEW, MND_OPER_DROP_VIEW,
MND_OPER_CONFIG_CLUSTER, MND_OPER_CONFIG_CLUSTER,
MND_OPER_BALANCE_VGROUP_LEADER,
} EOperType; } EOperType;
typedef enum { typedef enum {
@ -102,8 +103,8 @@ typedef enum {
TRN_CONFLICT_GLOBAL = 1, TRN_CONFLICT_GLOBAL = 1,
TRN_CONFLICT_DB = 2, TRN_CONFLICT_DB = 2,
TRN_CONFLICT_DB_INSIDE = 3, TRN_CONFLICT_DB_INSIDE = 3,
TRN_CONFLICT_TOPIC = 4, // TRN_CONFLICT_TOPIC = 4,
TRN_CONFLICT_TOPIC_INSIDE = 5, // TRN_CONFLICT_TOPIC_INSIDE = 5,
TRN_CONFLICT_ARBGROUP = 6, TRN_CONFLICT_ARBGROUP = 6,
} ETrnConflct; } ETrnConflct;
@ -322,15 +323,21 @@ typedef struct {
} SAcctObj; } SAcctObj;
typedef struct { typedef struct {
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN]; char pass[TSDB_PASSWORD_LEN];
char acct[TSDB_USER_LEN]; char acct[TSDB_USER_LEN];
int64_t createdTime; int64_t createdTime;
int64_t updateTime; int64_t updateTime;
int8_t superUser; int8_t superUser;
int8_t sysInfo; int8_t sysInfo;
int8_t enable; int8_t enable;
int8_t reserve; union {
uint8_t flag;
struct {
uint8_t createdb : 1;
uint8_t reserve : 7;
};
};
int32_t acctId; int32_t acctId;
int32_t authVersion; int32_t authVersion;
int32_t passVersion; int32_t passVersion;

41
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -26,13 +26,14 @@ extern "C" {
#define MND_STREAM_RESERVE_SIZE 64 #define MND_STREAM_RESERVE_SIZE 64
#define MND_STREAM_VER_NUMBER 5 #define MND_STREAM_VER_NUMBER 5
#define MND_STREAM_CREATE_NAME "stream-create" #define MND_STREAM_CREATE_NAME "stream-create"
#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint" #define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint"
#define MND_STREAM_PAUSE_NAME "stream-pause" #define MND_STREAM_PAUSE_NAME "stream-pause"
#define MND_STREAM_RESUME_NAME "stream-resume" #define MND_STREAM_RESUME_NAME "stream-resume"
#define MND_STREAM_DROP_NAME "stream-drop" #define MND_STREAM_DROP_NAME "stream-drop"
#define MND_STREAM_TASK_RESET_NAME "stream-task-reset" #define MND_STREAM_TASK_RESET_NAME "stream-task-reset"
#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update" #define MND_STREAM_TASK_UPDATE_NAME "stream-task-update"
#define MND_STREAM_CHKPT_UPDATE_NAME "stream-chkpt-update"
typedef struct SStreamTransInfo { typedef struct SStreamTransInfo {
int64_t startTime; int64_t startTime;
@ -51,6 +52,7 @@ typedef struct SStreamTransMgmt {
} SStreamTransMgmt; } SStreamTransMgmt;
typedef struct SStreamExecInfo { typedef struct SStreamExecInfo {
bool initTaskList;
SArray *pNodeList; SArray *pNodeList;
int64_t ts; // snapshot ts int64_t ts; // snapshot ts
SStreamTransMgmt transMgmt; SStreamTransMgmt transMgmt;
@ -58,6 +60,7 @@ typedef struct SStreamExecInfo {
SArray *pTaskList; SArray *pTaskList;
TdThreadMutex lock; TdThreadMutex lock;
SHashObj *pTransferStateStreams; SHashObj *pTransferStateStreams;
SHashObj *pChkptStreams;
} SStreamExecInfo; } SStreamExecInfo;
extern SStreamExecInfo execInfo; extern SStreamExecInfo execInfo;
@ -78,7 +81,18 @@ typedef struct SOrphanTask {
typedef struct { typedef struct {
SMsgHead head; SMsgHead head;
} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg; } SMStreamHbRspMsg, SMStreamReqCheckpointRsp, SMStreamUpdateChkptRsp;
typedef struct STaskChkptInfo {
int32_t nodeId;
int32_t taskId;
int64_t streamId;
int64_t checkpointId;
int64_t version;
int64_t ts;
int32_t transId;
int8_t dropHTask;
}STaskChkptInfo;
int32_t mndInitStream(SMnode *pMnode); int32_t mndInitStream(SMnode *pMnode);
void mndCleanupStream(SMnode *pMnode); void mndCleanupStream(SMnode *pMnode);
@ -96,7 +110,7 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream);
SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady);
void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName);
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode); int32_t retryCode, int32_t acceptCode);
STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg); STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg);
int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status);
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream);
@ -114,14 +128,19 @@ int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *p
int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList);
int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream);
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream);
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList);
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq);
void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo);
SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
void destroyStreamTaskIter(SStreamTaskIter *pIter); void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter);
void mndInitExecInfo(); void mndInitExecInfo();
void removeExpiredNodeInfo(const SArray *pNodeSnapshot); void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo);
void removeTasksInBuf(SArray* pTaskIds, SStreamExecInfo* pExecInfo); int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
#ifdef __cplusplus #ifdef __cplusplus
} }

15
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -91,7 +91,7 @@ void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SR
} }
} }
static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser,
bool enableReplay) { bool enableReplay) {
SMqTopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
int32_t code = 0; int32_t code = 0;
@ -135,11 +135,6 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
} }
} }
mndTransSetDbName(pTrans, pOneTopic, NULL);
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
code = -1;
goto FAILED;
}
mndReleaseTopic(pMnode, pTopic); mndReleaseTopic(pMnode, pTopic);
} }
@ -177,12 +172,12 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) {
goto END; goto END;
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC, pMsg, "recover-csm"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm");
if (pTrans == NULL) { if (pTrans == NULL) {
code = -1; code = -1;
goto END; goto END;
} }
code = validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false); code = validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false);
if (code != 0) { if (code != 0) {
goto END; goto END;
} }
@ -675,13 +670,13 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
goto _over; goto _over;
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "subscribe"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe");
if (pTrans == NULL) { if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _over; goto _over;
} }
code = validateTopics(pTrans, subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay); code = validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _over; goto _over;
} }

7
source/dnode/mnode/impl/src/mndDb.c
View File

@ -874,9 +874,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
} }
#endif #endif
mInfo("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); mInfo("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups);
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) {
goto _OVER;
}
pDb = mndAcquireDb(pMnode, createReq.db); pDb = mndAcquireDb(pMnode, createReq.db);
if (pDb != NULL) { if (pDb != NULL) {
@ -901,6 +898,10 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) {
} }
} }
if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) {
goto _OVER;
}
if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) { if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) {
code = terrno; code = terrno;
goto _OVER; goto _OVER;

220
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -14,9 +14,8 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include <stdio.h>
#include "tjson.h"
#include "mndDnode.h" #include "mndDnode.h"
#include <stdio.h>
#include "audit.h" #include "audit.h"
#include "mndCluster.h" #include "mndCluster.h"
#include "mndDb.h" #include "mndDb.h"
@ -28,9 +27,10 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#include "taos_monitor.h"
#include "tjson.h"
#include "tmisce.h" #include "tmisce.h"
#include "tunit.h" #include "tunit.h"
#include "taos_monitor.h"
#define TSDB_DNODE_VER_NUMBER 2 #define TSDB_DNODE_VER_NUMBER 2
#define TSDB_DNODE_RESERVE_SIZE 40 #define TSDB_DNODE_RESERVE_SIZE 40
@ -191,8 +191,8 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) {
SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER)
SDB_SET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER) SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER)
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible
SDB_SET_DATALEN(pRaw, dataPos, _OVER); SDB_SET_DATALEN(pRaw, dataPos, _OVER);
terrno = 0; terrno = 0;
@ -536,49 +536,49 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
int32_t code = -1; int32_t code = -1;
char strClusterId[TSDB_CLUSTER_ID_LEN] = {0}; char strClusterId[TSDB_CLUSTER_ID_LEN] = {0};
sprintf(strClusterId, "%"PRId64, pMnode->clusterId); sprintf(strClusterId, "%" PRId64, pMnode->clusterId);
if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG; terrno = TSDB_CODE_INVALID_MSG;
return code; return code;
} }
if(tsMonitorLogProtocol){ if (tsMonitorLogProtocol) {
mInfo("process statis req,\n %s", statisReq.pCont); mInfo("process statis req,\n %s", statisReq.pCont);
} }
SJson* pJson = tjsonParse(statisReq.pCont); SJson *pJson = tjsonParse(statisReq.pCont);
int32_t ts_size = tjsonGetArraySize(pJson); int32_t ts_size = tjsonGetArraySize(pJson);
for(int32_t i = 0; i < ts_size; i++){ for (int32_t i = 0; i < ts_size; i++) {
SJson* item = tjsonGetArrayItem(pJson, i); SJson *item = tjsonGetArrayItem(pJson, i);
SJson* tables = tjsonGetObjectItem(item, "tables"); SJson *tables = tjsonGetObjectItem(item, "tables");
int32_t tableSize = tjsonGetArraySize(tables); int32_t tableSize = tjsonGetArraySize(tables);
for(int32_t i = 0; i < tableSize; i++){ for (int32_t i = 0; i < tableSize; i++) {
SJson* table = tjsonGetArrayItem(tables, i); SJson *table = tjsonGetArrayItem(tables, i);
char tableName[MONITOR_TABLENAME_LEN] = {0}; char tableName[MONITOR_TABLENAME_LEN] = {0};
tjsonGetStringValue(table, "name", tableName); tjsonGetStringValue(table, "name", tableName);
SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); SJson *metricGroups = tjsonGetObjectItem(table, "metric_groups");
int32_t size = tjsonGetArraySize(metricGroups); int32_t size = tjsonGetArraySize(metricGroups);
for(int32_t i = 0; i < size; i++){ for (int32_t i = 0; i < size; i++) {
SJson* item = tjsonGetArrayItem(metricGroups, i); SJson *item = tjsonGetArrayItem(metricGroups, i);
SJson* arrayTag = tjsonGetObjectItem(item, "tags"); SJson *arrayTag = tjsonGetObjectItem(item, "tags");
int32_t tagSize = tjsonGetArraySize(arrayTag); int32_t tagSize = tjsonGetArraySize(arrayTag);
for(int32_t j = 0; j < tagSize; j++){ for (int32_t j = 0; j < tagSize; j++) {
SJson* item = tjsonGetArrayItem(arrayTag, j); SJson *item = tjsonGetArrayItem(arrayTag, j);
char tagName[MONITOR_TAG_NAME_LEN] = {0}; char tagName[MONITOR_TAG_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", tagName); tjsonGetStringValue(item, "name", tagName);
if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { if (strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
tjsonDeleteItemFromObject(item, "value"); tjsonDeleteItemFromObject(item, "value");
tjsonAddStringToObject(item, "value", strClusterId); tjsonAddStringToObject(item, "value", strClusterId);
} }
@ -590,12 +590,12 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
char *pCont = tjsonToString(pJson); char *pCont = tjsonToString(pJson);
monSendContent(pCont); monSendContent(pCont);
if(pJson != NULL){ if (pJson != NULL) {
tjsonDelete(pJson); tjsonDelete(pJson);
pJson = NULL; pJson = NULL;
} }
if(pCont != NULL){ if (pCont != NULL) {
taosMemoryFree(pCont); taosMemoryFree(pCont);
pCont = NULL; pCont = NULL;
} }
@ -603,132 +603,132 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) {
tFreeSStatisReq(&statisReq); tFreeSStatisReq(&statisReq);
return 0; return 0;
/* /*
SJson* pJson = tjsonParse(statisReq.pCont); SJson* pJson = tjsonParse(statisReq.pCont);
int32_t ts_size = tjsonGetArraySize(pJson); int32_t ts_size = tjsonGetArraySize(pJson);
for(int32_t i = 0; i < ts_size; i++){ for(int32_t i = 0; i < ts_size; i++){
SJson* item = tjsonGetArrayItem(pJson, i); SJson* item = tjsonGetArrayItem(pJson, i);
SJson* tables = tjsonGetObjectItem(item, "tables"); SJson* tables = tjsonGetObjectItem(item, "tables");
int32_t tableSize = tjsonGetArraySize(tables); int32_t tableSize = tjsonGetArraySize(tables);
for(int32_t i = 0; i < tableSize; i++){ for(int32_t i = 0; i < tableSize; i++){
SJson* table = tjsonGetArrayItem(tables, i); SJson* table = tjsonGetArrayItem(tables, i);
char tableName[MONITOR_TABLENAME_LEN] = {0}; char tableName[MONITOR_TABLENAME_LEN] = {0};
tjsonGetStringValue(table, "name", tableName); tjsonGetStringValue(table, "name", tableName);
SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups");
int32_t size = tjsonGetArraySize(metricGroups); int32_t size = tjsonGetArraySize(metricGroups);
for(int32_t i = 0; i < size; i++){ for(int32_t i = 0; i < size; i++){
SJson* item = tjsonGetArrayItem(metricGroups, i); SJson* item = tjsonGetArrayItem(metricGroups, i);
SJson* arrayTag = tjsonGetObjectItem(item, "tags"); SJson* arrayTag = tjsonGetObjectItem(item, "tags");
int32_t tagSize = tjsonGetArraySize(arrayTag); int32_t tagSize = tjsonGetArraySize(arrayTag);
char** labels = taosMemoryMalloc(sizeof(char*) * tagSize); char** labels = taosMemoryMalloc(sizeof(char*) * tagSize);
char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize); char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize);
for(int32_t j = 0; j < tagSize; j++){ for(int32_t j = 0; j < tagSize; j++){
SJson* item = tjsonGetArrayItem(arrayTag, j); SJson* item = tjsonGetArrayItem(arrayTag, j);
*(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN);
tjsonGetStringValue(item, "name", *(labels + j)); tjsonGetStringValue(item, "name", *(labels + j));
*(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN);
tjsonGetStringValue(item, "value", *(sample_labels + j)); tjsonGetStringValue(item, "value", *(sample_labels + j));
if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) {
strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN); strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN);
}
} }
}
SJson* metrics = tjsonGetObjectItem(item, "metrics"); SJson* metrics = tjsonGetObjectItem(item, "metrics");
int32_t metricLen = tjsonGetArraySize(metrics); int32_t metricLen = tjsonGetArraySize(metrics);
for(int32_t j = 0; j < metricLen; j++){ for(int32_t j = 0; j < metricLen; j++){
SJson *item = tjsonGetArrayItem(metrics, j); SJson *item = tjsonGetArrayItem(metrics, j);
char name[MONITOR_METRIC_NAME_LEN] = {0}; char name[MONITOR_METRIC_NAME_LEN] = {0};
tjsonGetStringValue(item, "name", name); tjsonGetStringValue(item, "name", name);
double value = 0; double value = 0;
tjsonGetDoubleValue(item, "value", &value); tjsonGetDoubleValue(item, "value", &value);
double type = 0; double type = 0;
tjsonGetDoubleValue(item, "type", &type); tjsonGetDoubleValue(item, "type", &type);
int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; int32_t metricNameLen = strlen(name) + strlen(tableName) + 2;
char* metricName = taosMemoryMalloc(metricNameLen); char* metricName = taosMemoryMalloc(metricNameLen);
memset(metricName, 0, metricNameLen); memset(metricName, 0, metricNameLen);
sprintf(metricName, "%s:%s", tableName, name); sprintf(metricName, "%s:%s", tableName, name);
taos_metric_t* metric = taos_collector_registry_get_metric(metricName); taos_metric_t* metric = taos_collector_registry_get_metric(metricName);
if(metric == NULL){ if(metric == NULL){
if(type == 0){
metric = taos_counter_new(metricName, "", tagSize, (const char**)labels);
}
if(type == 1){
metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels);
}
mTrace("fail to get metric from registry, new one metric:%p", metric);
if(taos_collector_registry_register_metric(metric) == 1){
if(type == 0){ if(type == 0){
taos_counter_destroy(metric); metric = taos_counter_new(metricName, "", tagSize, (const char**)labels);
} }
if(type == 1){ if(type == 1){
taos_gauge_destroy(metric); metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels);
} }
mTrace("fail to get metric from registry, new one metric:%p", metric);
metric = taos_collector_registry_get_metric(metricName); if(taos_collector_registry_register_metric(metric) == 1){
if(type == 0){
taos_counter_destroy(metric);
}
if(type == 1){
taos_gauge_destroy(metric);
}
mTrace("fail to register metric, get metric from registry:%p", metric); metric = taos_collector_registry_get_metric(metricName);
mTrace("fail to register metric, get metric from registry:%p", metric);
}
else{
mTrace("succeed to register metric:%p", metric);
}
} }
else{ else{
mTrace("succeed to register metric:%p", metric); mTrace("get metric from registry:%p", metric);
} }
}
else{ if(type == 0){
mTrace("get metric from registry:%p", metric); taos_counter_add(metric, value, (const char**)sample_labels);
}
if(type == 1){
taos_gauge_set(metric, value, (const char**)sample_labels);
}
taosMemoryFreeClear(metricName);
} }
if(type == 0){ for(int32_t j = 0; j < tagSize; j++){
taos_counter_add(metric, value, (const char**)sample_labels); taosMemoryFreeClear(*(labels + j));
} taosMemoryFreeClear(*(sample_labels + j));
if(type == 1){
taos_gauge_set(metric, value, (const char**)sample_labels);
} }
taosMemoryFreeClear(metricName); taosMemoryFreeClear(sample_labels);
taosMemoryFreeClear(labels);
} }
for(int32_t j = 0; j < tagSize; j++){
taosMemoryFreeClear(*(labels + j));
taosMemoryFreeClear(*(sample_labels + j));
}
taosMemoryFreeClear(sample_labels);
taosMemoryFreeClear(labels);
} }
} }
} code = 0;
code = 0; _OVER:
if(pJson != NULL){
tjsonDelete(pJson);
pJson = NULL;
}
_OVER: tFreeSStatisReq(&statisReq);
if(pJson != NULL){ return code;
tjsonDelete(pJson); */
pJson = NULL;
}
tFreeSStatisReq(&statisReq);
return code;
*/
} }
static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) {
@ -816,8 +816,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool reboot = (pDnode->rebootTime != statusReq.rebootTime);
bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes; bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes;
bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum; bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum;
bool enableWhiteListChanged = statusReq.clusterCfg.enableWhiteList != (tsEnableWhiteList ? 1 : 0);
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged || bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged ||
pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged; pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged || enableWhiteListChanged;
const STraceId *trace = &pReq->info.traceId; const STraceId *trace = &pReq->info.traceId;
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
@ -1148,7 +1149,6 @@ _OVER:
} }
tFreeSShowVariablesRsp(&rsp); tFreeSShowVariablesRsp(&rsp);
return code; return code;
} }

276
source/dnode/mnode/impl/src/mndStream.c
View File

@ -56,13 +56,14 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger); int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger);
static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList); static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList);
static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq); static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessCheckpointReport(SRpcMsg *pReq);
static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo);
static int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); static void removeExpiredNodeInfo(const SArray *pNodeSnapshot);
static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len);
static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
@ -103,6 +104,7 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_STOP_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_STOP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mndTransProcessRsp);
// for msgs inside mnode // for msgs inside mnode
// TODO change the name // TODO change the name
@ -114,8 +116,10 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_REPORT, mndProcessCheckpointReport);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_UPDATE_CHKPT_EVT, mndScanCheckpointReportInfo);
mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq);
mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq); mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq);
@ -131,9 +135,11 @@ int32_t mndInitStream(SMnode *pMnode) {
if (sdbSetTable(pMnode->pSdb, table) != 0) { if (sdbSetTable(pMnode->pSdb, table) != 0) {
return -1; return -1;
} }
if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) {
return -1; return -1;
} }
return 0; return 0;
} }
@ -143,6 +149,7 @@ void mndCleanupStream(SMnode *pMnode) {
taosHashCleanup(execInfo.pTaskMap); taosHashCleanup(execInfo.pTaskMap);
taosHashCleanup(execInfo.transMgmt.pDBTrans); taosHashCleanup(execInfo.transMgmt.pDBTrans);
taosHashCleanup(execInfo.pTransferStateStreams); taosHashCleanup(execInfo.pTransferStateStreams);
taosHashCleanup(execInfo.pChkptStreams);
taosThreadMutexDestroy(&execInfo.lock); taosThreadMutexDestroy(&execInfo.lock);
mDebug("mnd stream exec info cleanup"); mDebug("mnd stream exec info cleanup");
} }
@ -508,7 +515,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
tEncodeStreamTask(&encoder, pTask); tEncodeStreamTask(&encoder, pTask);
tEncoderClear(&encoder); tEncoderClear(&encoder);
int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0); int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(buf); taosMemoryFree(buf);
return -1; return -1;
@ -952,7 +959,7 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return -1; return -1;
} }
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY); code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(buf); taosMemoryFree(buf);
} }
@ -1088,7 +1095,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
} }
static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
bool ready = true; bool ready = true;
if (taskNodeIsUpdated(pMnode)) { if (taskNodeIsUpdated(pMnode)) {
return -1; return -1;
} }
@ -1099,6 +1106,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0);
} }
SArray* pInvalidList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
@ -1106,11 +1115,20 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
continue; continue;
} }
if (pEntry->status == TASK_STATUS__STOP) {
for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) {
STaskId* pId = taosArrayGet(pInvalidList, j);
if (pEntry->id.streamId == pId->streamId) {
taosArrayPush(pInvalidList, &pEntry->id);
break;
}
}
}
if (pEntry->status != TASK_STATUS__READY) { if (pEntry->status != TASK_STATUS__READY) {
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued",
pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status)); pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status));
ready = false; ready = false;
break;
} }
if (pEntry->hTaskId != 0) { if (pEntry->hTaskId != 0) {
@ -1123,6 +1141,9 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
} }
} }
removeTasksInBuf(pInvalidList, &execInfo);
taosArrayDestroy(pInvalidList);
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
return ready ? 0 : -1; return ready ? 0 : -1;
} }
@ -1151,7 +1172,8 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
int32_t numOfCheckpointTrans = 0; int32_t numOfCheckpointTrans = 0;
if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) { if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) {
return code; terrno = TSDB_CODE_STREAM_TASK_IVLD_STATUS;
return -1;
} }
SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval)); SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval));
@ -1798,6 +1820,10 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfRows = 0; int32_t numOfRows = 0;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
taosThreadMutexLock(&execInfo.lock);
mndInitStreamExecInfo(pMnode, &execInfo);
taosThreadMutexUnlock(&execInfo.lock);
while (numOfRows < rowsCapacity) { while (numOfRows < rowsCapacity) {
pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream);
if (pShow->pIter == NULL) { if (pShow->pIter == NULL) {
@ -2169,7 +2195,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
return 0; return 0;
} }
static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) { static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
@ -2215,48 +2241,6 @@ static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static bool taskNodeExists(SArray *pList, int32_t nodeId) {
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
SNodeEntry *pEntry = taosArrayGet(pList, i);
if (pEntry->nodeId == nodeId) {
return true;
}
}
return false;
}
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
for (int32_t i = 0; i < numOfTask; ++i) {
STaskId *pId = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
if (pEntry->nodeId == SNODE_HANDLE) {
continue;
}
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) {
taosArrayPush(pRemovedTasks, pId);
}
}
removeTasksInBuf(pRemovedTasks, &execInfo);
mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
(int32_t)taosArrayGetSize(execInfo.pTaskList));
removeExpiredNodeInfo(pNodeSnapshot);
taosArrayDestroy(pRemovedTasks);
return 0;
}
// this function runs by only one thread, so it is not multi-thread safe // this function runs by only one thread, so it is not multi-thread safe
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
int32_t code = 0; int32_t code = 0;
@ -2476,13 +2460,137 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
{ {
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)}; SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)};
rsp.pCont = rpcMallocCont(rsp.contLen); rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead *pHead = rsp.pCont; SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.nodeId); pHead->vgId = htonl(req.nodeId);
tmsgSendRsp(&rsp); tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp
}
return 0;
}
static void doAddTaskInfo(SArray* pList, SCheckpointReport* pReport) {
bool existed = false;
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
STaskChkptInfo* p = taosArrayGet(pList ,i);
if (p->taskId == pReport->taskId) {
existed = true;
break;
}
}
if (!existed) {
STaskChkptInfo info = {
.streamId = pReport->streamId,
.taskId = pReport->taskId,
.transId = pReport->transId,
.dropHTask = pReport->dropHTask,
.version = pReport->checkpointVer,
.ts = pReport->checkpointTs,
.checkpointId = pReport->checkpointId,
.nodeId = pReport->nodeId,
};
taosArrayPush(pList, &info);
}
}
int32_t mndProcessCheckpointReport(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SCheckpointReport req = {0};
SDecoder decoder = {0};
tDecoderInit(&decoder, pReq->pCont, pReq->contLen);
if (tDecodeStreamTaskChkptReport(&decoder, &req)) {
tDecoderClear(&decoder);
terrno = TSDB_CODE_INVALID_MSG;
mError("invalid task checkpoint-report msg received");
return -1;
}
tDecoderClear(&decoder);
mDebug("receive stream task checkpoint-report msg, vgId:%d, s-task:0x%x, checkpointId:%" PRId64
" checkpointVer:%" PRId64 " transId:%d",
req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId);
// register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans.
taosThreadMutexLock(&execInfo.lock);
SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId);
if (pStream == NULL) {
mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf",
req.streamId);
// not in meta-store yet, try to acquire the task in exec buffer
// the checkpoint req arrives too soon before the completion of the create stream trans.
STaskId id = {.streamId = req.streamId, .taskId = req.taskId};
void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
if (p == NULL) {
mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST;
taosThreadMutexUnlock(&execInfo.lock);
return -1;
} else {
mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet",
req.streamId, req.taskId);
}
}
int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream);
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
if (pReqTaskList == NULL) {
SArray *pList = taosArrayInit(4, sizeof(STaskChkptInfo));
doAddTaskInfo(pList, &req);
taosHashPut(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId), &pList, POINTER_BYTES);
pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
} else {
doAddTaskInfo(*pReqTaskList, &req);
}
int32_t total = taosArrayGetSize(*pReqTaskList);
if (total == numOfTasks) { // all tasks has send the reqs
mInfo("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, checkpoint meta-info for checkpointId:%" PRId64
" will be issued soon",
req.streamId, pStream->name, total, req.checkpointId);
// if (pStream != NULL) {
// bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
// if (conflict) {
// mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId);
// } else {
// int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, *pReqTaskList);
// if (code == TSDB_CODE_SUCCESS) { // remove this entry
// taosHashRemove(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId));
//
// int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
// mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", req.streamId,
// numOfStreams);
// } else {
// mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet",
// req.streamId);
// }
// }
// }
}
if (pStream != NULL) {
mndReleaseStream(pMnode, pStream);
}
taosThreadMutexUnlock(&execInfo.lock);
{
SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamUpdateChkptRsp)};
rsp.pCont = rpcMallocCont(rsp.contLen);
SMsgHead *pHead = rsp.pCont;
pHead->vgId = htonl(req.nodeId);
tmsgSendRsp(&rsp);
pReq->info.handle = NULL; // disable auto rsp pReq->info.handle = NULL; // disable auto rsp
} }
@ -2510,3 +2618,63 @@ static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) {
} }
return code; return code;
} }
void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
if (pExecInfo->initTaskList || pMnode == NULL) {
return;
}
addAllStreamTasksIntoBuf(pMnode, pExecInfo);
extractNodeListFromStream(pMnode, pExecInfo->pNodeList);
pExecInfo->initTaskList = true;
}
void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) {
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) {
break;
}
saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo);
sdbRelease(pSdb, pStream);
}
}
int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray* pChkptInfoList) {
STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, "update checkpoint-info");
if (pTrans == NULL) {
return terrno;
}
/*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid);
int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream);
if (code != 0) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return code;
}
code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY);
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return TSDB_CODE_ACTION_IN_PROGRESS;
}

88
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -22,54 +22,7 @@ typedef struct SFailedCheckpointInfo {
int32_t transId; int32_t transId;
} SFailedCheckpointInfo; } SFailedCheckpointInfo;
static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { static void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode);
SSdb *pSdb = pMnode->pSdb;
SStreamObj *pStream = NULL;
void *pIter = NULL;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) {
break;
}
saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo);
sdbRelease(pSdb, pStream);
}
}
static void removeDroppedStreamTasksInBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) {
if (pMnode == NULL) {
return;
}
int32_t num = taosArrayGetSize(pExecInfo->pTaskList);
SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
SArray *pIdList = taosArrayInit(4, sizeof(STaskId));
for (int32_t i = 0; i < num; ++i) {
STaskId* pId = taosArrayGet(pExecInfo->pTaskList, i);
void* p = taosHashGet(pHash, &pId->streamId, sizeof(int64_t));
if (p != NULL) {
continue;
}
void* pObj = mndGetStreamObj(pMnode, pId->streamId);
if (pObj != NULL) {
mndReleaseStream(pMnode, pObj);
taosHashPut(pHash, &pId->streamId, sizeof(int64_t), NULL, 0);
} else {
taosArrayPush(pIdList, pId);
}
}
removeTasksInBuf(pIdList, &execInfo);
taosArrayDestroy(pIdList);
taosHashCleanup(pHash);
}
static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) {
int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList); int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList);
@ -290,16 +243,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
taosThreadMutexLock(&execInfo.lock); taosThreadMutexLock(&execInfo.lock);
// extract stream task list mndInitStreamExecInfo(pMnode, &execInfo);
if (taosHashGetSize(execInfo.pTaskMap) == 0) {
addAllStreamTasksIntoBuf(pMnode, &execInfo);
} else {
// the already dropped tasks may be added by hb from vnode at the time when the pTaskMap happens to be empty.
// let's drop them here.
removeDroppedStreamTasksInBuf(pMnode, &execInfo);
}
extractStreamNodeList(pMnode);
int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes);
if (numOfUpdated > 0) { if (numOfUpdated > 0) {
@ -326,18 +270,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
snodeChanged = true; snodeChanged = true;
} }
} else { } else {
// task is idle for more than 50 sec.
// if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) {
// if (!pTaskEntry->inputQChanging) {
// pTaskEntry->inputQUnchangeCounter++;
// } else {
// pTaskEntry->inputQChanging = false;
// }
// } else {
// pTaskEntry->inputQChanging = true;
// pTaskEntry->inputQUnchangeCounter = 0;
// }
streamTaskStatusCopy(pTaskEntry, p); streamTaskStatusCopy(pTaskEntry, p);
STaskCkptInfo *pChkInfo = &p->checkpointInfo; STaskCkptInfo *pChkInfo = &p->checkpointInfo;
@ -348,6 +280,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SFailedCheckpointInfo info = { SFailedCheckpointInfo info = {
.transId = pChkInfo->activeTransId, .checkpointId = pChkInfo->activeId, .streamUid = p->id.streamId}; .transId = pChkInfo->activeTransId, .checkpointId = pChkInfo->activeId, .streamUid = p->id.streamId};
addIntoCheckpointList(pFailedChkpt, &info); addIntoCheckpointList(pFailedChkpt, &info);
// remove failed trans from pChkptStreams
taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId));
} }
} }
@ -393,6 +328,10 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
mndDropOrphanTasks(pMnode, pOrphanTasks); mndDropOrphanTasks(pMnode, pOrphanTasks);
} }
if (pMnode != NULL) { // make sure that the unit test case can work
mndStreamStartUpdateCheckpointInfo(pMnode);
}
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
tCleanupStreamHbMsg(&req); tCleanupStreamHbMsg(&req);
@ -411,3 +350,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
if (pMsg != NULL) {
int32_t size = sizeof(SMStreamDoCheckpointMsg);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_UPDATE_CHKPT_EVT, .pCont = pMsg, .contLen = size};
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}

14
source/dnode/mnode/impl/src/mndStreamTrans.c
View File

@ -127,7 +127,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p
return false; return false;
} }
int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) { int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) {
taosThreadMutexLock(&execInfo.lock); taosThreadMutexLock(&execInfo.lock);
int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans);
if (num <= 0) { if (num <= 0) {
@ -136,12 +136,13 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) {
} }
mndStreamClearFinishedTrans(pMnode, NULL); mndStreamClearFinishedTrans(pMnode, NULL);
SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid)); SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId));
if (pEntry != NULL) { if (pEntry != NULL) {
SStreamTransInfo tInfo = *pEntry; SStreamTransInfo tInfo = *pEntry;
taosThreadMutexUnlock(&execInfo.lock); taosThreadMutexUnlock(&execInfo.lock);
if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0) { if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0 ||
strcmp(tInfo.name, MND_STREAM_CHKPT_UPDATE_NAME) == 0) {
return tInfo.transId; return tInfo.transId;
} }
} else { } else {
@ -159,7 +160,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnCo
return NULL; return NULL;
} }
mInfo("s-task:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id); mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id);
mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, pTrans) != 0) { if (mndTransCheckConflict(pMnode, pTrans) != 0) {
@ -246,8 +247,9 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status)
} }
int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode) { int32_t retryCode, int32_t acceptCode) {
STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode}; STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode,
.acceptableCode = acceptCode};
return mndTransAppendRedoAction(pTrans, &action); return mndTransAppendRedoAction(pTrans, &action);
} }

184
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -230,7 +230,7 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT
return -1; return -1;
} }
code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0); code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -308,7 +308,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
epsetToStr(&epset, buf, tListLen(buf)); epsetToStr(&epset, buf, tListLen(buf));
mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf);
code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0); code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -356,7 +356,7 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -400,7 +400,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask
} }
// The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode.
code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0); code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0);
if (code != 0) { if (code != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -484,7 +484,7 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask
return code; return code;
} }
code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID); code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
} }
@ -534,7 +534,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa
return code; return code;
} }
code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0); code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0, 0);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
} }
@ -574,9 +574,11 @@ void mndInitExecInfo() {
execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK);
execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK); execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pChkptStreams = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry)); execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry));
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList);
} }
void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { void removeExpiredNodeInfo(const SArray *pNodeSnapshot) {
@ -645,4 +647,174 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
taosThreadMutexUnlock(&pExecNode->lock); taosThreadMutexUnlock(&pExecNode->lock);
destroyStreamTaskIter(pIter); destroyStreamTaskIter(pIter);
}
static bool taskNodeExists(SArray *pList, int32_t nodeId) {
size_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) {
SNodeEntry *pEntry = taosArrayGet(pList, i);
if (pEntry->nodeId == nodeId) {
return true;
}
}
return false;
}
int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) {
SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
for (int32_t i = 0; i < numOfTask; ++i) {
STaskId *pId = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
if (pEntry->nodeId == SNODE_HANDLE) {
continue;
}
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) {
taosArrayPush(pRemovedTasks, pId);
}
}
removeTasksInBuf(pRemovedTasks, &execInfo);
mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
(int32_t)taosArrayGetSize(execInfo.pTaskList));
removeExpiredNodeInfo(pNodeSnapshot);
taosArrayDestroy(pRemovedTasks);
return 0;
}
static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) {
SVUpdateCheckpointInfoReq *pReq = taosMemoryCalloc(1, sizeof(SVUpdateCheckpointInfoReq));
if (pReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVUpdateCheckpointInfoReq),
tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return terrno;
}
pReq->head.vgId = htonl(pTask->info.nodeId);
pReq->taskId = pTask->id.taskId;
pReq->streamId = pTask->id.streamId;
SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &pTask->id.streamId, sizeof(pTask->id.streamId));
ASSERT(pReqTaskList);
int32_t size = taosArrayGetSize(*pReqTaskList);
for(int32_t i = 0; i < size; ++i) {
STaskChkptInfo* pInfo = taosArrayGet(*pReqTaskList, i);
if (pInfo->taskId == pTask->id.taskId) {
pReq->checkpointId = pInfo->checkpointId;
pReq->checkpointVer = pInfo->version;
pReq->checkpointTs = pInfo->ts;
pReq->dropRelHTask = pInfo->dropHTask;
pReq->transId = pInfo->transId;
pReq->hStreamId = pTask->hTaskInfo.id.streamId;
pReq->hTaskId = pTask->hTaskInfo.id.taskId;
}
}
SEpSet epset = {0};
bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) {
taosMemoryFree(pReq);
return code;
}
code = setTransAction(pTrans, pReq, sizeof(SVUpdateCheckpointInfoReq), TDMT_STREAM_TASK_UPDATE_CHKPT, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pReq);
}
return code;
}
int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) {
taosWLockLatch(&pStream->lock);
SStreamTaskIter *pIter = createStreamTaskIter(pStream);
while (streamTaskIterNextTask(pIter)) {
SStreamTask *pTask = streamTaskIterGetCurrent(pIter);
int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask);
if (code != TSDB_CODE_SUCCESS) {
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return -1;
}
}
destroyStreamTaskIter(pIter);
taosWUnLockLatch(&pStream->lock);
return 0;
}
int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
void *pIter = NULL;
SArray *pDropped = taosArrayInit(4, sizeof(int64_t));
mDebug("start to scan checkpoint report info");
while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) {
SArray *pList = *(SArray **)pIter;
STaskChkptInfo* pInfo = taosArrayGet(pList, 0);
SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId);
if (pStream == NULL) {
mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId);
taosArrayPush(pDropped, &pInfo->streamId);
continue;
}
int32_t total = mndGetNumOfStreamTasks(pStream);
int32_t existed = (int32_t) taosArrayGetSize(pList);
if (total == existed) {
mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info",
pStream->uid, pStream->name, total);
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false);
if (!conflict) {
int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList);
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry
taosArrayPush(pDropped, &pInfo->streamId);
mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId);
} else {
mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet",
pInfo->streamId);
}
break;
} else {
mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId);
}
} else {
mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name,
existed, total, total - existed);
}
sdbRelease(pMnode->pSdb, pStream);
}
int32_t size = taosArrayGetSize(pDropped);
if (size > 0) {
for (int32_t i = 0; i < size; ++i) {
int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i);
taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId));
}
int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams);
mDebug("drop %d stream(s) in checkpoint-report list, remain:%d", size, numOfStreams);
}
taosArrayDestroy(pDropped);
return TSDB_CODE_SUCCESS;
} }

38
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -618,13 +618,13 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu
char cgroup[TSDB_CGROUP_LEN] = {0}; char cgroup[TSDB_CGROUP_LEN] = {0};
mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true); mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true);
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "tmq-reb"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb");
if (pTrans == NULL) { if (pTrans == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto END; goto END;
} }
mndTransSetDbName(pTrans, topic, cgroup); mndTransSetDbName(pTrans, pOutput->pSub->dbName, cgroup);
code = mndTransCheckConflict(pMnode, pTrans); code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) { if (code != 0) {
goto END; goto END;
@ -908,33 +908,37 @@ END:
} }
static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){
// iter all vnode to delete handle void* pIter = NULL;
int32_t sz = taosArrayGetSize(pSub->unassignedVgs); SVgObj* pVgObj = NULL;
for (int32_t i = 0; i < sz; i++) { while (1) {
SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i); pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgObj);
if (pIter == NULL) {
break;
}
if (!mndVgroupInDb(pVgObj, pSub->dbUid)) {
sdbRelease(pMnode->pSdb, pVgObj);
continue;
}
SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq));
if(pReq == NULL){ if(pReq == NULL){
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbRelease(pMnode->pSdb, pVgObj);
return -1; return -1;
} }
pReq->head.vgId = htonl(pVgEp->vgId); pReq->head.vgId = htonl(pVgObj->vgId);
pReq->vgId = pVgEp->vgId; pReq->vgId = pVgObj->vgId;
pReq->consumerId = -1; pReq->consumerId = -1;
memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
SVgObj *pVgObj = mndAcquireVgroup(pMnode, pVgEp->vgId);
if (pVgObj == NULL) {
taosMemoryFree(pReq);
terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST;
return -1;
}
STransAction action = {0}; STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);; action.epSet = mndGetVgroupEpset(pMnode, pVgObj);;
action.pCont = pReq; action.pCont = pReq;
action.contLen = sizeof(SMqVDeleteReq); action.contLen = sizeof(SMqVDeleteReq);
action.msgType = TDMT_VND_TMQ_DELETE_SUB; action.msgType = TDMT_VND_TMQ_DELETE_SUB;
action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST;
mndReleaseVgroup(pMnode, pVgObj); sdbRelease(pMnode->pSdb, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -996,7 +1000,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
goto end; goto end;
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "drop-cgroup"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr());
code = -1; code = -1;
@ -1004,7 +1008,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
} }
mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic); mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic);
mndTransSetDbName(pTrans, dropReq.topic, dropReq.cgroup); mndTransSetDbName(pTrans, pSub->dbName, dropReq.cgroup);
code = mndTransCheckConflict(pMnode, pTrans); code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) { if (code != 0) {
goto end; goto end;

8
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -422,14 +422,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
SQueryPlan *pPlan = NULL; SQueryPlan *pPlan = NULL;
SMqTopicObj topicObj = {0}; SMqTopicObj topicObj = {0};
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "create-topic"); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); mError("topic:%s, failed to create since %s", pCreate->name, terrstr());
code = -1; code = -1;
goto _OUT; goto _OUT;
} }
mndTransSetDbName(pTrans, pCreate->name, NULL); mndTransSetDbName(pTrans, pDb->name, NULL);
code = mndTransCheckConflict(pMnode, pTrans); code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) { if (code != 0) {
goto _OUT; goto _OUT;
@ -779,14 +779,14 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
} }
} }
pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "drop-topic"); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic");
if (pTrans == NULL) { if (pTrans == NULL) {
mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); mError("topic:%s, failed to drop since %s", pTopic->name, terrstr());
code = -1; code = -1;
goto end; goto end;
} }
mndTransSetDbName(pTrans, pTopic->name, NULL); mndTransSetDbName(pTrans, pTopic->db, NULL);
code = mndTransCheckConflict(pMnode, pTrans); code = mndTransCheckConflict(pMnode, pTrans);
if (code != 0) { if (code != 0) {
goto end; goto end;

34
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -836,26 +836,26 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) {
} }
} }
if (pNew->conflict == TRN_CONFLICT_TOPIC) { // if (pNew->conflict == TRN_CONFLICT_TOPIC) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; // if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { // if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; // if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
} // }
} // }
if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) { // if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; // if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_TOPIC) { // if (pTrans->conflict == TRN_CONFLICT_TOPIC) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; // if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true;
} // }
if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { // if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) {
if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0) // if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0)
conflict = true; // conflict = true;
} // }
} // }
if (pNew->conflict == TRN_CONFLICT_ARBGROUP) { if (pNew->conflict == TRN_CONFLICT_ARBGROUP) {
if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true;
if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) { if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) {
void *pIter = taosHashIterate(pNew->arbGroupIds, NULL); pIter = taosHashIterate(pNew->arbGroupIds, NULL);
while (pIter != NULL) { while (pIter != NULL) {
int32_t groupId = *(int32_t *)pIter; int32_t groupId = *(int32_t *)pIter;
if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) { if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) {

28
source/dnode/mnode/impl/src/mndUser.c
View File

@ -648,6 +648,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char
userObj.pIpWhiteList = createDefaultIpWhiteList(); userObj.pIpWhiteList = createDefaultIpWhiteList();
if (strcmp(user, TSDB_DEFAULT_USER) == 0) { if (strcmp(user, TSDB_DEFAULT_USER) == 0) {
userObj.superUser = 1; userObj.superUser = 1;
userObj.createdb = 1;
} }
SSdbRaw *pRaw = mndUserActionEncode(&userObj); SSdbRaw *pRaw = mndUserActionEncode(&userObj);
@ -817,7 +818,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->sysInfo, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->sysInfo, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER)
SDB_SET_INT8(pRaw, dataPos, pUser->reserve, _OVER) SDB_SET_UINT8(pRaw, dataPos, pUser->flag, _OVER)
SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER)
SDB_SET_INT32(pRaw, dataPos, pUser->passVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->passVersion, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER)
@ -1001,7 +1002,8 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->sysInfo, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->sysInfo, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pUser->reserve, _OVER) SDB_GET_UINT8(pRaw, dataPos, &pUser->flag, _OVER)
if (pUser->superUser) pUser->createdb = 1;
SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER)
if (sver >= 4) { if (sver >= 4) {
SDB_GET_INT32(pRaw, dataPos, &pUser->passVersion, _OVER) SDB_GET_INT32(pRaw, dataPos, &pUser->passVersion, _OVER)
@ -1395,6 +1397,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
pOld->passVersion = pNew->passVersion; pOld->passVersion = pNew->passVersion;
pOld->sysInfo = pNew->sysInfo; pOld->sysInfo = pNew->sysInfo;
pOld->enable = pNew->enable; pOld->enable = pNew->enable;
pOld->flag = pNew->flag;
memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN);
TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->readDbs, pNew->readDbs);
TSWAP(pOld->writeDbs, pNew->writeDbs); TSWAP(pOld->writeDbs, pNew->writeDbs);
@ -1445,6 +1448,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate
userObj.superUser = 0; // pCreate->superUser; userObj.superUser = 0; // pCreate->superUser;
userObj.sysInfo = pCreate->sysInfo; userObj.sysInfo = pCreate->sysInfo;
userObj.enable = pCreate->enable; userObj.enable = pCreate->enable;
userObj.createdb = 0;
if (pCreate->numIpRanges == 0) { if (pCreate->numIpRanges == 0) {
userObj.pIpWhiteList = createDefaultIpWhiteList(); userObj.pIpWhiteList = createDefaultIpWhiteList();
@ -1790,6 +1794,9 @@ static char *mndUserAuditTypeStr(int32_t type) {
if (type == TSDB_ALTER_USER_SYSINFO) { if (type == TSDB_ALTER_USER_SYSINFO) {
return "userSysInfo"; return "userSysInfo";
} }
if (type == TSDB_ALTER_USER_CREATEDB) {
return "userCreateDB";
}
return "error"; return "error";
} }
@ -2009,6 +2016,10 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
newUser.sysInfo = alterReq.sysInfo; newUser.sysInfo = alterReq.sysInfo;
} }
if(alterReq.alterType == TSDB_ALTER_USER_CREATEDB) {
newUser.createdb = alterReq.createdb;
}
if (ALTER_USER_ADD_PRIVS(alterReq.alterType) || ALTER_USER_DEL_PRIVS(alterReq.alterType)) { if (ALTER_USER_ADD_PRIVS(alterReq.alterType) || ALTER_USER_DEL_PRIVS(alterReq.alterType)) {
if (0 != mndProcessAlterUserPrivilegesReq(&alterReq, pMnode, &newUser)) goto _OVER; if (0 != mndProcessAlterUserPrivilegesReq(&alterReq, pMnode, &newUser)) goto _OVER;
} }
@ -2110,14 +2121,15 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
if(alterReq.alterType == TSDB_ALTER_USER_PASSWD){ if(alterReq.alterType == TSDB_ALTER_USER_PASSWD){
char detail[1000] = {0}; char detail[1000] = {0};
sprintf(detail, "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, tabName:%s, password:xxx", sprintf(detail, "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, createdb:%d, tabName:%s, password:xxx",
mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo, mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo,
alterReq.tabName); alterReq.createdb ? 1 : 0, alterReq.tabName);
auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, detail, strlen(detail)); auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, detail, strlen(detail));
} }
else if(alterReq.alterType == TSDB_ALTER_USER_SUPERUSER || else if(alterReq.alterType == TSDB_ALTER_USER_SUPERUSER ||
alterReq.alterType == TSDB_ALTER_USER_ENABLE || alterReq.alterType == TSDB_ALTER_USER_ENABLE ||
alterReq.alterType == TSDB_ALTER_USER_SYSINFO){ alterReq.alterType == TSDB_ALTER_USER_SYSINFO ||
alterReq.alterType == TSDB_ALTER_USER_CREATEDB){
auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, alterReq.sql, alterReq.sqlLen); auditRecord(pReq, pMnode->clusterId, "alterUser", "", alterReq.user, alterReq.sql, alterReq.sqlLen);
} }
else if(ALTER_USER_ADD_READ_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)|| else if(ALTER_USER_ADD_READ_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
@ -2290,6 +2302,7 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
int32_t numOfRows = 0; int32_t numOfRows = 0;
SUserObj *pUser = NULL; SUserObj *pUser = NULL;
int32_t cols = 0; int32_t cols = 0;
int8_t flag = 0;
char *pWrite; char *pWrite;
while (numOfRows < rows) { while (numOfRows < rows) {
@ -2314,6 +2327,11 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->sysInfo, false);
cols++;
flag = pUser->createdb ? 1 : 0;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
colDataSetVal(pColInfo, numOfRows, (const char *)&flag, false);
cols++; cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->createdTime, false);

27
source/dnode/mnode/impl/test/func/func.cpp
View File

@ -169,7 +169,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1; retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f1"); char name[TSDB_FUNC_NAME_LEN] = "f1";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
@ -220,7 +221,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
retrieveReq.numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; retrieveReq.numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1;
retrieveReq.pFuncNames = taosArrayInit(TSDB_FUNC_MAX_RETRIEVE + 1, TSDB_FUNC_NAME_LEN); retrieveReq.pFuncNames = taosArrayInit(TSDB_FUNC_MAX_RETRIEVE + 1, TSDB_FUNC_NAME_LEN);
for (int32_t i = 0; i < TSDB_FUNC_MAX_RETRIEVE + 1; ++i) { for (int32_t i = 0; i < TSDB_FUNC_MAX_RETRIEVE + 1; ++i) {
taosArrayPush(retrieveReq.pFuncNames, "1"); char name[TSDB_FUNC_NAME_LEN] = "1";
taosArrayPush(retrieveReq.pFuncNames, name);
} }
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
@ -237,7 +239,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1; retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2"); char name[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
@ -279,7 +282,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1; retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2"); char name[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
@ -316,8 +320,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 2; retrieveReq.numOfFuncs = 2;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2"); char name1[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, "f1"); taosArrayPush(retrieveReq.pFuncNames, name1);
char name2[TSDB_FUNC_NAME_LEN] = "f1";
taosArrayPush(retrieveReq.pFuncNames, name2);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
@ -367,8 +373,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 2; retrieveReq.numOfFuncs = 2;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "f2"); char name1[TSDB_FUNC_NAME_LEN] = "f2";
taosArrayPush(retrieveReq.pFuncNames, "f3"); taosArrayPush(retrieveReq.pFuncNames, name1);
char name2[TSDB_FUNC_NAME_LEN] = "f3";
taosArrayPush(retrieveReq.pFuncNames, name2);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);
@ -483,7 +491,8 @@ TEST_F(MndTestFunc, 05_Actual_code) {
SRetrieveFuncReq retrieveReq = {0}; SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1; retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "udf1"); char name[TSDB_FUNC_NAME_LEN] = "udf1";
taosArrayPush(retrieveReq.pFuncNames, name);
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen); void* pReq = rpcMallocCont(contLen);

4
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -57,6 +57,7 @@ extern "C" {
#define SDB_GET_INT32(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt32, int32_t) #define SDB_GET_INT32(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt32, int32_t)
#define SDB_GET_INT16(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt16, int16_t) #define SDB_GET_INT16(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt16, int16_t)
#define SDB_GET_INT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt8, int8_t) #define SDB_GET_INT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt8, int8_t)
#define SDB_GET_UINT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawUInt8, uint8_t)
#define SDB_GET_RESERVE(pRaw, dataPos, valLen, pos) \ #define SDB_GET_RESERVE(pRaw, dataPos, valLen, pos) \
{ \ { \
@ -76,6 +77,7 @@ extern "C" {
#define SDB_SET_INT32(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt32, int32_t) #define SDB_SET_INT32(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt32, int32_t)
#define SDB_SET_INT16(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt16, int16_t) #define SDB_SET_INT16(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt16, int16_t)
#define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t) #define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t)
#define SDB_SET_UINT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawUInt8, uint8_t)
#define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ #define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \
{ \ { \
@ -388,6 +390,7 @@ void sdbGetCommitInfo(SSdb *pSdb, int64_t *index, int64_t *term, int64_t *config
SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen);
void sdbFreeRaw(SSdbRaw *pRaw); void sdbFreeRaw(SSdbRaw *pRaw);
int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val); int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val);
int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val);
int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val); int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val);
int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val); int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val);
int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val); int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val);
@ -395,6 +398,7 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32
int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen); int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen);
int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status); int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status);
int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val); int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val);
int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val);
int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val); int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val);
int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val); int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val);
int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val); int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val);

30
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -67,6 +67,21 @@ int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) {
return 0; return 0;
} }
int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) {
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
}
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
}
*(uint8_t *)(pRaw->pData + dataPos) = val;
return 0;
}
int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) { int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) {
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;
@ -174,6 +189,21 @@ int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) {
return 0; return 0;
} }
int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) {
if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR;
return -1;
}
if (dataPos + sizeof(uint8_t) > pRaw->dataLen) {
terrno = TSDB_CODE_SDB_INVALID_DATA_LEN;
return -1;
}
*val = *(uint8_t *)(pRaw->pData + dataPos);
return 0;
}
int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) { int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) {
if (pRaw == NULL) { if (pRaw == NULL) {
terrno = TSDB_CODE_INVALID_PTR; terrno = TSDB_CODE_INVALID_PTR;

18
source/dnode/snode/src/snode.c
View File

@ -25,20 +25,6 @@
#define sndDebug(...) do { if (sndDebugFlag & DEBUG_DEBUG) { taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__);}} while (0) #define sndDebug(...) do { if (sndDebugFlag & DEBUG_DEBUG) { taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__);}} while (0)
// clang-format on // clang-format on
static STaskId replaceStreamTaskId(SStreamTask *pTask) {
ASSERT(pTask->info.fillHistory);
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
pTask->id.streamId = pTask->streamTaskId.streamId;
pTask->id.taskId = pTask->streamTaskId.taskId;
return id;
}
static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) {
ASSERT(pTask->info.fillHistory);
pTask->id.taskId = pId->taskId;
pTask->id.streamId = pId->streamId;
}
int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) { int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0); ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0);
int32_t code = streamTaskInit(pTask, pSnode->pMeta, &pSnode->msgCb, nextProcessVer); int32_t code = streamTaskInit(pTask, pSnode->pMeta, &pSnode->msgCb, nextProcessVer);
@ -85,7 +71,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
startRsync(); startRsync();
pSnode->msgCb = pOption->msgCb; pSnode->msgCb = pOption->msgCb;
pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback);
if (pSnode->pMeta == NULL) { if (pSnode->pMeta == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto FAIL; goto FAIL;
@ -136,6 +122,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) {
return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg); return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg);
case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg); return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg);
case TDMT_MND_STREAM_CHKPT_REPORT_RSP:
return tqStreamProcessChkptReportRsp(pSnode->pMeta, pMsg);
case TDMT_STREAM_RETRIEVE_TRIGGER: case TDMT_STREAM_RETRIEVE_TRIGGER:
return tqStreamTaskProcessRetrieveTriggerReq(pSnode->pMeta, pMsg); return tqStreamTaskProcessRetrieveTriggerReq(pSnode->pMeta, pMsg);
case TDMT_STREAM_RETRIEVE_TRIGGER_RSP: case TDMT_STREAM_RETRIEVE_TRIGGER_RSP:

3
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -261,9 +261,10 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqScanWal(STQ* pTq); int32_t tqScanWal(STQ* pTq);
int tqCommit(STQ*); int tqCommit(STQ*);

2
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -299,7 +299,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat
tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id);
pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask); pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask);
pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo();
pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1); pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId, true, -1, -1);
if (!pStreamState) { if (!pStreamState) {
terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN;
return TSDB_CODE_FAILED; return TSDB_CODE_FAILED;

31
source/dnode/vnode/src/tq/tq.c
View File

@ -50,6 +50,9 @@ void tqDestroyTqHandle(void* data) {
if (pData->block != NULL) { if (pData->block != NULL) {
blockDataDestroy(pData->block); blockDataDestroy(pData->block);
} }
if (pData->pRef) {
walCloseRef(pData->pRef->pWal, pData->pRef->refId);
}
} }
static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) {
@ -87,7 +90,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
int32_t tqInitialize(STQ* pTq) { int32_t tqInitialize(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, vgId, -1, tqStartTaskCompleteCallback); pTq->pStreamMeta =
streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback);
if (pTq->pStreamMeta == NULL) { if (pTq->pStreamMeta == NULL) {
return -1; return -1;
} }
@ -571,9 +575,6 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
taosMsleep(10); taosMsleep(10);
continue; continue;
} }
if (pHandle->pRef) {
walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId);
}
tqUnregisterPushHandle(pTq, pHandle); tqUnregisterPushHandle(pTq, pHandle);
@ -658,12 +659,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
taosRLockLatch(&pTq->lock); taosRLockLatch(&pTq->lock);
ret = tqMetaGetHandle(pTq, req.subKey); ret = tqMetaGetHandle(pTq, req.subKey);
taosRUnLockLatch(&pTq->lock); taosRUnLockLatch(&pTq->lock);
if (ret < 0) { if (ret < 0) {
break; break;
} }
} }
if (pHandle == NULL) { if (pHandle == NULL) {
if (req.oldConsumerId != -1) { if (req.oldConsumerId != -1) {
tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64,
@ -708,14 +707,16 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
} }
} }
end: end:
tDecoderClear(&dc); tDecoderClear(&dc);
return ret; return ret;
} }
static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); } static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); }
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessVer) {
STQ* pTq = (STQ*) pTqObj;
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
tqDebug("s-task:0x%x start to build task", pTask->id.taskId); tqDebug("s-task:0x%x start to build task", pTask->id.taskId);
@ -1012,16 +1013,6 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
} }
int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) { int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) {
int32_t vgId = TD_VID(pTq->pVnode);
SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg;
// if (!pTq->pVnode->restored) {
// tqDebug("vgId:%d update-checkpoint-info msg received during restoring, checkpointId:%" PRId64
// ", transId:%d s-task:0x%x ignore it",
// vgId, pReq->checkpointId, pReq->transId, pReq->taskId);
// return TSDB_CODE_SUCCESS;
// }
return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, msg, msgLen); return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, msg, msgLen);
} }
@ -1278,3 +1269,7 @@ int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) {
int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg); return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg);
} }
int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg) {
return tqStreamProcessChkptReportRsp(pTq->pStreamMeta, pMsg);
}

3
source/dnode/vnode/src/tq/tqMeta.c
View File

@ -352,7 +352,6 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN);
handle->consumerId = req->newConsumerId; handle->consumerId = req->newConsumerId;
handle->epoch = -1;
handle->execHandle.subType = req->subType; handle->execHandle.subType = req->subType;
handle->fetchMeta = req->withMeta; handle->fetchMeta = req->withMeta;
@ -371,7 +370,7 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){
if(buildHandle(pTq, handle) < 0){ if(buildHandle(pTq, handle) < 0){
return -1; return -1;
} }
tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId); tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer);
return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle));
} }

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -87,7 +87,7 @@ int tqUnregisterPushHandle(STQ* pTq, void *handle) {
int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey));
tqInfo("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); tqInfo("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId);
if(pHandle->msg != NULL) { if(ret == 0 && pHandle->msg != NULL) {
// tqPushDataRsp(pHandle, vgId); // tqPushDataRsp(pHandle, vgId);
tqPushEmptyDataRsp(pHandle, vgId); tqPushEmptyDataRsp(pHandle, vgId);

7
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -605,14 +605,15 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
numOfTasks = taosArrayGetSize(pMeta->pTaskList); numOfTasks = taosArrayGetSize(pMeta->pTaskList);
for (int32_t i = 0; i < numOfTasks; ++i) { for (int32_t i = 0; i < numOfTasks; ++i) {
STaskId* pId = taosArrayGet(pMeta->pTaskList, i); SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i);
if (pId->streamId != streamId) { if (pId->streamId != streamId) {
continue; continue;
} }
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId};
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (ppTask == NULL) { if (ppTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%" PRIx64 " in retrieving progress", pMeta->vgId, pId->taskId); tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId);
continue; continue;
} }

53
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -30,37 +30,26 @@ typedef struct SMStreamCheckpointReadyRspMsg {
static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg);
static STaskId replaceStreamTaskId(SStreamTask* pTask) {
ASSERT(pTask->info.fillHistory);
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
pTask->id.streamId = pTask->streamTaskId.streamId;
pTask->id.taskId = pTask->streamTaskId.taskId;
return id;
}
static void restoreStreamTaskId(SStreamTask* pTask, STaskId* pId) {
ASSERT(pTask->info.fillHistory);
pTask->id.taskId = pId->taskId;
pTask->id.streamId = pId->streamId;
}
int32_t tqExpandStreamTask(SStreamTask* pTask) { int32_t tqExpandStreamTask(SStreamTask* pTask) {
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
STaskId taskId = {0};
int64_t st = taosGetTimestampMs(); int64_t st = taosGetTimestampMs();
int64_t streamId = 0;
int32_t taskId = 0;
tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId); tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId);
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
taskId = replaceStreamTaskId(pTask); streamId = pTask->streamTaskId.streamId;
taskId = pTask->streamTaskId.taskId;
} else {
streamId = pTask->id.streamId;
taskId = pTask->id.taskId;
} }
// sink task does not need the pState // sink task does not need the pState
if (pTask->info.taskLevel != TASK_LEVEL__SINK) { if (pTask->info.taskLevel != TASK_LEVEL__SINK) {
pTask->pState = streamStateOpen(pMeta->path, pTask, false, -1, -1); pTask->pState = streamStateOpen(pMeta->path, pTask, false, streamId, taskId, -1, -1);
if (pTask->pState == NULL) { if (pTask->pState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId); tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId);
return -1; return -1;
@ -69,10 +58,6 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) {
} }
} }
if (pTask->info.fillHistory) {
restoreStreamTaskId(pTask, &taskId);
}
SReadHandle handle = { SReadHandle handle = {
.checkpointId = pTask->chkInfo.checkpointId, .checkpointId = pTask->chkInfo.checkpointId,
.pStateBackend = pTask->pState, .pStateBackend = pTask->pState,
@ -185,7 +170,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
SStreamTask* pTask = *ppTask; SStreamTask* pTask = *ppTask;
const char* idstr = pTask->id.idStr; const char* idstr = pTask->id.idStr;
if (pMeta->updateInfo.transId != req.transId) { if ((pMeta->updateInfo.transId != req.transId) && (pMeta->updateInfo.transId != -1)) {
if (req.transId < pMeta->updateInfo.transId) { if (req.transId < pMeta->updateInfo.transId) {
tqError("s-task:%s vgId:%d disorder update nodeEp msg recv, discarded, newest transId:%d, recv:%d", idstr, vgId, tqError("s-task:%s vgId:%d disorder update nodeEp msg recv, discarded, newest transId:%d, recv:%d", idstr, vgId,
pMeta->updateInfo.transId, req.transId); pMeta->updateInfo.transId, req.transId);
@ -197,10 +182,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
} else { } else {
tqInfo("s-task:%s vgId:%d receive new trans to update nodeEp msg from mnode, transId:%d, prev transId:%d", idstr, tqInfo("s-task:%s vgId:%d receive new trans to update nodeEp msg from mnode, transId:%d, prev transId:%d", idstr,
vgId, req.transId, pMeta->updateInfo.transId); vgId, req.transId, pMeta->updateInfo.transId);
// info needs to be kept till the new trans to update the nodeEp arrived. // info needs to be kept till the new trans to update the nodeEp arrived.
taosHashClear(pMeta->updateInfo.pTasks); streamMetaInitUpdateTaskList(pMeta, req.transId);
pMeta->updateInfo.transId = req.transId;
} }
} else { } else {
tqDebug("s-task:%s vgId:%d recv trans to update nodeEp from mnode, transId:%d", idstr, vgId, req.transId); tqDebug("s-task:%s vgId:%d recv trans to update nodeEp from mnode, transId:%d", idstr, vgId, req.transId);
@ -280,6 +263,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM
// persist to disk // persist to disk
} }
streamMetaClearUpdateTaskList(pMeta);
if (!restored) { if (!restored) {
tqDebug("vgId:%d vnode restore not completed, not start the tasks, clear the start after nodeUpdate flag", vgId); tqDebug("vgId:%d vnode restore not completed, not start the tasks, clear the start after nodeUpdate flag", vgId);
pMeta->startInfo.tasksWillRestart = 0; pMeta->startInfo.tasksWillRestart = 0;
@ -362,6 +347,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
pRsp->upstreamNodeId = htonl(pRsp->upstreamNodeId);
pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId);
pRsp->streamId = htobe64(pRsp->streamId); pRsp->streamId = htobe64(pRsp->streamId);
pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId);
@ -369,6 +355,9 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
pRsp->stage = htobe64(pRsp->stage); pRsp->stage = htobe64(pRsp->stage);
pRsp->msgId = htonl(pRsp->msgId); pRsp->msgId = htonl(pRsp->msgId);
tqDebug("s-task:0x%x vgId:%d recv dispatch-rsp from 0x%x vgId:%d", pRsp->upstreamTaskId, pRsp->upstreamNodeId,
pRsp->downstreamTaskId, pRsp->downstreamNodeId);
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId);
if (pTask) { if (pTask) {
streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamProcessDispatchRsp(pTask, pRsp, pMsg->code);
@ -414,7 +403,9 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
tCleanupStreamRetrieveReq(&req); tCleanupStreamRetrieveReq(&req);
return code;
// always return success, to disable the auto rsp
return TSDB_CODE_SUCCESS;
} }
int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) {
@ -629,8 +620,8 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
// drop the related fill-history task firstly // drop the related fill-history task firstly
if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { if (hTaskId.taskId != 0 && hTaskId.streamId != 0) {
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId);
streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId);
} }
// drop the stream task now // drop the stream task now
@ -646,8 +637,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen
} }
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
// tqStreamRemoveTaskBackend(pMeta, &id);
return 0; return 0;
} }
@ -1078,6 +1067,8 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d
int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); }
int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {return doProcessDummyRspMsg(pMeta, pMsg);}
int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {
SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont;

16
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1070,10 +1070,10 @@ static int32_t tsdbCacheUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, SArray
rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch; rocksdb_writebatch_t *wb = pTsdb->rCache.writebatch;
for (int i = 0; i < num_keys; ++i) { for (int i = 0; i < num_keys; ++i) {
SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i]; SIdxKey *idxKey = &((SIdxKey *)TARRAY_DATA(remainCols))[i];
SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i); SLastUpdateCtx *updCtx = (SLastUpdateCtx *)taosArrayGet(updCtxArray, i);
SRowKey *pRowKey = &updCtx->tsdbRowKey.key; SRowKey *pRowKey = &updCtx->tsdbRowKey.key;
SColVal *pColVal = &updCtx->colVal; SColVal *pColVal = &updCtx->colVal;
SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]); SLastCol *pLastCol = tsdbCacheDeserialize(values_list[i], values_list_sizes[i]);
SLastCol *PToFree = pLastCol; SLastCol *PToFree = pLastCol;
@ -1156,9 +1156,9 @@ int32_t tsdbCacheRowFormatUpdate(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, int6
// 1. prepare last // 1. prepare last
TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version}; TSDBROW lRow = {.type = TSDBROW_ROW_FMT, .pTSRow = aRow[nRow - 1], .version = version};
STSchema *pTSchema = NULL; STSchema *pTSchema = NULL;
int32_t sver = TSDBROW_SVERSION(&lRow); int32_t sver = TSDBROW_SVERSION(&lRow);
SArray *ctxArray = NULL; SArray *ctxArray = NULL;
SSHashObj *iColHash = NULL; SSHashObj *iColHash = NULL;
code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema); code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, sver, &pTSchema);
@ -1577,7 +1577,7 @@ static int32_t tsdbCacheLoadFromRaw(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArr
if (/*!pTmpColArray*/ lastTmpIndexArray && !lastTmpColArray) { if (/*!pTmpColArray*/ lastTmpIndexArray && !lastTmpColArray) {
continue; continue;
} }
if (/*!pTmpColArray*/ lastrowTmpIndexArray && lastrowTmpColArray) { if (/*!pTmpColArray*/ lastrowTmpIndexArray && !lastrowTmpColArray) {
continue; continue;
} }
@ -2772,7 +2772,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
} }
state->state = SFSNEXTROW_INDEXLIST; state->state = SFSNEXTROW_INDEXLIST;
state->iBrinIndex = indexSize; state->iBrinIndex = 1;
} }
_check_stt_data: _check_stt_data:

24
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -46,6 +46,12 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList,
} }
pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk));
if (pLoadInfo->aSttBlk == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFreeClear(pLoadInfo);
return NULL;
}
pLoadInfo->pSchema = pSchema; pLoadInfo->pSchema = pSchema;
pLoadInfo->colIds = colList; pLoadInfo->colIds = colList;
pLoadInfo->numOfCols = numOfCols; pLoadInfo->numOfCols = numOfCols;
@ -107,15 +113,21 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa
SArray *pList = taosArrayGetP(pLDataIterArray, i); SArray *pList = taosArrayGetP(pLDataIterArray, i);
for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) {
SLDataIter *pIter = taosArrayGetP(pList, j); SLDataIter *pIter = taosArrayGetP(pList, j);
if (pIter->pBlockLoadInfo == NULL) {
continue;
}
SSttBlockLoadCostInfo* pCost = &pIter->pBlockLoadInfo->cost;
if (pLoadCost != NULL) { if (pLoadCost != NULL) {
pLoadCost->loadBlocks += pIter->pBlockLoadInfo->cost.loadBlocks; pLoadCost->loadBlocks += pCost->loadBlocks;
pLoadCost->loadStatisBlocks += pIter->pBlockLoadInfo->cost.loadStatisBlocks; pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks;
pLoadCost->blockElapsedTime += pIter->pBlockLoadInfo->cost.blockElapsedTime; pLoadCost->blockElapsedTime += pCost->blockElapsedTime;
pLoadCost->statisElapsedTime += pIter->pBlockLoadInfo->cost.statisElapsedTime; pLoadCost->statisElapsedTime += pCost->statisElapsedTime;
} }
destroyLDataIter(pIter); destroyLDataIter(pIter);
} }
taosArrayDestroy(pList); taosArrayDestroy(pList);
} }
@ -903,6 +915,10 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF
if (pLoadInfo == NULL) { if (pLoadInfo == NULL) {
pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pLoadInfo == NULL) {
code = terrno;
goto _end;
}
} }
memset(pIter, 0, sizeof(SLDataIter)); memset(pIter, 0, sizeof(SLDataIter));

19
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -2240,7 +2240,8 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan
}; };
SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))}; SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))};
int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return false; return false;
} }
@ -4257,9 +4258,6 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi
blockDataEnsureCapacity(pResBlock, capacity); blockDataEnsureCapacity(pResBlock, capacity);
} }
// for debug purpose
// capacity = 7;
int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _err; goto _err;
@ -4899,7 +4897,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
} }
int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) {
SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg;
int32_t code = 0; int32_t code = 0;
*allHave = false; *allHave = false;
@ -4954,7 +4952,13 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
if (pResBlock->pBlockAgg == NULL) { if (pResBlock->pBlockAgg == NULL) {
size_t num = taosArrayGetSize(pResBlock->pDataBlock); size_t num = taosArrayGetSize(pResBlock->pDataBlock);
pResBlock->pBlockAgg = taosMemoryCalloc(num, POINTER_BYTES); pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
if (pResBlock->pBlockAgg == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for(int i = 0; i < num; ++i) {
pResBlock->pBlockAgg[i].colId = -1;
}
} }
// do fill all null column value SMA info // do fill all null column value SMA info
@ -4966,13 +4970,12 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock,
while (j < numOfCols && i < size) { while (j < numOfCols && i < size) {
SColumnDataAgg* pAgg = &pSup->colAggArray.data[i]; SColumnDataAgg* pAgg = &pSup->colAggArray.data[i];
if (pAgg->colId == pSup->colId[j]) { if (pAgg->colId == pSup->colId[j]) {
pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg; pResBlock->pBlockAgg[pSup->slotId[j]] = *pAgg;
i += 1; i += 1;
j += 1; j += 1;
} else if (pAgg->colId < pSup->colId[j]) { } else if (pAgg->colId < pSup->colId[j]) {
i += 1; i += 1;
} else if (pSup->colId[j] < pAgg->colId) { } else if (pSup->colId[j] < pAgg->colId) {
pResBlock->pBlockAgg[pSup->slotId[j]] = NULL;
*allHave = false; *allHave = false;
j += 1; j += 1;
} }

4
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -985,6 +985,10 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra
if (pIter->pBlockLoadInfo == NULL) { if (pIter->pBlockLoadInfo == NULL) {
pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols);
if (pIter->pBlockLoadInfo == NULL) {
tsdbError("failed to create block load info, code: out of memory, %s", pstr);
continue;
}
} }
// load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file

31
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -38,6 +38,34 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
return TARRAY2_APPEND(&rtner->fopArr, op); return TARRAY2_APPEND(&rtner->fopArr, op);
} }
static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) {
int64_t total = 0;
int64_t interval = 1000; // 1s
int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX;
int64_t offset = 0;
int64_t remain = size;
while (remain > 0) {
int64_t n;
int64_t last = taosGetTimestampMs();
if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) {
return -1;
}
total += n;
remain -= n;
if (remain > 0) {
int64_t elapsed = taosGetTimestampMs() - last;
if (elapsed < interval) {
taosMsleep(interval - elapsed);
}
}
}
return total;
}
static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) { static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
@ -98,7 +126,8 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile
if (fdTo == NULL) code = terrno; if (fdTo == NULL) code = terrno;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
int64_t n = taosFSendFile(fdTo, fdFrom, 0, tsdbLogicToFileSize(from->f->size, rtner->szPage)); int64_t n = tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage),
tsRetentionSpeedLimitMB);
if (n < 0) { if (n < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);

36
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -120,7 +120,8 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
} }
if (metaRsp.pSchemaExt) { if (metaRsp.pSchemaExt) {
code = fillTableColCmpr(&mer1, metaRsp.pSchemaExt, metaRsp.numOfColumns); SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1;
code = fillTableColCmpr(pReader, metaRsp.pSchemaExt, metaRsp.numOfColumns);
if (code < 0) { if (code < 0) {
code = TSDB_CODE_INVALID_MSG; code = TSDB_CODE_INVALID_MSG;
goto _exit; goto _exit;
@ -254,15 +255,18 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols);
} }
if (useCompress(cfgRsp.tableType)) { // if (useCompress(cfgRsp.tableType)) {
SColCmprWrapper *pColCmpr = &mer1.me.colCmpr;
for (int32_t i = 0; i < cfgRsp.numOfColumns; i++) { SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1;
SColCmpr *pCmpr = &pColCmpr->pColCmpr[i]; SColCmprWrapper *pColCmpr = &pReader->me.colCmpr;
SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i;
pSchExt->colId = pCmpr->id; for (int32_t i = 0; i < cfgRsp.numOfColumns; i++) {
pSchExt->compress = pCmpr->alg; SColCmpr *pCmpr = &pColCmpr->pColCmpr[i];
} SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i;
pSchExt->colId = pCmpr->id;
pSchExt->compress = pCmpr->alg;
} }
//}
// encode and send response // encode and send response
rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp);
@ -752,13 +756,13 @@ int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64
return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid); return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
} }
int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct) { int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) {
int32_t code = 0; int32_t code = 0;
SStreamProgressReq req; SStreamProgressReq req;
SStreamProgressRsp rsp = {0}; SStreamProgressRsp rsp = {0};
SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0}; SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0};
char * buf = NULL; char *buf = NULL;
int32_t rspLen = 0; int32_t rspLen = 0;
code = tDeserializeStreamProgressReq(pMsg->pCont, pMsg->contLen, &req); code = tDeserializeStreamProgressReq(pMsg->pCont, pMsg->contLen, &req);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -854,6 +854,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg); return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
case TDMT_VND_GET_STREAM_PROGRESS: case TDMT_VND_GET_STREAM_PROGRESS:
return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg); return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
case TDMT_MND_STREAM_CHKPT_REPORT_RSP:
return tqProcessTaskChkptReportRsp(pVnode->pTq, pMsg);
default: default:
vError("unknown msg type:%d in stream queue", pMsg->msgType); vError("unknown msg type:%d in stream queue", pMsg->msgType);
return TSDB_CODE_APP_ERROR; return TSDB_CODE_APP_ERROR;

11
source/libs/command/src/command.c
View File

@ -26,7 +26,7 @@
extern SConfig* tsCfg; extern SConfig* tsCfg;
static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) {
size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
*pRsp = taosMemoryCalloc(1, rspSize); *pRsp = taosMemoryCalloc(1, rspSize);
if (NULL == *pRsp) { if (NULL == *pRsp) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -36,11 +36,16 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe
(*pRsp)->completed = 1; (*pRsp)->completed = 1;
(*pRsp)->precision = 0; (*pRsp)->precision = 0;
(*pRsp)->compressed = 0; (*pRsp)->compressed = 0;
(*pRsp)->compLen = 0;
(*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows);
(*pRsp)->numOfCols = htonl(numOfCols); (*pRsp)->numOfCols = htonl(numOfCols);
int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, numOfCols);
SET_PAYLOAD_LEN((*pRsp)->data, len, len);
int32_t payloadLen = len + PAYLOAD_PREFIX_LEN;
(*pRsp)->payloadLen = htonl(payloadLen);
(*pRsp)->compLen = htonl(payloadLen);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

9
source/libs/command/src/explain.c
View File

@ -70,7 +70,6 @@ char* qExplainGetTimerangeTargetStr(int32_t target) {
return targetName[target]; return targetName[target];
} }
void qExplainFreeResNode(SExplainResNode *resNode) { void qExplainFreeResNode(SExplainResNode *resNode) {
if (NULL == resNode) { if (NULL == resNode) {
return; return;
@ -1942,7 +1941,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
pBlock->info.rows = rowNum; pBlock->info.rows = rowNum;
int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
if (NULL == rsp) { if (NULL == rsp) {
@ -1954,9 +1953,13 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
rsp->completed = 1; rsp->completed = 1;
rsp->numOfRows = htobe64((int64_t)rowNum); rsp->numOfRows = htobe64((int64_t)rowNum);
int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); int32_t len = blockEncode(pBlock, rsp->data + PAYLOAD_PREFIX_LEN, taosArrayGetSize(pBlock->pDataBlock));
rsp->compLen = htonl(len); rsp->compLen = htonl(len);
rsp->payloadLen = htonl(len);
rsp->compressed = 0;
SET_PAYLOAD_LEN(rsp->data, len, len);
blockDataDestroy(pBlock); blockDataDestroy(pBlock);

3
source/libs/executor/inc/dataSinkInt.h
View File

@ -25,7 +25,6 @@ extern "C" {
#include "storageapi.h" #include "storageapi.h"
#include "tcommon.h" #include "tcommon.h"
struct SDataSink;
struct SDataSinkHandle; struct SDataSinkHandle;
typedef struct SDataSinkManager { typedef struct SDataSinkManager {
@ -36,7 +35,7 @@ typedef struct SDataSinkManager {
typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue);
typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds); typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds);
typedef void (*FReset)(struct SDataSinkHandle* pHandle); typedef void (*FReset)(struct SDataSinkHandle* pHandle);
typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd); typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd);
typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput);
typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle);
typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size); typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size);

4
source/libs/executor/inc/executorInt.h
View File

@ -624,6 +624,8 @@ typedef struct SDataGroupInfo {
uint64_t groupId; uint64_t groupId;
int64_t numOfRows; int64_t numOfRows;
SArray* pPageList; SArray* pPageList;
SArray* blockForNotLoaded; // SSDataBlock that data is not loaded
int32_t offsetForNotLoaded; // read offset for SSDataBlock that data is not loaded
} SDataGroupInfo; } SDataGroupInfo;
typedef struct SWindowRowsSup { typedef struct SWindowRowsSup {
@ -861,7 +863,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData,
void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId); void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId);
void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer); void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset);
void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput);
SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData,

5
source/libs/executor/src/aggregateoperator.c
View File

@ -406,7 +406,10 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin
} }
} }
setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); int32_t ret = setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset);
if (ret != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, ret);
}
} }
// a new buffer page for each table. Needs to opt this design // a new buffer page for each table. Needs to opt this design

5
source/libs/executor/src/dataDeleter.c
View File

@ -154,7 +154,7 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
taosThreadMutexUnlock(&pDeleter->mutex); taosThreadMutexUnlock(&pDeleter->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle;
if (taosQueueEmpty(pDeleter->pDataBlocks)) { if (taosQueueEmpty(pDeleter->pDataBlocks)) {
*pQueryEnd = pDeleter->queryEnd; *pQueryEnd = pDeleter->queryEnd;
@ -171,6 +171,8 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData; SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData;
*pLen = pEntry->dataLen; *pLen = pEntry->dataLen;
*pRawLen = pEntry->dataLen;
*pQueryEnd = pDeleter->queryEnd; *pQueryEnd = pDeleter->queryEnd;
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, qDebug("got data len %" PRId64 ", row num %d in sink", *pLen,
((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows);
@ -186,6 +188,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
pOutput->queryEnd = pDeleter->queryEnd; pOutput->queryEnd = pDeleter->queryEnd;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData); SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
pDeleter->pParam->pUidList = NULL; pDeleter->pParam->pUidList = NULL;

64
source/libs/executor/src/dataDispatcher.c
View File

@ -31,6 +31,7 @@ typedef struct SDataDispatchBuf {
} SDataDispatchBuf; } SDataDispatchBuf;
typedef struct SDataCacheEntry { typedef struct SDataCacheEntry {
int32_t rawLen;
int32_t dataLen; int32_t dataLen;
int32_t numOfRows; int32_t numOfRows;
int32_t numOfCols; int32_t numOfCols;
@ -48,6 +49,8 @@ typedef struct SDataDispatchHandle {
bool queryEnd; bool queryEnd;
uint64_t useconds; uint64_t useconds;
uint64_t cachedSize; uint64_t cachedSize;
void* pCompressBuf;
int32_t bufSize;
TdThreadMutex mutex; TdThreadMutex mutex;
} SDataDispatchHandle; } SDataDispatchHandle;
@ -63,22 +66,60 @@ typedef struct SDataDispatchHandle {
static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
int32_t numOfCols = 0; int32_t numOfCols = 0;
SNode* pNode; SNode* pNode;
FOREACH(pNode, pHandle->pSchema->pSlots) { FOREACH(pNode, pHandle->pSchema->pSlots) {
SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode;
if (pSlotDesc->output) { if (pSlotDesc->output) {
++numOfCols; ++numOfCols;
} }
} }
SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
pEntry->compressed = 0; pEntry->compressed = 0;
pEntry->numOfRows = pInput->pData->info.rows; pEntry->numOfRows = pInput->pData->info.rows;
pEntry->numOfCols = numOfCols; pEntry->numOfCols = numOfCols;
pEntry->dataLen = 0; pEntry->dataLen = 0;
pEntry->rawLen = 0;
pBuf->useSize = sizeof(SDataCacheEntry); pBuf->useSize = sizeof(SDataCacheEntry);
pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols);
// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); {
// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); if ((pBuf->allocSize > tsCompressMsgSize) && (tsCompressMsgSize > 0) && pHandle->pManager->cfg.compress) {
if (pHandle->pCompressBuf == NULL) {
// allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size
pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8);
pHandle->bufSize = pBuf->allocSize + 8;
} else {
if (pHandle->bufSize < pBuf->allocSize + 8) {
pHandle->bufSize = pBuf->allocSize + 8;
void* p = taosMemoryRealloc(pHandle->pCompressBuf, pHandle->bufSize);
if (p != NULL) {
pHandle->pCompressBuf = p;
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
qError("failed to prepare compress buf:%d, code: out of memory", pHandle->bufSize);
return;
}
}
}
int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, numOfCols);
int32_t len = tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0);
if (len < dataLen) {
pEntry->compressed = 1;
pEntry->dataLen = len;
pEntry->rawLen = dataLen;
} else { // no need to compress data
pEntry->compressed = 0;
pEntry->dataLen = dataLen;
pEntry->rawLen = dataLen;
memcpy(pEntry->data, pHandle->pCompressBuf, dataLen);
}
} else {
pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols);
pEntry->rawLen = pEntry->dataLen;
}
}
pBuf->useSize += pEntry->dataLen; pBuf->useSize += pEntry->dataLen;
@ -163,7 +204,7 @@ static void resetDispatcher(struct SDataSinkHandle* pHandle) {
taosThreadMutexUnlock(&pDispatcher->mutex); taosThreadMutexUnlock(&pDispatcher->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
if (taosQueueEmpty(pDispatcher->pDataBlocks)) { if (taosQueueEmpty(pDispatcher->pDataBlocks)) {
*pQueryEnd = pDispatcher->queryEnd; *pQueryEnd = pDispatcher->queryEnd;
@ -180,9 +221,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE
SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData; SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData;
*pLen = pEntry->dataLen; *pLen = pEntry->dataLen;
*pRowLen = pEntry->rawLen;
// ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8));
// ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4));
*pQueryEnd = pDispatcher->queryEnd; *pQueryEnd = pDispatcher->queryEnd;
qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, qDebug("got data len %" PRId64 ", row num %d in sink", *pLen,
@ -200,6 +239,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
pOutput->queryEnd = pDispatcher->queryEnd; pOutput->queryEnd = pDispatcher->queryEnd;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData);
memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen);
pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfRows = pEntry->numOfRows;
@ -224,6 +264,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize);
taosMemoryFreeClear(pDispatcher->nextOutput.pData); taosMemoryFreeClear(pDispatcher->nextOutput.pData);
while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { while (!taosQueueEmpty(pDispatcher->pDataBlocks)) {
SDataDispatchBuf* pBuf = NULL; SDataDispatchBuf* pBuf = NULL;
taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
@ -232,7 +273,11 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
taosFreeQitem(pBuf); taosFreeQitem(pBuf);
} }
} }
taosCloseQueue(pDispatcher->pDataBlocks); taosCloseQueue(pDispatcher->pDataBlocks);
taosMemoryFreeClear(pDispatcher->pCompressBuf);
pDispatcher->bufSize = 0;
taosThreadMutexDestroy(&pDispatcher->mutex); taosThreadMutexDestroy(&pDispatcher->mutex);
taosMemoryFree(pDispatcher->pManager); taosMemoryFree(pDispatcher->pManager);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -251,6 +296,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
} }
dispatcher->sink.fPut = putDataBlock; dispatcher->sink.fPut = putDataBlock;
dispatcher->sink.fEndPut = endPut; dispatcher->sink.fEndPut = endPut;
dispatcher->sink.fReset = resetDispatcher; dispatcher->sink.fReset = resetDispatcher;
@ -258,22 +304,24 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD
dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fGetData = getDataBlock;
dispatcher->sink.fDestroy = destroyDataSinker; dispatcher->sink.fDestroy = destroyDataSinker;
dispatcher->sink.fGetCacheSize = getCacheSize; dispatcher->sink.fGetCacheSize = getCacheSize;
dispatcher->pManager = pManager; dispatcher->pManager = pManager;
dispatcher->pSchema = pDataSink->pInputDataBlockDesc; dispatcher->pSchema = pDataSink->pInputDataBlockDesc;
dispatcher->status = DS_BUF_EMPTY; dispatcher->status = DS_BUF_EMPTY;
dispatcher->queryEnd = false; dispatcher->queryEnd = false;
dispatcher->pDataBlocks = taosOpenQueue(); dispatcher->pDataBlocks = taosOpenQueue();
taosThreadMutexInit(&dispatcher->mutex, NULL); taosThreadMutexInit(&dispatcher->mutex, NULL);
if (NULL == dispatcher->pDataBlocks) { if (NULL == dispatcher->pDataBlocks) {
taosMemoryFree(dispatcher); taosMemoryFree(dispatcher);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _return; goto _return;
} }
*pHandle = dispatcher; *pHandle = dispatcher;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
taosMemoryFree(pManager); taosMemoryFree(pManager);
return terrno; return terrno;
} }

2
source/libs/executor/src/dataInserter.c
View File

@ -370,7 +370,7 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
taosThreadMutexUnlock(&pInserter->mutex); taosThreadMutexUnlock(&pInserter->mutex);
} }
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) { static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
SDataInserterHandle* pDispatcher = (SDataInserterHandle*)pHandle; SDataInserterHandle* pDispatcher = (SDataInserterHandle*)pHandle;
*pLen = pDispatcher->submitRes.affectedRows; *pLen = pDispatcher->submitRes.affectedRows;
qDebug("got total affectedRows %" PRId64, *pLen); qDebug("got total affectedRows %" PRId64, *pLen);

7
source/libs/executor/src/dataSinkMgt.c
View File

@ -18,13 +18,14 @@
#include "planner.h" #include "planner.h"
#include "tarray.h" #include "tarray.h"
SDataSinkStat gDataSinkStat = {0}; SDataSinkStat gDataSinkStat = {0};
int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) { int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) {
SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager)); SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager));
if (NULL == pSinkManager) { if (NULL == pSinkManager) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pSinkManager->cfg = *cfg; pSinkManager->cfg = *cfg;
pSinkManager->pAPI = pAPI; pSinkManager->pAPI = pAPI;
@ -75,9 +76,9 @@ void dsReset(DataSinkHandle handle) {
} }
} }
void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd) { void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) {
SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle;
pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd); pHandleImpl->fGetLen(pHandleImpl, pLen, pRawLen, pQueryEnd);
} }
int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) {

3
source/libs/executor/src/eventwindowoperator.c
View File

@ -221,8 +221,7 @@ static int32_t setSingleOutputTupleBufv1(SResultRowInfo* pResultRowInfo, STimeWi
(*pResult)->win = *win; (*pResult)->win = *win;
setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); return setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
return TSDB_CODE_SUCCESS;
} }
static void doEventWindowAggImpl(SEventWindowOperatorInfo* pInfo, SExprSupp* pSup, int32_t startIndex, int32_t endIndex, static void doEventWindowAggImpl(SEventWindowOperatorInfo* pInfo, SExprSupp* pSup, int32_t startIndex, int32_t endIndex,

43
source/libs/executor/src/exchangeoperator.c
View File

@ -41,6 +41,8 @@ typedef struct SSourceDataInfo {
SArray* pSrcUidList; SArray* pSrcUidList;
int32_t srcOpType; int32_t srcOpType;
bool tableSeq; bool tableSeq;
char* decompBuf;
int32_t decompBufSize;
} SSourceDataInfo; } SSourceDataInfo;
static void destroyExchangeOperatorInfo(void* param); static void destroyExchangeOperatorInfo(void* param);
@ -371,7 +373,10 @@ void freeBlock(void* pParam) {
void freeSourceDataInfo(void* p) { void freeSourceDataInfo(void* p) {
SSourceDataInfo* pInfo = (SSourceDataInfo*)p; SSourceDataInfo* pInfo = (SSourceDataInfo*)p;
taosMemoryFreeClear(pInfo->decompBuf);
taosMemoryFreeClear(pInfo->pRsp); taosMemoryFreeClear(pInfo->pRsp);
pInfo->decompBufSize = 0;
} }
void doDestroyExchangeOperatorInfo(void* param) { void doDestroyExchangeOperatorInfo(void* param) {
@ -411,6 +416,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) {
SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp;
pRsp->numOfRows = htobe64(pRsp->numOfRows); pRsp->numOfRows = htobe64(pRsp->numOfRows);
pRsp->compLen = htonl(pRsp->compLen); pRsp->compLen = htonl(pRsp->compLen);
pRsp->payloadLen = htonl(pRsp->payloadLen);
pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->numOfCols = htonl(pRsp->numOfCols);
pRsp->useconds = htobe64(pRsp->useconds); pRsp->useconds = htobe64(pRsp->useconds);
pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks);
@ -673,11 +679,32 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) {
int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo) { int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo) {
SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp;
char* pStart = pRetrieveRsp->data; char* pNextStart = pRetrieveRsp->data;
char* pStart = pNextStart;
int32_t index = 0; int32_t index = 0;
int32_t code = 0; int32_t code = 0;
if (pRetrieveRsp->compressed) { // decompress the data
if (pDataInfo->decompBuf == NULL) {
pDataInfo->decompBuf = taosMemoryMalloc(pRetrieveRsp->payloadLen);
pDataInfo->decompBufSize = pRetrieveRsp->payloadLen;
} else {
if (pDataInfo->decompBufSize < pRetrieveRsp->payloadLen) {
char* p = taosMemoryRealloc(pDataInfo->decompBuf, pRetrieveRsp->payloadLen);
if (p != NULL) {
pDataInfo->decompBuf = p;
pDataInfo->decompBufSize = pRetrieveRsp->payloadLen;
}
}
}
}
while (index++ < pRetrieveRsp->numOfBlocks) { while (index++ < pRetrieveRsp->numOfBlocks) {
SSDataBlock* pb = NULL; SSDataBlock* pb = NULL;
pStart = pNextStart;
if (taosArrayGetSize(pExchangeInfo->pRecycledBlocks) > 0) { if (taosArrayGetSize(pExchangeInfo->pRecycledBlocks) > 0) {
pb = *(SSDataBlock**)taosArrayPop(pExchangeInfo->pRecycledBlocks); pb = *(SSDataBlock**)taosArrayPop(pExchangeInfo->pRecycledBlocks);
blockDataCleanup(pb); blockDataCleanup(pb);
@ -685,6 +712,20 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa
pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
} }
int32_t compLen = *(int32_t*) pStart;
pStart += sizeof(int32_t);
int32_t rawLen = *(int32_t*) pStart;
pStart += sizeof(int32_t);
ASSERT(compLen <= rawLen && compLen != 0);
pNextStart = pStart + compLen;
if (pRetrieveRsp->compressed && (compLen < rawLen)) {
int32_t t = tsDecompressString(pStart, compLen, 1, pDataInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0);
ASSERT(t == rawLen);
pStart = pDataInfo->decompBuf;
}
code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart);
if (code != 0) { if (code != 0) {
taosMemoryFreeClear(pDataInfo->pRsp); taosMemoryFreeClear(pDataInfo->pRsp);

28
source/libs/executor/src/executil.c
View File

@ -2178,9 +2178,25 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
return code; return code;
} }
if (group == NULL || groupByTbname) { if (group == NULL || groupByTbname) {
for (int32_t i = 0; i < numOfTables; i++) { if (tsCountAlwaysReturnValue && QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode) && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); pTableListInfo->remainGroups =
info->groupId = groupByTbname ? info->uid : 0; taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
if (pTableListInfo->remainGroups == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
for (int i = 0; i < numOfTables; i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
info->groupId = info->uid;
taosHashPut(pTableListInfo->remainGroups, &(info->groupId), sizeof(info->groupId), &(info->uid),
sizeof(info->uid));
}
} else {
for (int32_t i = 0; i < numOfTables; i++) {
STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i);
info->groupId = groupByTbname ? info->uid : 0;
}
} }
pTableListInfo->oneTableForEachGroup = groupByTbname; pTableListInfo->oneTableForEachGroup = groupByTbname;
@ -2193,8 +2209,6 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle*
pTableListInfo->numOfOuputGroups = numOfTables; pTableListInfo->numOfOuputGroups = numOfTables;
} else if (groupByTbname && pScanNode->groupOrderScan) { } else if (groupByTbname && pScanNode->groupOrderScan) {
pTableListInfo->numOfOuputGroups = numOfTables; pTableListInfo->numOfOuputGroups = numOfTables;
} else if (groupByTbname && tsCountAlwaysReturnValue && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) {
pTableListInfo->numOfOuputGroups = numOfTables;
} else { } else {
pTableListInfo->numOfOuputGroups = 1; pTableListInfo->numOfOuputGroups = 1;
} }
@ -2354,7 +2368,7 @@ int32_t compKeys(const SArray* pSortGroupCols, const char* oldkeyBuf, int32_t ol
for (int32_t i = 0; i < pSortGroupCols->size; ++i) { for (int32_t i = 0; i < pSortGroupCols->size; ++i) {
const SColumn* pCol = (SColumn*)TARRAY_GET_ELEM(pSortGroupCols, i); const SColumn* pCol = (SColumn*)TARRAY_GET_ELEM(pSortGroupCols, i);
const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId); const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId);
if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId]; if (pBlock->pBlockAgg) pColAgg = &pBlock->pBlockAgg[pCol->slotId];
if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) {
if (isNull[i] != 1) return 1; if (isNull[i] != 1) return 1;
@ -2389,7 +2403,7 @@ int32_t buildKeys(char* keyBuf, const SArray* pSortGroupCols, const SSDataBlock*
const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId); const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId);
if (pCol->slotId > pBlock->pDataBlock->size) continue; if (pCol->slotId > pBlock->pDataBlock->size) continue;
if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId]; if (pBlock->pBlockAgg) pColAgg = &pBlock->pBlockAgg[pCol->slotId];
if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) {
isNull[i] = 1; isNull[i] = 1;

9
source/libs/executor/src/executor.c
View File

@ -286,7 +286,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3
} }
qTaskInfo_t pTaskInfo = NULL; qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(pReaderHandle, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_QUEUE); code = qCreateExecTask(pReaderHandle, vgId, 0, pPlan, &pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_QUEUE);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
nodesDestroyNode((SNode*)pPlan); nodesDestroyNode((SNode*)pPlan);
qDestroyTask(pTaskInfo); qDestroyTask(pTaskInfo);
@ -322,7 +322,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
} }
qTaskInfo_t pTaskInfo = NULL; qTaskInfo_t pTaskInfo = NULL;
code = qCreateExecTask(readers, vgId, taskId, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); code = qCreateExecTask(readers, vgId, taskId, pPlan, &pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_STREAM);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
nodesDestroyNode((SNode*)pPlan); nodesDestroyNode((SNode*)pPlan);
qDestroyTask(pTaskInfo); qDestroyTask(pTaskInfo);
@ -524,7 +524,8 @@ void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam) {
} }
int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan,
qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) { qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, int8_t compressResult, char* sql,
EOPTR_EXEC_MODEL model) {
SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo;
taosThreadOnce(&initPoolOnce, initRefPool); taosThreadOnce(&initPoolOnce, initRefPool);
@ -537,7 +538,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId,
} }
if (handle) { if (handle) {
SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50}; SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50, .compress = compressResult};
void* pSinkManager = NULL; void* pSinkManager = NULL;
code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI, &pSinkManager); code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI, &pSinkManager);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {

Some files were not shown because too many files have changed in this diff Show More