xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into mark/3.0

This commit is contained in:
wangmm0220 2024-02-06 14:02:46 +08:00
parent 5796f411e4 e54db7e74d
commit 871ebfa7ba
153 changed files with 7877 additions and 7109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
docs/en/12-taos-sql/18-escape.md
View File

@ -7,14 +7,14 @@ description: This document describes the usage of escape characters in TDengine.
| Escape Character | **Actual Meaning** |
| :--------------: | ------------------------ |
| `\'` | Single quote ' |
| `\"` | Double quote " |
| \n | Line Break |
| \r | Carriage Return |
| \t | tab |
| `\\` | Back Slash \ |
| `\%` | % see below for details |
| `\_` | \_ see below for details |
| `\'` | Single quote `'` |
| `\"` | Double quote `"` |
| `\n` | Line Break |
| `\r` | Carriage Return |
| `\t` | tab |
| `\\` | Back Slash `\ ` |
| `\%` | `%` see below for details |
| `\_` | `_` see below for details |
## Restrictions
@ -22,5 +22,5 @@ description: This document describes the usage of escape characters in TDengine.
- Identifier without ``: Error will be returned because identifier must be constituted of digits, ASCII characters or underscore and can't be started with digits
- Identifier quoted with ``: Original content is kept, no escaping
2. If there are escape characters in values
- The escape characters will be escaped as the above table. If the escape character doesn't match any supported one, the escape character "\" will be ignored.
- "%" and "\_" are used as wildcards in `like`. `\%` and `\_` should be used to represent literal "%" and "\_" in `like`,. If `\%` and `\_` are used out of `like` context, the evaluation result is "`\%`"and "`\_`", instead of "%" and "\_".
- The escape characters will be escaped as the above table. If the escape character doesn't match any supported one, the escape character `\ ` will be ignored(`\x` remaining).
- `%` and `_` are used as wildcards in `like`. `\%` and `\_` should be used to represent literal `%` and `_` in `like`. If `\%` and `\_` are used out of `like` context, the evaluation result is `\%` and `\_`, instead of `%` and `_`.

21
docs/zh/12-taos-sql/18-escape.md
View File

@ -8,16 +8,15 @@ description: TDengine 中使用转义字符的详细规则
| 字符序列 | **代表的字符** |
| :------: | -------------- |
| `\'` | 单引号' |
| `\"` | 双引号" |
| \n | 换行符 |
| \r | 回车符 |
| \t | tab 符 |
| `\\` | 斜杠\ |
| `\%` | % 规则见下 |
| `\_` | \_ 规则见下 |
| `\'` | 单引号`'` |
| `\"` | 双引号`"` |
| `\n` | 换行符 |
| `\r` | 回车符 |
| `\t` | tab 符 |
| `\\` | 斜杠 `\ ` |
| `\%` | `%` 规则见下 |
| `\_` | `_` 规则见下 |
:::
## 转义字符使用规则
@ -25,5 +24,5 @@ description: TDengine 中使用转义字符的详细规则
1. 普通标识符： 直接提示错误的标识符，因为标识符规定必须是数字、字母和下划线，并且不能以数字开头。
2. 反引号``标识符： 保持原样，不转义
2. 数据里有转义字符
1. 遇到上面定义的转义字符会转义（%和\_见下面说明），如果没有匹配的转义字符会忽略掉转义符\。
2. 对于%和\_，因为在 like 里这两个字符是通配符，所以在模式匹配 like 里用`\%`%和`\_`表示字符里本身的%和\_，如果在 like 模式匹配上下文之外使用`\%`或`\_`，则它们的计算结果为字符串`\%`和`\_`，而不是%和\_。
1. 遇到上面定义的转义字符会转义（`%`和`_`见下面说明），如果没有匹配的转义字符会忽略掉转义符`\ `（`\x`保持原样）。
2. 对于`%`和`_`，因为在`like`里这两个字符是通配符，所以在模式匹配`like`里用`\%`和`\_`表示字符里本身的`%`和`_`，如果在`like`模式匹配上下文之外使用`\%`或`\_`，则它们的计算结果为字符串`\%`和`\_`，而不是`%`和`_`。

8
include/common/systable.h
View File

@ -52,6 +52,9 @@ extern "C" {
#define TSDB_INS_TABLE_VIEWS "ins_views"
#define TSDB_INS_TABLE_COMPACTS "ins_compacts"
#define TSDB_INS_TABLE_COMPACT_DETAILS "ins_compact_details"
#define TSDB_INS_TABLE_GRANTS_FULL "ins_grants_full"
#define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs"
#define TSDB_INS_TABLE_MACHINES "ins_machines"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas"
@ -62,6 +65,11 @@ extern "C" {
#define TSDB_PERFS_TABLE_TRANS "perf_trans"
#define TSDB_PERFS_TABLE_APPS "perf_apps"
#define TSDB_AUDIT_DB "audit"
#define TSDB_AUDIT_STB_OPERATION "operations"
#define TSDB_AUDIT_CTB_OPERATION "t_operations_"
#define TSDB_AUDIT_CTB_OPERATION_LEN 13
typedef struct SSysDbTableSchema {
const char* name;
const int32_t type;

1
include/common/tcommon.h
View File

@ -206,6 +206,7 @@ typedef struct SDataBlockInfo {
int16_t hasVarCol;
int16_t dataLoad; // denote if the data is loaded or not
uint8_t scanFlag;
bool blankFill;
// TODO: optimize and remove following
int64_t version; // used for stream, and need serialization

2
include/common/tglobal.h
View File

@ -232,7 +232,7 @@ struct SConfig *taosGetCfg();
void taosSetAllDebugFlag(int32_t flag);
void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal);
void taosLocalCfgForbiddenToChange(char *name, bool *forbidden);
int8_t taosGranted();
int8_t taosGranted(int8_t type);
#ifdef __cplusplus
}

79
include/common/tgrant.h
View File

@ -22,15 +22,17 @@ extern "C" {
#include "os.h"
#include "taoserror.h"
#ifdef GRANTS_CFG
#include "tgrantCfg.h"
#endif
#include "tdef.h"
#ifndef GRANTS_COL_MAX_LEN
#define GRANTS_COL_MAX_LEN 196
#endif
#define GRANT_HEART_BEAT_MIN 2
#define GRANT_ACTIVE_CODE "activeCode"
#define GRANT_FLAG_ALL (0x01)
#define GRANT_FLAG_AUDIT (0x02)
#define GRANT_FLAG_VIEW (0x04)
typedef enum {
TSDB_GRANT_ALL,
@ -48,14 +50,36 @@ typedef enum {
TSDB_GRANT_CPU_CORES,
TSDB_GRANT_STABLE,
TSDB_GRANT_TABLE,
TSDB_GRANT_SUBSCRIBE,
TSDB_GRANT_SUBSCRIPTION,
TSDB_GRANT_AUDIT,
TSDB_GRANT_CSV,
TSDB_GRANT_VIEW,
TSDB_GRANT_MULTI_TIER,
TSDB_GRANT_BACKUP_RESTORE,
} EGrantType;
int32_t grantCheck(EGrantType grant);
int32_t grantCheck(EGrantType grant); // less
int32_t grantCheckLE(EGrantType grant); // less or equal
char* tGetMachineId();
#ifndef TD_UNIQ_GRANT
int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type);
#endif
#ifndef GRANTS_CFG
// #ifndef GRANTS_CFG
#ifdef TD_ENTERPRISE
#ifdef TD_UNIQ_GRANT
#define GRANTS_SCHEMA \
static const SSysDbTableSchema grantsSchema[] = { \
{.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
}
#else
#define GRANTS_SCHEMA \
static const SSysDbTableSchema grantsSchema[] = { \
{.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
@ -79,33 +103,28 @@ int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, ch
{.name = "influxdb", .bytes = GRANTS_COL_MAX_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "mqtt", .bytes = GRANTS_COL_MAX_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
}
#endif
#else
#define GRANTS_SCHEMA \
static const SSysDbTableSchema grantsSchema[] = { \
{.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "storage", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "databases", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "users", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "accounts", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "connections", .bytes = 11 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "streams", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "cpu_cores", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "speed", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "querytime", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
#define GRANTS_SCHEMA \
static const SSysDbTableSchema grantsSchema[] = { \
{.name = "version", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "state", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
{.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \
}
#endif
#define GRANT_CFG_ADD
#define GRANT_CFG_SET
#define GRANT_CFG_GET
#define GRANT_CFG_CHECK
#define GRANT_CFG_SKIP
#define GRANT_CFG_DECLARE
#define GRANT_CFG_EXTERN
#endif
// #define GRANT_CFG_ADD
// #define GRANT_CFG_SET
// #define GRANT_CFG_GET
// #define GRANT_CFG_CHECK
// #define GRANT_CFG_SKIP
// #define GRANT_CFG_DECLARE
// #define GRANT_CFG_EXTERN
// #endif
#ifdef __cplusplus
}

30
include/common/tmsg.h
View File

@ -147,6 +147,9 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_COMPACT,
TSDB_MGMT_TABLE_COMPACT_DETAIL,
TSDB_MGMT_TABLE_GRANTS_FULL,
TSDB_MGMT_TABLE_GRANTS_LOGS,
TSDB_MGMT_TABLE_MACHINES,
TSDB_MGMT_TABLE_MAX,
} EShowType;
@ -298,7 +301,8 @@ typedef enum ENodeType {
QUERY_NODE_SYNCDB_STMT,
QUERY_NODE_GRANT_STMT,
QUERY_NODE_REVOKE_STMT,
// placeholder for [152, 180]
QUERY_NODE_ALTER_CLUSTER_STMT,
// placeholder for [153, 180]
QUERY_NODE_SHOW_CREATE_VIEW_STMT = 181,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT,
@ -359,6 +363,9 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_VIEWS_STMT,
QUERY_NODE_SHOW_COMPACTS_STMT,
QUERY_NODE_SHOW_COMPACT_DETAILS_STMT,
QUERY_NODE_SHOW_GRANTS_FULL_STMT,
QUERY_NODE_SHOW_GRANTS_LOGS_STMT,
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -1557,9 +1564,11 @@ typedef struct {
int64_t updateTime;
float numOfCores;
int32_t numOfSupportVnodes;
int32_t numOfDiskCfg;
int64_t memTotal;
int64_t memAvail;
char dnodeEp[TSDB_EP_LEN];
char machineId[TSDB_MACHINE_ID_LEN + 1];
SMnodeLoad mload;
SQnodeLoad qload;
SClusterCfg clusterCfg;
@ -1600,6 +1609,7 @@ typedef struct {
SEp ep;
char active[TSDB_ACTIVE_KEY_LEN];
char connActive[TSDB_CONN_ACTIVE_KEY_LEN];
char machineId[TSDB_MACHINE_ID_LEN + 1];
} SDnodeInfo;
typedef struct {
@ -2031,6 +2041,17 @@ int32_t tSerializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
int32_t tDeserializeSExplainRsp(void* buf, int32_t bufLen, SExplainRsp* pRsp);
void tFreeSExplainRsp(SExplainRsp* pRsp);
typedef struct {
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_CLUSTER_VALUE_LEN];
int32_t sqlLen;
char* sql;
} SMCfgClusterReq;
int32_t tSerializeSMCfgClusterReq(void* buf, int32_t bufLen, SMCfgClusterReq* pReq);
int32_t tDeserializeSMCfgClusterReq(void* buf, int32_t bufLen, SMCfgClusterReq* pReq);
void tFreeSMCfgClusterReq(SMCfgClusterReq* pReq);
typedef struct {
char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port
int32_t port;
@ -2408,6 +2429,11 @@ typedef struct SColLocation {
int8_t type;
} SColLocation;
typedef struct SVgroupVer {
int32_t vgId;
int64_t ver;
} SVgroupVer;
typedef struct {
char name[TSDB_STREAM_FNAME_LEN];
char sourceDB[TSDB_DB_FNAME_LEN];
@ -2431,6 +2457,7 @@ typedef struct {
int64_t deleteMark;
int8_t igUpdate;
int64_t lastTs;
SArray* pVgroupVerList;
} SCMCreateStreamReq;
typedef struct {
@ -3896,6 +3923,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq);
#define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1
#define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2
#define SUBMIT_REQ_FROM_FILE 0x4
#define SOURCE_NULL 0
#define SOURCE_TAOSX 1

4
include/common/tmsgdef.h
View File

@ -218,6 +218,7 @@
TD_DEF_MSG_TYPE(TDMT_MND_KILL_COMPACT, "kill-compact", SKillCompactReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_TIMER, "compact-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_REQ_CHKPT, "stream-req-checkpoint", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CONFIG_CLUSTER, "config-cluster", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
@ -337,8 +338,9 @@
TD_DEF_MSG_TYPE(TDMT_SYNC_LOCAL_CMD, "sync-local-cmd", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT, "sync-prep-snapshot", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_PREP_SNAPSHOT_REPLY, "sync-prep-snapshot-reply", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_UNUSED_CODE, "sync-unused", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_FORCE_FOLLOWER, "sync-force-become-follower", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_SYNC_MSG)
TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) //7 << 8

581
include/common/ttokendef.h
View File

@ -74,295 +74,298 @@
#define TK_NK_IPTOKEN 55
#define TK_FORCE 56
#define TK_UNSAFE 57
#define TK_LOCAL 58
#define TK_QNODE 59
#define TK_BNODE 60
#define TK_SNODE 61
#define TK_MNODE 62
#define TK_VNODE 63
#define TK_DATABASE 64
#define TK_USE 65
#define TK_FLUSH 66
#define TK_TRIM 67
#define TK_COMPACT 68
#define TK_IF 69
#define TK_NOT 70
#define TK_EXISTS 71
#define TK_BUFFER 72
#define TK_CACHEMODEL 73
#define TK_CACHESIZE 74
#define TK_COMP 75
#define TK_DURATION 76
#define TK_NK_VARIABLE 77
#define TK_MAXROWS 78
#define TK_MINROWS 79
#define TK_KEEP 80
#define TK_PAGES 81
#define TK_PAGESIZE 82
#define TK_TSDB_PAGESIZE 83
#define TK_PRECISION 84
#define TK_REPLICA 85
#define TK_VGROUPS 86
#define TK_SINGLE_STABLE 87
#define TK_RETENTIONS 88
#define TK_SCHEMALESS 89
#define TK_WAL_LEVEL 90
#define TK_WAL_FSYNC_PERIOD 91
#define TK_WAL_RETENTION_PERIOD 92
#define TK_WAL_RETENTION_SIZE 93
#define TK_WAL_ROLL_PERIOD 94
#define TK_WAL_SEGMENT_SIZE 95
#define TK_STT_TRIGGER 96
#define TK_TABLE_PREFIX 97
#define TK_TABLE_SUFFIX 98
#define TK_KEEP_TIME_OFFSET 99
#define TK_NK_COLON 100
#define TK_BWLIMIT 101
#define TK_START 102
#define TK_TIMESTAMP 103
#define TK_END 104
#define TK_TABLE 105
#define TK_NK_LP 106
#define TK_NK_RP 107
#define TK_STABLE 108
#define TK_COLUMN 109
#define TK_MODIFY 110
#define TK_RENAME 111
#define TK_TAG 112
#define TK_SET 113
#define TK_NK_EQ 114
#define TK_USING 115
#define TK_TAGS 116
#define TK_BOOL 117
#define TK_TINYINT 118
#define TK_SMALLINT 119
#define TK_INT 120
#define TK_INTEGER 121
#define TK_BIGINT 122
#define TK_FLOAT 123
#define TK_DOUBLE 124
#define TK_BINARY 125
#define TK_NCHAR 126
#define TK_UNSIGNED 127
#define TK_JSON 128
#define TK_VARCHAR 129
#define TK_MEDIUMBLOB 130
#define TK_BLOB 131
#define TK_VARBINARY 132
#define TK_GEOMETRY 133
#define TK_DECIMAL 134
#define TK_COMMENT 135
#define TK_MAX_DELAY 136
#define TK_WATERMARK 137
#define TK_ROLLUP 138
#define TK_TTL 139
#define TK_SMA 140
#define TK_DELETE_MARK 141
#define TK_FIRST 142
#define TK_LAST 143
#define TK_SHOW 144
#define TK_PRIVILEGES 145
#define TK_DATABASES 146
#define TK_TABLES 147
#define TK_STABLES 148
#define TK_MNODES 149
#define TK_QNODES 150
#define TK_FUNCTIONS 151
#define TK_INDEXES 152
#define TK_ACCOUNTS 153
#define TK_APPS 154
#define TK_CONNECTIONS 155
#define TK_LICENCES 156
#define TK_GRANTS 157
#define TK_QUERIES 158
#define TK_SCORES 159
#define TK_TOPICS 160
#define TK_VARIABLES 161
#define TK_CLUSTER 162
#define TK_BNODES 163
#define TK_SNODES 164
#define TK_TRANSACTIONS 165
#define TK_DISTRIBUTED 166
#define TK_CONSUMERS 167
#define TK_SUBSCRIPTIONS 168
#define TK_VNODES 169
#define TK_ALIVE 170
#define TK_VIEWS 171
#define TK_VIEW 172
#define TK_COMPACTS 173
#define TK_NORMAL 174
#define TK_CHILD 175
#define TK_LIKE 176
#define TK_TBNAME 177
#define TK_QTAGS 178
#define TK_AS 179
#define TK_SYSTEM 180
#define TK_INDEX 181
#define TK_FUNCTION 182
#define TK_INTERVAL 183
#define TK_COUNT 184
#define TK_LAST_ROW 185
#define TK_META 186
#define TK_ONLY 187
#define TK_TOPIC 188
#define TK_CONSUMER 189
#define TK_GROUP 190
#define TK_DESC 191
#define TK_DESCRIBE 192
#define TK_RESET 193
#define TK_QUERY 194
#define TK_CACHE 195
#define TK_EXPLAIN 196
#define TK_ANALYZE 197
#define TK_VERBOSE 198
#define TK_NK_BOOL 199
#define TK_RATIO 200
#define TK_NK_FLOAT 201
#define TK_OUTPUTTYPE 202
#define TK_AGGREGATE 203
#define TK_BUFSIZE 204
#define TK_LANGUAGE 205
#define TK_REPLACE 206
#define TK_STREAM 207
#define TK_INTO 208
#define TK_PAUSE 209
#define TK_RESUME 210
#define TK_TRIGGER 211
#define TK_AT_ONCE 212
#define TK_WINDOW_CLOSE 213
#define TK_IGNORE 214
#define TK_EXPIRED 215
#define TK_FILL_HISTORY 216
#define TK_UPDATE 217
#define TK_SUBTABLE 218
#define TK_UNTREATED 219
#define TK_KILL 220
#define TK_CONNECTION 221
#define TK_TRANSACTION 222
#define TK_BALANCE 223
#define TK_VGROUP 224
#define TK_LEADER 225
#define TK_MERGE 226
#define TK_REDISTRIBUTE 227
#define TK_SPLIT 228
#define TK_DELETE 229
#define TK_INSERT 230
#define TK_NULL 231
#define TK_NK_QUESTION 232
#define TK_NK_ALIAS 233
#define TK_NK_ARROW 234
#define TK_ROWTS 235
#define TK_QSTART 236
#define TK_QEND 237
#define TK_QDURATION 238
#define TK_WSTART 239
#define TK_WEND 240
#define TK_WDURATION 241
#define TK_IROWTS 242
#define TK_ISFILLED 243
#define TK_CAST 244
#define TK_NOW 245
#define TK_TODAY 246
#define TK_TIMEZONE 247
#define TK_CLIENT_VERSION 248
#define TK_SERVER_VERSION 249
#define TK_SERVER_STATUS 250
#define TK_CURRENT_USER 251
#define TK_CASE 252
#define TK_WHEN 253
#define TK_THEN 254
#define TK_ELSE 255
#define TK_BETWEEN 256
#define TK_IS 257
#define TK_NK_LT 258
#define TK_NK_GT 259
#define TK_NK_LE 260
#define TK_NK_GE 261
#define TK_NK_NE 262
#define TK_MATCH 263
#define TK_NMATCH 264
#define TK_CONTAINS 265
#define TK_IN 266
#define TK_JOIN 267
#define TK_INNER 268
#define TK_SELECT 269
#define TK_NK_HINT 270
#define TK_DISTINCT 271
#define TK_WHERE 272
#define TK_PARTITION 273
#define TK_BY 274
#define TK_SESSION 275
#define TK_STATE_WINDOW 276
#define TK_EVENT_WINDOW 277
#define TK_SLIDING 278
#define TK_FILL 279
#define TK_VALUE 280
#define TK_VALUE_F 281
#define TK_NONE 282
#define TK_PREV 283
#define TK_NULL_F 284
#define TK_LINEAR 285
#define TK_NEXT 286
#define TK_HAVING 287
#define TK_RANGE 288
#define TK_EVERY 289
#define TK_ORDER 290
#define TK_SLIMIT 291
#define TK_SOFFSET 292
#define TK_LIMIT 293
#define TK_OFFSET 294
#define TK_ASC 295
#define TK_NULLS 296
#define TK_ABORT 297
#define TK_AFTER 298
#define TK_ATTACH 299
#define TK_BEFORE 300
#define TK_BEGIN 301
#define TK_BITAND 302
#define TK_BITNOT 303
#define TK_BITOR 304
#define TK_BLOCKS 305
#define TK_CHANGE 306
#define TK_COMMA 307
#define TK_CONCAT 308
#define TK_CONFLICT 309
#define TK_COPY 310
#define TK_DEFERRED 311
#define TK_DELIMITERS 312
#define TK_DETACH 313
#define TK_DIVIDE 314
#define TK_DOT 315
#define TK_EACH 316
#define TK_FAIL 317
#define TK_FILE 318
#define TK_FOR 319
#define TK_GLOB 320
#define TK_ID 321
#define TK_IMMEDIATE 322
#define TK_IMPORT 323
#define TK_INITIALLY 324
#define TK_INSTEAD 325
#define TK_ISNULL 326
#define TK_KEY 327
#define TK_MODULES 328
#define TK_NK_BITNOT 329
#define TK_NK_SEMI 330
#define TK_NOTNULL 331
#define TK_OF 332
#define TK_PLUS 333
#define TK_PRIVILEGE 334
#define TK_RAISE 335
#define TK_RESTRICT 336
#define TK_ROW 337
#define TK_SEMI 338
#define TK_STAR 339
#define TK_STATEMENT 340
#define TK_STRICT 341
#define TK_STRING 342
#define TK_TIMES 343
#define TK_VALUES 344
#define TK_VARIABLE 345
#define TK_WAL 346
#define TK_CLUSTER 58
#define TK_LOCAL 59
#define TK_QNODE 60
#define TK_BNODE 61
#define TK_SNODE 62
#define TK_MNODE 63
#define TK_VNODE 64
#define TK_DATABASE 65
#define TK_USE 66
#define TK_FLUSH 67
#define TK_TRIM 68
#define TK_COMPACT 69
#define TK_IF 70
#define TK_NOT 71
#define TK_EXISTS 72
#define TK_BUFFER 73
#define TK_CACHEMODEL 74
#define TK_CACHESIZE 75
#define TK_COMP 76
#define TK_DURATION 77
#define TK_NK_VARIABLE 78
#define TK_MAXROWS 79
#define TK_MINROWS 80
#define TK_KEEP 81
#define TK_PAGES 82
#define TK_PAGESIZE 83
#define TK_TSDB_PAGESIZE 84
#define TK_PRECISION 85
#define TK_REPLICA 86
#define TK_VGROUPS 87
#define TK_SINGLE_STABLE 88
#define TK_RETENTIONS 89
#define TK_SCHEMALESS 90
#define TK_WAL_LEVEL 91
#define TK_WAL_FSYNC_PERIOD 92
#define TK_WAL_RETENTION_PERIOD 93
#define TK_WAL_RETENTION_SIZE 94
#define TK_WAL_ROLL_PERIOD 95
#define TK_WAL_SEGMENT_SIZE 96
#define TK_STT_TRIGGER 97
#define TK_TABLE_PREFIX 98
#define TK_TABLE_SUFFIX 99
#define TK_KEEP_TIME_OFFSET 100
#define TK_NK_COLON 101
#define TK_BWLIMIT 102
#define TK_START 103
#define TK_TIMESTAMP 104
#define TK_END 105
#define TK_TABLE 106
#define TK_NK_LP 107
#define TK_NK_RP 108
#define TK_STABLE 109
#define TK_COLUMN 110
#define TK_MODIFY 111
#define TK_RENAME 112
#define TK_TAG 113
#define TK_SET 114
#define TK_NK_EQ 115
#define TK_USING 116
#define TK_TAGS 117
#define TK_BOOL 118
#define TK_TINYINT 119
#define TK_SMALLINT 120
#define TK_INT 121
#define TK_INTEGER 122
#define TK_BIGINT 123
#define TK_FLOAT 124
#define TK_DOUBLE 125
#define TK_BINARY 126
#define TK_NCHAR 127
#define TK_UNSIGNED 128
#define TK_JSON 129
#define TK_VARCHAR 130
#define TK_MEDIUMBLOB 131
#define TK_BLOB 132
#define TK_VARBINARY 133
#define TK_GEOMETRY 134
#define TK_DECIMAL 135
#define TK_COMMENT 136
#define TK_MAX_DELAY 137
#define TK_WATERMARK 138
#define TK_ROLLUP 139
#define TK_TTL 140
#define TK_SMA 141
#define TK_DELETE_MARK 142
#define TK_FIRST 143
#define TK_LAST 144
#define TK_SHOW 145
#define TK_PRIVILEGES 146
#define TK_DATABASES 147
#define TK_TABLES 148
#define TK_STABLES 149
#define TK_MNODES 150
#define TK_QNODES 151
#define TK_FUNCTIONS 152
#define TK_INDEXES 153
#define TK_ACCOUNTS 154
#define TK_APPS 155
#define TK_CONNECTIONS 156
#define TK_LICENCES 157
#define TK_GRANTS 158
#define TK_FULL 159
#define TK_LOGS 160
#define TK_MACHINES 161
#define TK_QUERIES 162
#define TK_SCORES 163
#define TK_TOPICS 164
#define TK_VARIABLES 165
#define TK_BNODES 166
#define TK_SNODES 167
#define TK_TRANSACTIONS 168
#define TK_DISTRIBUTED 169
#define TK_CONSUMERS 170
#define TK_SUBSCRIPTIONS 171
#define TK_VNODES 172
#define TK_ALIVE 173
#define TK_VIEWS 174
#define TK_VIEW 175
#define TK_COMPACTS 176
#define TK_NORMAL 177
#define TK_CHILD 178
#define TK_LIKE 179
#define TK_TBNAME 180
#define TK_QTAGS 181
#define TK_AS 182
#define TK_SYSTEM 183
#define TK_INDEX 184
#define TK_FUNCTION 185
#define TK_INTERVAL 186
#define TK_COUNT 187
#define TK_LAST_ROW 188
#define TK_META 189
#define TK_ONLY 190
#define TK_TOPIC 191
#define TK_CONSUMER 192
#define TK_GROUP 193
#define TK_DESC 194
#define TK_DESCRIBE 195
#define TK_RESET 196
#define TK_QUERY 197
#define TK_CACHE 198
#define TK_EXPLAIN 199
#define TK_ANALYZE 200
#define TK_VERBOSE 201
#define TK_NK_BOOL 202
#define TK_RATIO 203
#define TK_NK_FLOAT 204
#define TK_OUTPUTTYPE 205
#define TK_AGGREGATE 206
#define TK_BUFSIZE 207
#define TK_LANGUAGE 208
#define TK_REPLACE 209
#define TK_STREAM 210
#define TK_INTO 211
#define TK_PAUSE 212
#define TK_RESUME 213
#define TK_TRIGGER 214
#define TK_AT_ONCE 215
#define TK_WINDOW_CLOSE 216
#define TK_IGNORE 217
#define TK_EXPIRED 218
#define TK_FILL_HISTORY 219
#define TK_UPDATE 220
#define TK_SUBTABLE 221
#define TK_UNTREATED 222
#define TK_KILL 223
#define TK_CONNECTION 224
#define TK_TRANSACTION 225
#define TK_BALANCE 226
#define TK_VGROUP 227
#define TK_LEADER 228
#define TK_MERGE 229
#define TK_REDISTRIBUTE 230
#define TK_SPLIT 231
#define TK_DELETE 232
#define TK_INSERT 233
#define TK_NULL 234
#define TK_NK_QUESTION 235
#define TK_NK_ALIAS 236
#define TK_NK_ARROW 237
#define TK_ROWTS 238
#define TK_QSTART 239
#define TK_QEND 240
#define TK_QDURATION 241
#define TK_WSTART 242
#define TK_WEND 243
#define TK_WDURATION 244
#define TK_IROWTS 245
#define TK_ISFILLED 246
#define TK_CAST 247
#define TK_NOW 248
#define TK_TODAY 249
#define TK_TIMEZONE 250
#define TK_CLIENT_VERSION 251
#define TK_SERVER_VERSION 252
#define TK_SERVER_STATUS 253
#define TK_CURRENT_USER 254
#define TK_CASE 255
#define TK_WHEN 256
#define TK_THEN 257
#define TK_ELSE 258
#define TK_BETWEEN 259
#define TK_IS 260
#define TK_NK_LT 261
#define TK_NK_GT 262
#define TK_NK_LE 263
#define TK_NK_GE 264
#define TK_NK_NE 265
#define TK_MATCH 266
#define TK_NMATCH 267
#define TK_CONTAINS 268
#define TK_IN 269
#define TK_JOIN 270
#define TK_INNER 271
#define TK_SELECT 272
#define TK_NK_HINT 273
#define TK_DISTINCT 274
#define TK_WHERE 275
#define TK_PARTITION 276
#define TK_BY 277
#define TK_SESSION 278
#define TK_STATE_WINDOW 279
#define TK_EVENT_WINDOW 280
#define TK_SLIDING 281
#define TK_FILL 282
#define TK_VALUE 283
#define TK_VALUE_F 284
#define TK_NONE 285
#define TK_PREV 286
#define TK_NULL_F 287
#define TK_LINEAR 288
#define TK_NEXT 289
#define TK_HAVING 290
#define TK_RANGE 291
#define TK_EVERY 292
#define TK_ORDER 293
#define TK_SLIMIT 294
#define TK_SOFFSET 295
#define TK_LIMIT 296
#define TK_OFFSET 297
#define TK_ASC 298
#define TK_NULLS 299
#define TK_ABORT 300
#define TK_AFTER 301
#define TK_ATTACH 302
#define TK_BEFORE 303
#define TK_BEGIN 304
#define TK_BITAND 305
#define TK_BITNOT 306
#define TK_BITOR 307
#define TK_BLOCKS 308
#define TK_CHANGE 309
#define TK_COMMA 310
#define TK_CONCAT 311
#define TK_CONFLICT 312
#define TK_COPY 313
#define TK_DEFERRED 314
#define TK_DELIMITERS 315
#define TK_DETACH 316
#define TK_DIVIDE 317
#define TK_DOT 318
#define TK_EACH 319
#define TK_FAIL 320
#define TK_FILE 321
#define TK_FOR 322
#define TK_GLOB 323
#define TK_ID 324
#define TK_IMMEDIATE 325
#define TK_IMPORT 326
#define TK_INITIALLY 327
#define TK_INSTEAD 328
#define TK_ISNULL 329
#define TK_KEY 330
#define TK_MODULES 331
#define TK_NK_BITNOT 332
#define TK_NK_SEMI 333
#define TK_NOTNULL 334
#define TK_OF 335
#define TK_PLUS 336
#define TK_PRIVILEGE 337
#define TK_RAISE 338
#define TK_RESTRICT 339
#define TK_ROW 340
#define TK_SEMI 341
#define TK_STAR 342
#define TK_STATEMENT 343
#define TK_STRICT 344
#define TK_STRING 345
#define TK_TIMES 346
#define TK_VALUES 347
#define TK_VARIABLE 348
#define TK_WAL 349
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601

2
include/dnode/vnode/tqCommon.h
View File

@ -18,7 +18,7 @@
// message process
int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart);
int32_t tqStreamOneTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId);
int32_t tqStreamStartOneTaskAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId);
int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pMsg, bool restored);
int32_t tqStreamTaskProcessDispatchReq(SStreamMeta* pMeta, SRpcMsg* pMsg);
int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg);

2
include/libs/catalog/catalog.h
View File

@ -358,7 +358,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f
int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes);
int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists);
int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists);
int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth);

8
include/libs/executor/storageapi.h
View File

@ -211,6 +211,7 @@ typedef struct SStoreTqReader {
bool (*tqNextBlockImpl)(); // todo remove it
SSDataBlock* (*tqGetResultBlock)();
int64_t (*tqGetResultBlockTime)();
int32_t (*tqGetStreamExecProgress)();
void (*tqReaderSetColIdList)();
int32_t (*tqReaderSetQueryTableList)();
@ -266,16 +267,11 @@ typedef struct SStoreMeta {
// support filter and non-filter cases. [vnodeGetCtbIdList & vnodeGetCtbIdListByFilter]
int32_t (*getChildTableList)(void* pVnode, int64_t suid, SArray* list);
int32_t (*storeGetTableList)(void* pVnode, int8_t type, SArray* pList);
void* storeGetVersionRange;
void* storeGetLastTimestamp;
int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid); // tsdbGetTableSchema
int32_t (*getTableSchema)(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid);
int32_t (*getNumOfChildTables)(void* pVnode, int64_t uid, int64_t* numOfTables, int32_t* numOfCols);
void (*getBasicInfo)(void* pVnode, const char** dbname, int32_t* vgId, int64_t* numOfTables,
int64_t* numOfNormalTables);
int64_t (*getNumOfRowsInMem)(void* pVnode);
SMCtbCursor* (*openCtbCursor)(void* pVnode, tb_uid_t uid, int lock);
int32_t (*resumeCtbCursor)(SMCtbCursor* pCtbCur, int8_t first);
void (*pauseCtbCursor)(SMCtbCursor* pCtbCur);

1
include/libs/function/function.h
View File

@ -114,6 +114,7 @@ typedef struct SInputColumnInfoData {
int32_t totalRows; // total rows in current columnar data
int32_t startRowIndex; // handle started row index
int64_t numOfRows; // the number of rows needs to be handled
bool blankFill; // fill blank data to block for empty table
int32_t numOfInputCols; // PTS is not included
bool colDataSMAIsSet; // if agg is set or not
SColumnInfoData *pPTS; // primary timestamp column

1
include/libs/function/functionMgt.h
View File

@ -126,6 +126,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_TAGS,
FUNCTION_TYPE_TBUID,
FUNCTION_TYPE_VGID,
FUNCTION_TYPE_VGVER,
// internal function
FUNCTION_TYPE_SELECT_VALUE = 3750,

6
include/libs/nodes/cmdnodes.h
View File

@ -420,6 +420,12 @@ typedef struct SDropCGroupStmt {
bool ignoreNotExists;
} SDropCGroupStmt;
typedef struct SAlterClusterStmt {
ENodeType type;
char config[TSDB_DNODE_CONFIG_LEN];
char value[TSDB_CLUSTER_VALUE_LEN];
} SAlterClusterStmt;
typedef struct SAlterLocalStmt {
ENodeType type;
char config[TSDB_DNODE_CONFIG_LEN];

4
include/libs/nodes/plannodes.h
View File

@ -713,8 +713,10 @@ typedef struct SSubplan {
SNode* pTagCond;
SNode* pTagIndexCond;
bool showRewrite;
int32_t rowsThreshold;
bool isView;
bool isAudit;
bool dynamicRowThreshold;
int32_t rowsThreshold;
} SSubplan;
typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode;

6
include/libs/parser/parser.h
View File

@ -86,8 +86,10 @@ typedef struct SParseContext {
bool enableSysInfo;
bool async;
bool hasInvisibleCol;
const char* svrVer;
bool isView;
bool isAudit;
bool nodeOffline;
const char* svrVer;
SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos
int64_t allocatorId;
@ -106,7 +108,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata
const struct SMetaData* pMetaData, SQuery* pQuery);
int32_t qContinueParseSql(SParseContext* pCxt, struct SCatalogReq* pCatalogReq, const struct SMetaData* pMetaData,
SQuery* pQuery);
int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pResRow);
int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, SSDataBlock* pBlock);
void qDestroyParseContext(SParseContext* pCxt);

2
include/libs/planner/planner.h
View File

@ -32,6 +32,8 @@ typedef struct SPlanContext {
bool streamQuery;
bool rSmaQuery;
bool showRewrite;
bool isView;
bool isAudit;
int8_t triggerType;
int64_t watermark;
int64_t deleteMark;

11
include/libs/qcom/query.h
View File

@ -66,7 +66,11 @@ typedef enum {
#define QUERY_RSP_POLICY_QUICK 1
#define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0)
#define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0)
#define QUERY_MSG_MASK_AUDIT() (1 << 1)
#define QUERY_MSG_MASK_VIEW() (1 << 2)
#define TEST_SHOW_REWRITE_MASK(m) (((m) & QUERY_MSG_MASK_SHOW_REWRITE()) != 0)
#define TEST_AUDIT_MASK(m) (((m) & QUERY_MSG_MASK_AUDIT()) != 0)
#define TEST_VIEW_MASK(m) (((m) & QUERY_MSG_MASK_VIEW()) != 0)
typedef struct STableComInfo {
uint8_t numOfTags; // the number of tags in schema
@ -338,6 +342,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define IS_SYS_DBNAME(_dbname) (IS_INFORMATION_SCHEMA_DB(_dbname) || IS_PERFORMANCE_SCHEMA_DB(_dbname))
#define IS_AUDIT_DBNAME(_dbname) ((*(_dbname) == 'a') && (0 == strcmp(_dbname, TSDB_AUDIT_DB)))
#define IS_AUDIT_STB_NAME(_stbname) ((*(_stbname) == 'o') && (0 == strcmp(_stbname, TSDB_AUDIT_STB_OPERATION)))
#define IS_AUDIT_CTB_NAME(_ctbname) \
((*(_ctbname) == 't') && (0 == strncmp(_ctbname, TSDB_AUDIT_CTB_OPERATION, TSDB_AUDIT_CTB_OPERATION_LEN)))
#define qFatal(...) \
do { \
if (qDebugFlag & DEBUG_FATAL) { \

4
include/libs/scalar/scalar.h
View File

@ -96,9 +96,7 @@ int32_t winDurFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu
int32_t qStartTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qEndTsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qTbnameFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qTbUidFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qVgIdFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t qPseudoTagFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
/* Aggregation functions */
int32_t countScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

6
include/libs/stream/streamState.h
View File

@ -13,6 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _STREAM_STATE_H_
#define _STREAM_STATE_H_
#include "tdatablock.h"
#include "rocksdb/c.h"
@ -20,9 +23,6 @@
#include "tsimplehash.h"
#include "tstreamFileState.h"
#ifndef _STREAM_STATE_H_
#define _STREAM_STATE_H_
#ifdef __cplusplus
extern "C" {
#endif

10
include/libs/stream/tstream.h
View File

@ -13,6 +13,9 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _STREAM_H_
#define _STREAM_H_
#include "os.h"
#include "streamState.h"
#include "tdatablock.h"
@ -26,9 +29,6 @@
extern "C" {
#endif
#ifndef _STREAM_H_
#define _STREAM_H_
#define ONE_MiB_F (1048576.0)
#define ONE_KiB_F (1024.0)
#define SIZE_IN_MiB(_v) ((_v) / ONE_MiB_F)
@ -313,7 +313,7 @@ typedef struct SCheckpointInfo {
int64_t failedId; // record the latest failed checkpoint id
int64_t checkpointingId;
int32_t downstreamAlignNum;
int32_t checkpointNotReadyTasks;
int32_t numOfNotReady;
bool dispatchCheckpointTrigger;
int64_t msgVer;
int32_t transId;
@ -531,7 +531,7 @@ typedef struct SStreamMeta {
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, bool fillHistory, int64_t triggerParam,
SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, bool fillHistory, int64_t triggerParam,
SArray* pTaskList, bool hasFillhistory);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);

4
include/os/osDef.h
View File

@ -222,6 +222,10 @@ void syslog(int unused, const char *format, ...);
do { \
prctl(PR_SET_NAME, (name)); \
} while (0)
#define getThreadName(name) \
do { \
prctl(PR_GET_NAME, (name)); \
} while (0)
#endif
#else
// Windows

16
include/util/taoserror.h
View File

@ -414,6 +414,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MNODE_ONLY_TWO_MNODE TAOS_DEF_ERROR_CODE(0, 0x0414) // internal
#define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416)
#define TSDB_CODE_DNODE_NO_MACHINE_CODE TAOS_DEF_ERROR_CODE(0, 0x0417)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
@ -531,7 +532,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x0725)
#define TSDB_CODE_QRY_TASK_CANCELLING TAOS_DEF_ERROR_CODE(0, 0x0726)
#define TSDB_CODE_QRY_TASK_DROPPING TAOS_DEF_ERROR_CODE(0, 0x0727)
#define TSDB_CODE_QRY_DUPLICATTED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0728)
#define TSDB_CODE_QRY_DUPLICATED_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0728)
#define TSDB_CODE_QRY_TASK_MSG_ERROR TAOS_DEF_ERROR_CODE(0, 0x0729)
#define TSDB_CODE_QRY_JOB_FREED TAOS_DEF_ERROR_CODE(0, 0x072A)
#define TSDB_CODE_QRY_TASK_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x072B)
@ -554,7 +555,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_GRANT_STREAM_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0807)
#define TSDB_CODE_GRANT_SPEED_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0808)
#define TSDB_CODE_GRANT_STORAGE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0809)
#define TSDB_CODE_GRANT_QUERYTIME_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A)
#define TSDB_CODE_GRANT_SUBSCRIPTION_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080A)
#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B)
#define TSDB_CODE_GRANT_STABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080C)
#define TSDB_CODE_GRANT_TABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080D)
@ -563,8 +564,17 @@ int32_t* taosGetErrno();
#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0810)
#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0811)
#define TSDB_CODE_GRANT_GEN_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0812)
#define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813)
#define TSDB_CODE_GRANT_GEN_ACTIVE_LEN TAOS_DEF_ERROR_CODE(0, 0x0813)
#define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814)
#define TSDB_CODE_GRANT_PAR_IVLD_DIST TAOS_DEF_ERROR_CODE(0, 0x0815)
#define TSDB_CODE_GRANT_UNLICENSED_CLUSTER TAOS_DEF_ERROR_CODE(0, 0x0816)
#define TSDB_CODE_GRANT_LACK_OF_BASIC TAOS_DEF_ERROR_CODE(0, 0x0817)
#define TSDB_CODE_GRANT_OBJ_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0818)
#define TSDB_CODE_GRANT_LAST_ACTIVE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0819)
#define TSDB_CODE_GRANT_MACHINES_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x0820)
#define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821)
#define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822)
#define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823)
// sync
// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x

32
include/util/tbase58.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_UTIL_BASE58_H_
#define _TD_UTIL_BASE58_H_
#include "os.h"
#ifdef __cplusplus
extern "C" {
#endif
uint8_t *base58_decode(const char *value, size_t inlen, int32_t *outlen);
char *base58_encode(const uint8_t *value, int32_t vlen);
#ifdef __cplusplus
}
#endif
#endif /*_TD_UTIL_BASE58_H_*/

4
include/util/tdef.h
View File

@ -264,6 +264,7 @@ typedef enum ELogicConditionType {
#define TSDB_JOB_STATUS_LEN 32
#define TSDB_CLUSTER_ID_LEN 40
#define TSDB_MACHINE_ID_LEN 24
#define TSDB_FQDN_LEN 128
#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6)
#define TSDB_IPv4ADDR_LEN 16
@ -285,6 +286,9 @@ typedef enum ELogicConditionType {
#define TSDB_DNODE_CONFIG_LEN 128
#define TSDB_DNODE_VALUE_LEN 256
#define TSDB_CLUSTER_VALUE_LEN 1000
#define TSDB_GRANT_LOG_COL_LEN 15072
#define TSDB_ACTIVE_KEY_LEN 109
#define TSDB_CONN_ACTIVE_KEY_LEN 255

2
source/client/src/clientHb.c
View File

@ -843,7 +843,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S
view->version = htonl(view->version);
}
tscDebug("hb got %d expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum);
tscDebug("hb got %u expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum);
if (NULL == req->info) {
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);

59
source/client/src/clientImpl.c
View File

@ -880,19 +880,21 @@ static bool incompletaFileParsing(SNode* pStmt) {
return QUERY_NODE_VNODE_MODIFY_STMT != nodeType(pStmt) ? false : ((SVnodeModifyOpStmt*)pStmt)->fileProcessing;
}
void continuePostSubQuery(SRequestObj* pRequest, TAOS_ROW row) {
void continuePostSubQuery(SRequestObj* pRequest, SSDataBlock* pBlock) {
SSqlCallbackWrapper* pWrapper = pRequest->pWrapper;
int32_t code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId);
int32_t code = nodesAcquireAllocator(pWrapper->pParseCtx->allocatorId);
if (TSDB_CODE_SUCCESS == code) {
int64_t analyseStart = taosGetTimestampUs();
code = qContinueParsePostQuery(pWrapper->pParseCtx, pRequest->pQuery, (void**)row);
code = qContinueParsePostQuery(pWrapper->pParseCtx, pRequest->pQuery, pBlock);
pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart;
}
if (TSDB_CODE_SUCCESS == code) {
code = qContinuePlanPostQuery(pRequest->pPostPlan);
}
nodesReleaseAllocator(pWrapper->pParseCtx->allocatorId);
nodesReleaseAllocator(pWrapper->pParseCtx->allocatorId);
handleQueryAnslyseRes(pWrapper, NULL, code);
}
@ -916,6 +918,43 @@ void returnToUser(SRequestObj* pRequest) {
}
}
static SSDataBlock* createResultBlock(TAOS_RES* pRes, int32_t numOfRows) {
int64_t lastTs = 0;
TAOS_FIELD* pResFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
SSDataBlock* pBlock = createDataBlock();
for(int32_t i = 0; i < numOfFields; ++i) {
SColumnInfoData colInfoData = createColumnInfoData(pResFields[i].type, pResFields[i].bytes, i + 1);
blockDataAppendColInfo(pBlock, &colInfoData);
}
blockDataEnsureCapacity(pBlock, numOfRows);
for (int32_t i = 0; i < numOfRows; ++i) {
TAOS_ROW pRow = taos_fetch_row(pRes);
int64_t ts = *(int64_t*)pRow[0];
if (lastTs < ts) {
lastTs = ts;
}
for(int32_t j = 0; j < numOfFields; ++j) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, j);
colDataSetVal(pColInfoData, i, pRow[j], false);
}
tscDebug("lastKey:%" PRId64 " vgId:%d, vgVer:%" PRId64, ts, *(int32_t*)pRow[1], *(int64_t*)pRow[2]);
}
pBlock->info.window.ekey = lastTs;
pBlock->info.rows = numOfRows;
tscDebug("lastKey:%"PRId64" numOfRows:%d from all vgroups", lastTs, numOfRows);
return pBlock;
}
void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
SRequestObj* pRequest = (SRequestObj*)res;
if (pRequest->code) {
@ -923,19 +962,17 @@ void postSubQueryFetchCb(void* param, TAOS_RES* res, int32_t rowNum) {
return;
}
TAOS_ROW row = NULL;
if (rowNum > 0) {
row = taos_fetch_row(res); // for single row only now
}
SSDataBlock* pBlock = createResultBlock(res, rowNum);
SRequestObj* pNextReq = acquireRequest(pRequest->relation.nextRefId);
if (pNextReq) {
continuePostSubQuery(pNextReq, row);
continuePostSubQuery(pNextReq, pBlock);
releaseRequest(pRequest->relation.nextRefId);
} else {
tscError("0x%" PRIx64 ", next req ref 0x%" PRIx64 " is not there, reqId:0x%" PRIx64, pRequest->self,
pRequest->relation.nextRefId, pRequest->requestId);
}
blockDataDestroy(pBlock);
}
void handlePostSubQuery(SSqlCallbackWrapper* pWrapper) {
@ -1117,6 +1154,8 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite,
.isView = pWrapper->pParseCtx->isView,
.isAudit = pWrapper->pParseCtx->isAudit,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pUser = pRequest->pTscObj->user,

1
source/client/src/clientTmq.c
View File

@ -767,6 +767,7 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) {
}
}
taosWUnLockLatch(&tmq->lock);
taosReleaseRef(tmqMgmt.rsetId, refId);
}
tDeatroySMqHbRsp(&rsp);
taosMemoryFree(pMsg->pData);

5
source/client/test/clientTests.cpp
View File

@ -829,7 +829,10 @@ TEST(clientCase, projection_query_tables) {
TAOS_RES* pRes = taos_query(pConn, "use abc1");
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int, f varchar(4096)) tags(a int)");
// TAOS_RES* pRes = taos_query(pConn, "select tbname, last(ts) from abc1.stable_1 group by tbname");
// taos_free_result(pRes);
pRes = taos_query(pConn, "create stream stream_1 trigger at_once fill_history 1 ignore expired 0 into str_res1 as select _wstart as ts, count(*) from stable_1 interval(10s);");
if (taos_errno(pRes) != 0) {
printf("failed to create table tu, reason:%s\n", taos_errstr(pRes));
}

24
source/common/src/systable.c
View File

@ -37,8 +37,7 @@ static const SSysDbTableSchema dnodesSchema[] = {
{.name = "reboot_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
{.name = "note", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
#ifdef TD_ENTERPRISE
{.name = "active_code", .bytes = TSDB_ACTIVE_KEY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "c_active_code", .bytes = TSDB_CONN_ACTIVE_KEY_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
{.name = "machine_id", .bytes = TSDB_MACHINE_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
#endif
};
@ -349,6 +348,24 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema useGrantsFullSchema[] = {
{.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema useGrantsLogsSchema[] = {
{.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema useMachinesSchema[] = {
{.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@ -378,6 +395,9 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
{TSDB_INS_TABLE_COMPACTS, userCompactsSchema, tListLen(userCompactsSchema), false},
{TSDB_INS_TABLE_COMPACT_DETAILS, userCompactsDetailSchema, tListLen(userCompactsDetailSchema), false},
{TSDB_INS_TABLE_GRANTS_FULL, useGrantsFullSchema, tListLen(useGrantsFullSchema), false},
{TSDB_INS_TABLE_GRANTS_LOGS, useGrantsLogsSchema, tListLen(useGrantsLogsSchema), false},
{TSDB_INS_TABLE_MACHINES, useMachinesSchema, tListLen(useMachinesSchema), false},
};
static const SSysDbTableSchema connectionsSchema[] = {

25
source/common/src/tglobal.c
View File

@ -27,7 +27,7 @@
#include "cus_name.h"
#endif
GRANT_CFG_DECLARE;
// GRANT_CFG_DECLARE;
SConfig *tsCfg = NULL;
@ -58,7 +58,7 @@ int32_t tsNumOfMnodeQueryThreads = 4;
int32_t tsNumOfMnodeFetchThreads = 1;
int32_t tsNumOfMnodeReadThreads = 1;
int32_t tsNumOfVnodeQueryThreads = 4;
float tsRatioOfVnodeStreamThreads = 1.0;
float tsRatioOfVnodeStreamThreads = 0.5F;
int32_t tsNumOfVnodeFetchThreads = 4;
int32_t tsNumOfVnodeRsmaThreads = 2;
int32_t tsNumOfQnodeQueryThreads = 4;
@ -622,7 +622,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
0)
return -1;
if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 10, CFG_SCOPE_SERVER,
if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 4, CFG_SCOPE_SERVER,
CFG_DYN_NONE) != 0)
return -1;
@ -804,7 +804,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "experimental", tsExperimental, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1;
GRANT_CFG_ADD;
// GRANT_CFG_ADD;
return 0;
}
@ -1229,7 +1229,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
GRANT_CFG_GET;
// GRANT_CFG_GET;
return 0;
}
@ -1801,4 +1801,17 @@ void taosSetAllDebugFlag(int32_t flag) {
if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist
}
int8_t taosGranted() { return atomic_load_8(&tsGrant); }
int8_t taosGranted(int8_t type) {
switch (type) {
case TSDB_GRANT_ALL:
return atomic_load_8(&tsGrant) & GRANT_FLAG_ALL;
case TSDB_GRANT_AUDIT:
return atomic_load_8(&tsGrant) & GRANT_FLAG_AUDIT;
case TSDB_GRANT_VIEW:
return atomic_load_8(&tsGrant) & GRANT_FLAG_VIEW;
default:
ASSERTS(0, "undefined grant type:%" PRIi8, type);
break;
}
return 0;
}

1
source/common/src/tgrant.c
View File

@ -19,5 +19,6 @@
#ifndef _GRANT
int32_t grantCheck(EGrantType grant) {return TSDB_CODE_SUCCESS;}
int32_t grantCheckLE(EGrantType grant) {return TSDB_CODE_SUCCESS;}
#endif

72
source/common/src/tmsg.c
View File

@ -1163,9 +1163,11 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tEncodeI64(&encoder, pReq->updateTime) < 0) return -1;
if (tEncodeFloat(&encoder, pReq->numOfCores) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfSupportVnodes) < 0) return -1;
if (tEncodeI32v(&encoder, pReq->numOfDiskCfg) < 0) return -1;
if (tEncodeI64(&encoder, pReq->memTotal) < 0) return -1;
if (tEncodeI64(&encoder, pReq->memAvail) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->dnodeEp) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->machineId) < 0) return -1;
// cluster cfg
if (tEncodeI32(&encoder, pReq->clusterCfg.statusInterval) < 0) return -1;
@ -1253,9 +1255,11 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
if (tDecodeI64(&decoder, &pReq->updateTime) < 0) return -1;
if (tDecodeFloat(&decoder, &pReq->numOfCores) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfSupportVnodes) < 0) return -1;
if (tDecodeI32v(&decoder, &pReq->numOfDiskCfg) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->memTotal) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->memAvail) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->dnodeEp) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->machineId) < 0) return -1;
// cluster cfg
if (tDecodeI32(&decoder, &pReq->clusterCfg.statusInterval) < 0) return -1;
@ -2266,6 +2270,37 @@ int32_t tDeserializeSGetUserWhiteListRsp(void *buf, int32_t bufLen, SGetUserWhit
void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp *pRsp) { taosMemoryFree(pRsp->pWhiteLists); }
int32_t tSerializeSMCfgClusterReq(void *buf, int32_t bufLen, SMCfgClusterReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->config) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->value) < 0) return -1;
ENCODESQL();
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMCfgClusterReq(void *buf, int32_t bufLen, SMCfgClusterReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1;
DECODESQL();
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSMCfgClusterReq(SMCfgClusterReq *pReq) { FREESQL(); }
int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
@ -7222,6 +7257,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeI8(&encoder, pReq->createStb) < 0) return -1;
if (tEncodeU64(&encoder, pReq->targetStbUid) < 0) return -1;
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->fillNullCols)) < 0) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pReq->fillNullCols); ++i) {
SColLocation *pCol = taosArrayGet(pReq->fillNullCols, i);
@ -7229,10 +7265,19 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeI16(&encoder, pCol->colId) < 0) return -1;
if (tEncodeI8(&encoder, pCol->type) < 0) return -1;
}
if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igUpdate) < 0) return -1;
if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1;
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1;
for(int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
SVgroupVer* p = taosArrayGet(pReq->pVgroupVerList, i);
if (tEncodeI32(&encoder, p->vgId) < 0) return -1;
if (tEncodeI64(&encoder, p->ver) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -7243,6 +7288,8 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStreamReq *pReq) {
int32_t sqlLen = 0;
int32_t astLen = 0;
int32_t numOfFillNullCols = 0;
int32_t numOfVgVer = 0;
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
@ -7294,7 +7341,6 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
}
if (tDecodeI8(&decoder, &pReq->createStb) < 0) return -1;
if (tDecodeU64(&decoder, &pReq->targetStbUid) < 0) return -1;
int32_t numOfFillNullCols = 0;
if (tDecodeI32(&decoder, &numOfFillNullCols) < 0) return -1;
if (numOfFillNullCols > 0) {
pReq->fillNullCols = taosArrayInit(numOfFillNullCols, sizeof(SColLocation));
@ -7319,9 +7365,28 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
if (tDecodeI8(&decoder, &pReq->igUpdate) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1;
tEndDecode(&decoder);
if (tDecodeI32(&decoder, &numOfVgVer) < 0) return -1;
if (numOfVgVer > 0) {
pReq->pVgroupVerList = taosArrayInit(numOfVgVer, sizeof(SVgroupVer));
if (pReq->pVgroupVerList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < numOfVgVer; ++i) {
SVgroupVer v = {0};
if (tDecodeI32(&decoder, &v.vgId) < 0) return -1;
if (tDecodeI64(&decoder, &v.ver) < 0) return -1;
if (taosArrayPush(pReq->pVgroupVerList, &v) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
@ -7393,10 +7458,11 @@ void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) {
if (NULL == pReq) {
return;
}
taosArrayDestroy(pReq->pTags);
taosMemoryFreeClear(pReq->sql);
taosMemoryFreeClear(pReq->ast);
taosArrayDestroy(pReq->pTags);
taosArrayDestroy(pReq->fillNullCols);
taosArrayDestroy(pReq->pVgroupVerList);
}
int32_t tEncodeSRSmaParam(SEncoder *pCoder, const SRSmaParam *pRSmaParam) {

10
source/dnode/mgmt/mgmt_dnode/src/dmHandle.c
View File

@ -114,9 +114,15 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
req.updateTime = pMgmt->pData->updateTime;
req.numOfCores = tsNumOfCores;
req.numOfSupportVnodes = tsNumOfSupportVnodes;
req.numOfDiskCfg = tsDiskCfgNum;
req.memTotal = tsTotalMemoryKB * 1024;
req.memAvail = req.memTotal - tsRpcQueueMemoryAllowed - 16 * 1024 * 1024;
tstrncpy(req.dnodeEp, tsLocalEp, TSDB_EP_LEN);
char *machine = tGetMachineId();
if (machine) {
tstrncpy(req.machineId, machine, TSDB_MACHINE_ID_LEN + 1);
taosMemoryFreeClear(machine);
}
req.clusterCfg.statusInterval = tsStatusInterval;
req.clusterCfg.checkTime = 0;
@ -319,7 +325,7 @@ int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) {
for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
SConfigItem *pItem = taosArrayGet(tsCfg->array, i);
GRANT_CFG_SKIP;
// GRANT_CFG_SKIP;
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++);
colDataSetVal(pColInfo, i, (const char *)&dnodeId, false);
@ -429,7 +435,7 @@ SArray *dmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;
// Requests handled by MNODE
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -163,7 +163,6 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@ -193,6 +192,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_COMPACT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CONFIG_CLUSTER, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_COMPACT_PROGRESS_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;

27
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -72,7 +72,6 @@ typedef struct SUdfdData {
int32_t dnodeId;
} SUdfdData;
#ifndef TD_MODULE_OPTIMIZE
typedef struct SDnode {
int8_t once;
bool stop;
@ -86,21 +85,6 @@ typedef struct SDnode {
SMgmtWrapper wrappers[NODE_END];
SDnodeTrans trans;
} SDnode;
#else
typedef struct SDnode {
int8_t once;
bool stop;
EDndRunStatus status;
SStartupInfo startup;
SDnodeTrans trans;
SUdfdData udfdData;
TdThreadMutex mutex;
TdFilePtr lockfile;
SDnodeData data;
STfs *pTfs;
SMgmtWrapper wrappers[NODE_END];
} SDnode;
#endif
// dmEnv.c
SDnode *dmInstance();
@ -115,12 +99,7 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode);
void dmClearVars(SDnode *pDnode);
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
#else
int32_t dmInitModule(SDnode *pDnode);
#endif
int32_t dmInitModule(SDnode *pDnode);
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
void dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
void dmProcessServerStartupStatus(SDnode *pDnode, SRpcMsg *pMsg);
@ -143,11 +122,7 @@ void dmCleanupClient(SDnode *pDnode);
void dmCleanupStatusClient(SDnode *pDnode);
void dmCleanupSyncClient(SDnode *pDnode);
SMsgCb dmGetMsgcb(SDnode *pDnode);
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitMsgHandle(SDnode *pDnode);
#endif
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c

8
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -24,7 +24,6 @@
#include "tglobal.h"
#endif
#ifndef TD_MODULE_OPTIMIZE
static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
@ -38,7 +37,6 @@ static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) {
return required;
}
#endif
int32_t dmInitDnode(SDnode *pDnode) {
dDebug("start to create dnode");
@ -81,15 +79,9 @@ int32_t dmInitDnode(SDnode *pDnode) {
if (pDnode->lockfile == NULL) {
goto _OVER;
}
#ifdef TD_MODULE_OPTIMIZE
if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
goto _OVER;
}
#else
if (dmInitModule(pDnode) != 0) {
goto _OVER;
}
#endif
indexInit(tsNumOfCommitThreads);
streamMetaInit();

28
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -251,33 +251,6 @@ _OVER:
dmReleaseWrapper(pWrapper);
}
#ifdef TD_MODULE_OPTIMIZE
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
SDnodeTrans *pTrans = &pDnode->trans;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = wrappers + ntype;
SArray *pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
}
if (!pMgmt->needCheckVgId) {
pHandle->defaultNtype = ntype;
}
pWrapper->msgFps[TMSG_INDEX(pMgmt->msgType)] = pMgmt->msgFp;
}
taosArrayDestroy(pArray);
}
return 0;
}
#else
int32_t dmInitMsgHandle(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
@ -303,7 +276,6 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
return 0;
}
#endif
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance();

76
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -76,6 +76,7 @@ typedef enum {
MND_OPER_DROP_TOPIC,
MND_OPER_CREATE_VIEW,
MND_OPER_DROP_VIEW,
MND_OPER_CONFIG_CLUSTER,
} EOperType;
typedef enum {
@ -204,6 +205,7 @@ typedef struct {
int32_t numOfVnodes;
int32_t numOfOtherNodes;
int32_t numOfSupportVnodes;
int32_t numOfDiskCfg;
float numOfCores;
int64_t memTotal;
int64_t memAvail;
@ -214,6 +216,7 @@ typedef struct {
char ep[TSDB_EP_LEN];
char active[TSDB_ACTIVE_KEY_LEN];
char connActive[TSDB_CONN_ACTIVE_KEY_LEN];
char machineId[TSDB_MACHINE_ID_LEN + 1];
} SDnodeObj;
typedef struct {
@ -553,7 +556,7 @@ typedef struct {
} SMqConsumerObj;
SMqConsumerObj* tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_LEN]);
void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool delete);
void tDeleteSMqConsumerObj(SMqConsumerObj* pConsumer, bool isDeleted);
int32_t tEncodeSMqConsumerObj(void** buf, const SMqConsumerObj* pConsumer);
void* tDecodeSMqConsumerObj(const void* buf, SMqConsumerObj* pConsumer, int8_t sver);
@ -753,6 +756,77 @@ typedef struct {
SArray* compactDetail;
} SCompactObj;
// SGrantLogObj
typedef enum {
GRANT_STATE_INIT = 0,
GRANT_STATE_UNGRANTED = 1,
GRANT_STATE_GRANTED = 2,
GRANT_STATE_EXPIRED = 3,
GRANT_STATE_REVOKED = 4,
GRANT_STATE_MAX,
} EGrantState;
typedef enum {
GRANT_STATE_REASON_INIT = 0,
GRANT_STATE_REASON_ALTER = 1, // alter activeCode 'revoked' or 'xxx'
GRANT_STATE_REASON_MISMATCH = 2, // dnode machine mismatch
GRANT_STATE_REASON_EXPIRE = 3, // expire
GRANT_STATE_REASON_MAX,
} EGrantStateReason;
#define GRANT_STATE_NUM 30
#define GRANT_ACTIVE_NUM 10
#define GRANT_ACTIVE_HEAD_LEN 30
typedef struct {
union {
int64_t u0;
struct {
int64_t ts : 40;
int64_t lastState : 4;
int64_t state : 4;
int64_t reason : 8;
int64_t reserve : 8;
};
};
} SGrantState;
typedef struct {
union {
int64_t u0;
struct {
int64_t ts : 40;
int64_t reserve : 24;
};
};
char active[GRANT_ACTIVE_HEAD_LEN + 1];
} SGrantActive;
typedef struct {
union {
int64_t u0;
struct {
int64_t ts : 40;
int64_t id : 24;
};
};
char machine[TSDB_MACHINE_ID_LEN + 1];
} SGrantMachine;
typedef struct {
int32_t id;
int8_t nStates;
int8_t nActives;
int64_t createTime;
int64_t updateTime;
int64_t upgradeTime;
SGrantState states[GRANT_STATE_NUM];
SGrantActive actives[GRANT_ACTIVE_NUM];
char* active;
SArray* pMachines; // SGrantMachine
SRWLatch lock;
} SGrantLogObj;
#ifdef __cplusplus
}
#endif

24
source/dnode/mnode/impl/inc/mndGrant.h
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_GTANT_H
#define TDENGINE_GTANT_H
#ifndef _TD_MND_GRANT_H_
#define _TD_MND_GRANT_H_
#ifdef __cplusplus
"C" {
@ -29,6 +29,26 @@
void grantAdd(EGrantType grant, uint64_t value);
void grantRestore(EGrantType grant, uint64_t value);
#ifdef TD_ENTERPRISE
SSdbRaw *mndGrantActionEncode(SGrantLogObj * pGrant);
SSdbRow *mndGrantActionDecode(SSdbRaw * pRaw);
int32_t mndGrantActionInsert(SSdb * pSdb, SGrantLogObj * pGrant);
int32_t mndGrantActionDelete(SSdb * pSdb, SGrantLogObj * pGrant);
int32_t mndGrantActionUpdate(SSdb * pSdb, SGrantLogObj * pOldGrant, SGrantLogObj * pNewGrant);
#ifdef TD_UNIQ_GRANT
int32_t grantAlterActiveCode(SMnode * pMnode, SGrantLogObj * pObj, const char *oldActive, const char *newActive,
char **mergeActive);
#endif
int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg);
int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState);
int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState);
SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter);
void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter);
#endif
#ifdef __cplusplus
}
#endif

2
source/dnode/mnode/impl/inc/mndScheduler.h
View File

@ -27,7 +27,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t mndConvertRsmaTask(char** pDst, int32_t* pDstLen, const char* ast, int64_t uid, int8_t triggerType,
int64_t watermark, int64_t deleteMark);
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t nextWindowSkey);
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVerList);
#ifdef __cplusplus
}

1
source/dnode/mnode/impl/inc/mndStream.h
View File

@ -124,6 +124,7 @@ SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream);
void destroyStreamTaskIter(SStreamTaskIter *pIter);
bool streamTaskIterNextTask(SStreamTaskIter *pIter);
SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter);
void mndInitExecInfo();
#ifdef __cplusplus
}

69
source/dnode/mnode/impl/src/mndCluster.c
View File

@ -14,7 +14,10 @@
*/
#define _DEFAULT_SOURCE
#include "audit.h"
#include "mndCluster.h"
#include "mndGrant.h"
#include "mndPrivilege.h"
#include "mndShow.h"
#include "mndTrans.h"
@ -31,6 +34,8 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode);
static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter);
static int32_t mndProcessUptimeTimer(SRpcMsg *pReq);
static int32_t mndProcessConfigClusterReq(SRpcMsg *pReq);
static int32_t mndProcessConfigClusterRsp(SRpcMsg *pReq);
int32_t mndInitCluster(SMnode *pMnode) {
SSdbTable table = {
@ -45,6 +50,8 @@ int32_t mndInitCluster(SMnode *pMnode) {
};
mndSetMsgHandle(pMnode, TDMT_MND_UPTIME_TIMER, mndProcessUptimeTimer);
mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_CLUSTER, mndProcessConfigClusterReq);
mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_CLUSTER_RSP, mndProcessConfigClusterRsp);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndRetrieveClusters);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CLUSTER, mndCancelGetNextCluster);
@ -147,6 +154,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) {
SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER)
SDB_SET_INT32(pRaw, dataPos, pCluster->upTime, _OVER)
SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER);
terrno = 0;
@ -164,7 +172,7 @@ _OVER:
static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
SClusterObj *pCluster = NULL;
SSdbRow *pRow = NULL;
SSdbRow *pRow = NULL;
int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER;
@ -359,3 +367,62 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) {
mndTransDrop(pTrans);
return 0;
}
int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) {
int32_t code = 0;
SMnode *pMnode = pReq->info.node;
SMCfgClusterReq cfgReq = {0};
if (tDeserializeSMCfgClusterReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("cluster: start to config, option:%s, value:%s", cfgReq.config, cfgReq.value);
if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_CLUSTER) != 0) {
code = terrno != 0 ? terrno : TSDB_CODE_MND_NO_RIGHTS;
goto _exit;
}
SClusterObj clusterObj = {0};
void *pIter = NULL;
SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter);
if (!pCluster || pCluster->id <= 0) {
code = TSDB_CODE_APP_IS_STARTING;
if (pCluster) mndReleaseCluster(pMnode, pCluster, pIter);
goto _exit;
}
memcpy(&clusterObj, pCluster, sizeof(SClusterObj));
mndReleaseCluster(pMnode, pCluster, pIter);
if (strncmp(cfgReq.config, GRANT_ACTIVE_CODE, TSDB_DNODE_CONFIG_LEN) == 0) {
#ifdef TD_ENTERPRISE
if (0 != (code = mndProcessConfigGrantReq(pMnode, pReq, &cfgReq))) {
goto _exit;
}
#else
code = TSDB_CODE_OPS_NOT_SUPPORT;
goto _exit;
#endif
} else {
code = TSDB_CODE_OPS_NOT_SUPPORT;
goto _exit;
}
{ // audit
auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1));
}
_exit:
tFreeSMCfgClusterReq(&cfgReq);
if (code != 0) {
terrno = code;
mError("cluster: failed to config:%s %s since %s", cfgReq.config, cfgReq.value, terrstr());
} else {
mInfo("cluster: success to config:%s %s", cfgReq.config, cfgReq.value);
}
return code;
}
int32_t mndProcessConfigClusterRsp(SRpcMsg *pRsp) {
mInfo("config rsp from cluster");
return 0;
}

3
source/dnode/mnode/impl/src/mndCompact.c
View File

@ -599,7 +599,8 @@ static int32_t mndSaveCompactProgress(SMnode *pMnode, int32_t compactId) {
pDetail->compactId, pDetail->vgId, pDetail->dnodeId, pDetail->numberFileset, pDetail->finished,
pDetail->newNumberFileset, pDetail->newFinished);
if(pDetail->numberFileset < pDetail->newNumberFileset || pDetail->finished < pDetail->newFinished)
//these 2 number will jump back after dnode restart, so < is not used here
if(pDetail->numberFileset != pDetail->newNumberFileset || pDetail->finished != pDetail->newFinished)
needSave = true;
}

21
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -101,12 +101,17 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *
goto FAILED;
}
if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIBE) < 0) {
if (mndCheckTopicPrivilege(pMnode, pUser, MND_OPER_SUBSCRIBE, pTopic) != 0) {
code = TSDB_CODE_MND_NO_RIGHTS;
terrno = TSDB_CODE_MND_NO_RIGHTS;
goto FAILED;
}
if ((terrno = grantCheckLE(TSDB_GRANT_SUBSCRIPTION)) < 0) {
code = terrno;
goto FAILED;
}
if(enableReplay){
if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){
code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT;
@ -235,9 +240,10 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbR
}
STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1);
strcpy(data->topic, topic);
if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIBE) < 0) {
if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 ||
grantCheckLE(TSDB_GRANT_SUBSCRIPTION) < 0) {
data->noPrivilege = 1;
} else{
} else {
data->noPrivilege = 0;
}
mndReleaseTopic(pMnode, pTopic);
@ -544,10 +550,11 @@ static void freeItem(void *param) {
int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node;
char *msgStr = pMsg->pCont;
int32_t code = -1;
if(grantCheck(TSDB_GRANT_SUBSCRIBE) < 0){
terrno = TSDB_CODE_GRANT_EXPIRED;
return -1;
if ((terrno = grantCheck(TSDB_GRANT_SUBSCRIPTION)) < 0) {
code = terrno;
return code;
}
SCMSubscribeReq subscribe = {0};
@ -559,7 +566,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) {
SMqConsumerObj *pConsumerNew = NULL;
STrans *pTrans = NULL;
int32_t code = -1;
SArray *pTopicList = subscribe.topicNames;
taosArraySort(pTopicList, taosArrayCompareString);
taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem);

158
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -136,6 +136,16 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
tstrncpy(dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN);
dnodeObj.fqdn[TSDB_FQDN_LEN - 1] = 0;
snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", tsLocalFqdn, tsServerPort);
char *machineId = tGetMachineId();
if (machineId) {
memcpy(dnodeObj.machineId, machineId, TSDB_MACHINE_ID_LEN);
taosMemoryFreeClear(machineId);
} else {
#ifdef TD_UNIQ_GRANT
terrno = TSDB_CODE_DNODE_NO_MACHINE_CODE;
goto _OVER;
#endif
}
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode");
if (pTrans == NULL) goto _OVER;
@ -402,6 +412,7 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
tstrncpy(dInfo.machineId, pDnode->machineId, TSDB_MACHINE_ID_LEN + 1);
sdbRelease(pSdb, pDnode);
if (mndIsMnode(pMnode, pDnode->id)) {
dInfo.isMnode = 1;
@ -657,8 +668,12 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
pDnode->rebootTime = statusReq.rebootTime;
pDnode->numOfCores = statusReq.numOfCores;
pDnode->numOfSupportVnodes = statusReq.numOfSupportVnodes;
pDnode->numOfDiskCfg = statusReq.numOfDiskCfg;
pDnode->memAvail = statusReq.memAvail;
pDnode->memTotal = statusReq.memTotal;
if (pDnode->machineId[0] == 0 && statusReq.machineId[0] != 0) {
tstrncpy(pDnode->machineId, statusReq.machineId, TSDB_MACHINE_ID_LEN + 1);
}
SStatusRsp statusRsp = {0};
statusRsp.statusSeq++;
@ -761,109 +776,6 @@ _OVER:
return code;
}
static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfgReq, int8_t action) {
SSdbRaw *pRaw = NULL;
STrans *pTrans = NULL;
SDnodeObj *pDnode = NULL;
SArray *failRecord = NULL;
bool cfgAll = pCfgReq->dnodeId == -1;
int32_t cfgAllErr = 0;
int32_t iter = 0;
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
if (cfgAll) {
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode);
if (pIter == NULL) break;
++iter;
} else if (!(pDnode = mndAcquireDnode(pMnode, pCfgReq->dnodeId))) {
goto _OVER;
}
SDnodeObj tmpDnode = *pDnode;
if (action == DND_ACTIVE_CODE) {
if (grantAlterActiveCode(pDnode->id, pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
if (TSDB_CODE_DUP_KEY != terrno) {
mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
if (cfgAll) { // alter all dnodes:
if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
if (failRecord) taosArrayPush(failRecord, &pDnode->id);
if (0 == cfgAllErr) cfgAllErr = terrno; // output 1st terrno.
}
} else {
terrno = 0; // no action for dup active code
}
if (cfgAll) continue;
goto _OVER;
}
} else if (action == DND_CONN_ACTIVE_CODE) {
if (grantAlterActiveCode(pDnode->id, pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
if (TSDB_CODE_DUP_KEY != terrno) {
mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
if (cfgAll) {
if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
if (failRecord) taosArrayPush(failRecord, &pDnode->id);
if (0 == cfgAllErr) cfgAllErr = terrno;
}
} else {
terrno = 0;
}
if (cfgAll) continue;
goto _OVER;
}
} else {
terrno = TSDB_CODE_INVALID_CFG;
goto _OVER;
}
if (!pTrans) {
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "config-dnode");
if (!pTrans) goto _OVER;
if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
}
pRaw = mndDnodeActionEncode(&tmpDnode);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
mInfo("dnode:%d, config dnode:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle,
pCfgReq->config, pCfgReq->value);
if (cfgAll) {
sdbRelease(pSdb, pDnode);
pDnode = NULL;
} else {
break;
}
}
if (pTrans && mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
tsGrantHBInterval = TMIN(TMAX(5, iter / 2), 30);
terrno = 0;
_OVER:
if (cfgAll) {
sdbRelease(pSdb, pDnode);
if (cfgAllErr != 0) terrno = cfgAllErr;
int32_t nFail = taosArrayGetSize(failRecord);
if (nFail > 0) {
mError("config dnode, cfg:%d, app:%p config:%s value:%s. total:%d, fail:%d", pCfgReq->dnodeId, pReq->info.ahandle,
pCfgReq->config, pCfgReq->value, iter, nFail);
}
} else {
mndReleaseDnode(pMnode, pDnode);
}
sdbCancelFetch(pSdb, pIter);
mndTransDrop(pTrans);
sdbFreeRaw(pRaw);
taosArrayDestroy(failRecord);
return terrno;
}
static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb;
@ -1013,8 +925,10 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) {
}
code = mndCreateDnode(pMnode, pReq, &createReq);
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
tsGrantHBInterval = 5;
if (code == 0) {
code = TSDB_CODE_ACTION_IN_PROGRESS;
tsGrantHBInterval = 5;
}
char obj[200] = {0};
sprintf(obj, "%s:%d", createReq.fqdn, createReq.port);
@ -1310,34 +1224,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
strcpy(dcfgReq.config, "supportvnodes");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
} else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) {
int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE;
int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11;
if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) {
mError("dnode:%d, failed to config activeCode since invalid conf:%s", cfgReq.dnodeId, cfgReq.config);
terrno = TSDB_CODE_INVALID_CFG;
goto _err_out;
}
int32_t vlen = strlen(cfgReq.value);
if (vlen > 0 && ((opt == DND_ACTIVE_CODE && vlen != (TSDB_ACTIVE_KEY_LEN - 1)) ||
(opt == DND_CONN_ACTIVE_CODE &&
(vlen > (TSDB_CONN_ACTIVE_KEY_LEN - 1) || vlen < (TSDB_ACTIVE_KEY_LEN - 1))))) {
mError("dnode:%d, failed to config activeCode since invalid vlen:%d. conf:%s, val:%s", cfgReq.dnodeId, vlen,
cfgReq.config, cfgReq.value);
terrno = TSDB_CODE_INVALID_CFG;
goto _err_out;
}
strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value);
if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) {
mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr());
terrno = TSDB_CODE_INVALID_CFG;
goto _err_out;
}
tFreeSMCfgDnodeReq(&cfgReq);
return 0;
} else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) {
int32_t optLen = strlen("s3blocksize");
int32_t flag = -1;
@ -1497,11 +1383,7 @@ static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
taosMemoryFreeClear(b);
#ifdef TD_ENTERPRISE
STR_TO_VARSTR(buf, pDnode->active);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, buf, false);
STR_TO_VARSTR(buf, pDnode->connActive);
STR_TO_VARSTR(buf, pDnode->machineId);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, buf, false);
#endif

110
source/dnode/mnode/impl/src/mndGrant.c
View File

@ -19,6 +19,15 @@
#ifndef _GRANT
#define GRANT_ITEM_SHOW(display) \
do { \
cols++; \
pColInfo = taosArrayGet(pBlock->pDataBlock, cols); \
src = (display); \
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); \
colDataSetVal(pColInfo, numOfRows, tmp, false); \
} while (0)
static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
int32_t numOfRows = 0;
int32_t cols = 0;
@ -31,95 +40,32 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
GRANT_ITEM_SHOW("unlimited");
GRANT_ITEM_SHOW("limited");
GRANT_ITEM_SHOW("false");
GRANT_ITEM_SHOW("ungranted");
GRANT_ITEM_SHOW("unlimited");
GRANT_ITEM_SHOW("unlimited");
GRANT_ITEM_SHOW("unlimited");
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "false";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
cols++;
pColInfo = taosArrayGet(pBlock->pDataBlock, cols);
src = "unlimited";
STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32);
colDataSetVal(pColInfo, numOfRows, tmp, false);
numOfRows++;
++numOfRows;
}
pShow->numOfRows += numOfRows;
return numOfRows;
}
static int32_t mndRetrieveGrantFull(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveGrantLogs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndRetrieveMachines(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { return 0; }
static int32_t mndProcessGrantHB(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; }
int32_t mndInitGrant(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_FULL, mndRetrieveGrantFull);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS_LOGS, mndRetrieveGrantLogs);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_MACHINES, mndRetrieveMachines);
mndSetMsgHandle(pMnode, TDMT_MND_GRANT_HB_TIMER, mndProcessGrantHB);
return 0;
}
@ -129,12 +75,10 @@ void grantParseParameter() { mError("can't parsed parameter k"); }
void grantReset(SMnode *pMnode, EGrantType grant, uint64_t value) {}
void grantAdd(EGrantType grant, uint64_t value) {}
void grantRestore(EGrantType grant, uint64_t value) {}
char *tGetMachineId() { return NULL; };
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
int32_t grantAlterActiveCode(int32_t did, const char *old, const char *new, char *out, int8_t type) {
return TSDB_CODE_SUCCESS;
}
int32_t mndProcessConfigGrantReq(SMnode *pMnode, SRpcMsg *pReq, SMCfgClusterReq *pCfg) { return 0; }
#endif
void mndGenerateMachineCode() { grantParseParameter(); }
void mndGenerateMachineCode() { grantParseParameter(); }

2
source/dnode/mnode/impl/src/mndMain.c
View File

@ -193,7 +193,7 @@ static void mndPullupGrant(SMnode *pMnode) {
if (pReq != NULL) {
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_GRANT_HB_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9527};
tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg);
tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg);
}
}

118
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -234,36 +234,72 @@ int32_t doAddShuffleSinkTask(SMnode* pMnode, SArray* pTaskList, SStreamObj* pStr
int32_t doAddSinkTask(SStreamObj* pStream, SArray* pTaskList, SMnode* pMnode, int32_t vgId, SVgObj* pVgroup,
SEpSet* pEpset, bool isFillhistory) {
int64_t uid = (isFillhistory)? pStream->hTaskUid:pStream->uid;
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SINK, isFillhistory, 0, pTaskList, pStream->conf.fillHistory);
int64_t uid = (isFillhistory) ? pStream->hTaskUid : pStream->uid;
SStreamTask* pTask =
tNewStreamTask(uid, TASK_LEVEL__SINK, pEpset, isFillhistory, 0, pTaskList, pStream->conf.fillHistory);
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
epsetAssign(&(pTask)->info.mnodeEpset, pEpset);
pTask->info.nodeId = vgId;
pTask->info.epSet = mndGetVgroupEpset(pMnode, pVgroup);
mndSetSinkTaskInfo(pStream, pTask);
return 0;
}
static int64_t getVgroupLastVer(const SArray* pList, int32_t vgId) {
for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
SVgroupVer* pVer = taosArrayGet(pList, i);
if (pVer->vgId == vgId) {
return pVer->ver;
}
}
mError("failed to find the vgId:%d for extract last version", vgId);
return -1;
}
static void streamTaskSetDataRange(SStreamTask* pTask, int64_t skey, SArray* pVerList, int32_t vgId) {
int64_t latestVer = getVgroupLastVer(pVerList, vgId);
if (latestVer < 0) {
latestVer = 0;
}
// set the correct ts, which is the last key of queried table.
SDataRange* pRange = &pTask->dataRange;
STimeWindow* pWindow = &pRange->window;
if (pTask->info.fillHistory) {
pWindow->skey = INT64_MIN;
pWindow->ekey = skey - 1;
pRange->range.minVer = 0;
pRange->range.maxVer = latestVer;
mDebug("add fill-history source task 0x%x timeWindow:%" PRId64 "-%" PRId64 " verRange:%" PRId64 "-%" PRId64,
pTask->id.taskId, pWindow->skey, pWindow->ekey, pRange->range.minVer, pRange->range.maxVer);
} else {
pWindow->skey = skey;
pWindow->ekey = INT64_MAX;
pRange->range.minVer = latestVer + 1;
pRange->range.maxVer = INT64_MAX;
mDebug("add source task 0x%x timeWindow:%" PRId64 "-%" PRId64 " verRange:%" PRId64 "-%" PRId64,
pTask->id.taskId, pWindow->skey, pWindow->ekey, pRange->range.minVer, pRange->range.maxVer);
}
}
static int32_t addSourceTask(SMnode* pMnode, SVgObj* pVgroup, SArray* pTaskList, SArray* pSinkTaskList,
SStreamObj* pStream, SSubplan* plan, uint64_t uid, SEpSet* pEpset, bool fillHistory,
bool hasExtraSink, int64_t nextWindowSkey, bool hasFillHistory) {
SStreamTask* pTask =
tNewStreamTask(uid, TASK_LEVEL__SOURCE, fillHistory, pStream->conf.triggerParam, pTaskList, hasFillHistory);
SStreamObj* pStream, SSubplan* plan, uint64_t uid, SEpSet* pEpset, int64_t skey,
SArray* pVerList, bool fillHistory, bool hasExtraSink, bool hasFillHistory) {
int64_t t = pStream->conf.triggerParam;
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, fillHistory, t, pTaskList, hasFillHistory);
if (pTask == NULL) {
return terrno;
}
epsetAssign(&pTask->info.mnodeEpset, pEpset);
STimeWindow* pWindow = &pTask->dataRange.window;
pWindow->skey = INT64_MIN;
pWindow->ekey = nextWindowSkey - 1;
mDebug("add source task 0x%x window:%" PRId64 " - %" PRId64, pTask->id.taskId, pWindow->skey, pWindow->ekey);
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
// sink or dispatch
if (hasExtraSink) {
@ -308,7 +344,7 @@ static void setHTasksId(SArray* pTaskList, const SArray* pHTaskList) {
}
static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan* pPlan, SStreamObj* pStream,
SEpSet* pEpset, bool hasExtraSink, int64_t nextWindowSkey) {
SEpSet* pEpset, bool hasExtraSink, int64_t skey, SArray* pVerList) {
// create exec stream task, since only one level, the exec task is also the source task
SArray* pTaskList = addNewTaskList(pStream->tasks);
SSdb* pSdb = pMnode->pSdb;
@ -345,8 +381,8 @@ static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan*
// new stream task
SArray** pSinkTaskList = taosArrayGet(pStream->tasks, SINK_NODE_LEVEL);
int32_t code = addSourceTask(pMnode, pVgroup, pTaskList, *pSinkTaskList, pStream, plan, pStream->uid, pEpset,
false, hasExtraSink, nextWindowSkey, pStream->conf.fillHistory);
int32_t code = addSourceTask(pMnode, pVgroup, pTaskList, *pSinkTaskList, pStream, plan, pStream->uid, pEpset, skey,
pVerList, false, hasExtraSink, pStream->conf.fillHistory);
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pSdb, pVgroup);
return -1;
@ -354,8 +390,8 @@ static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan*
if (pStream->conf.fillHistory) {
SArray** pHSinkTaskList = taosArrayGet(pStream->pHTasksList, SINK_NODE_LEVEL);
code = addSourceTask(pMnode, pVgroup, pHTaskList, *pHSinkTaskList, pStream, plan, pStream->hTaskUid,
pEpset, true, hasExtraSink, nextWindowSkey, true);
code = addSourceTask(pMnode, pVgroup, pHTaskList, *pHSinkTaskList, pStream, plan, pStream->hTaskUid, pEpset, skey,
pVerList, true, hasExtraSink, true);
}
sdbRelease(pSdb, pVgroup);
@ -371,24 +407,17 @@ static int32_t addSourceTasksForOneLevelStream(SMnode* pMnode, const SQueryPlan*
return TSDB_CODE_SUCCESS;
}
static int32_t doAddSourceTask(SArray* pTaskList, bool isFillhistory, int64_t uid, SStreamTask* pDownstreamTask,
SMnode* pMnode, SSubplan* pPlan, SVgObj* pVgroup, SEpSet* pEpset,
int64_t nextWindowSkey, bool hasFillHistory) {
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, isFillhistory, 0, pTaskList, hasFillHistory);
static int32_t addSourceTaskForMultiLevelStream(SArray* pTaskList, bool isFillhistory, int64_t uid,
SStreamTask* pDownstreamTask, SMnode* pMnode, SSubplan* pPlan,
SVgObj* pVgroup, SEpSet* pEpset, int64_t skey, SArray* pVerList,
bool hasFillHistory) {
SStreamTask* pTask = tNewStreamTask(uid, TASK_LEVEL__SOURCE, pEpset, isFillhistory, 0, pTaskList, hasFillHistory);
if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
epsetAssign(&(pTask)->info.mnodeEpset, pEpset);
// set the correct ts, which is the last key of queried table.
STimeWindow* pWindow = &pTask->dataRange.window;
pWindow->skey = INT64_MIN;
pWindow->ekey = nextWindowSkey - 1;
mDebug("s-task:0x%x level:%d set time window:%" PRId64 " - %" PRId64, pTask->id.taskId, pTask->info.taskLevel,
pWindow->skey, pWindow->ekey);
streamTaskSetDataRange(pTask, skey, pVerList, pVgroup->vgId);
// all the source tasks dispatch result to a single agg node.
streamTaskSetFixedDownstreamInfo(pTask, pDownstreamTask);
@ -401,14 +430,12 @@ static int32_t doAddSourceTask(SArray* pTaskList, bool isFillhistory, int64_t ui
static int32_t doAddAggTask(uint64_t uid, SArray* pTaskList, SArray* pSinkNodeList, SMnode* pMnode, SStreamObj* pStream,
SEpSet* pEpset, bool fillHistory, SStreamTask** pAggTask, bool hasFillhistory) {
*pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, fillHistory, pStream->conf.triggerParam, pTaskList, hasFillhistory);
*pAggTask = tNewStreamTask(uid, TASK_LEVEL__AGG, pEpset, fillHistory, pStream->conf.triggerParam, pTaskList, hasFillhistory);
if (*pAggTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
epsetAssign(&(*pAggTask)->info.mnodeEpset, pEpset);
// dispatch
if (mndAddDispatcherForInternalTask(pMnode, pStream, pSinkNodeList, *pAggTask) < 0) {
return -1;
@ -492,7 +519,7 @@ static int32_t addAggTask(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan
static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPlan, SStreamObj* pStream,
SStreamTask* pDownstreamTask, SStreamTask* pHDownstreamTask,
SEpSet* pEpset, int64_t nextWindowSkey) {
SEpSet* pEpset, int64_t skey, SArray* pVerList) {
SArray* pSourceTaskList = addNewTaskList(pStream->tasks);
SArray* pHSourceTaskList = NULL;
@ -521,8 +548,8 @@ static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPl
continue;
}
int32_t code = doAddSourceTask(pSourceTaskList, false, pStream->uid, pDownstreamTask, pMnode, plan, pVgroup, pEpset,
nextWindowSkey, pStream->conf.fillHistory);
int32_t code = addSourceTaskForMultiLevelStream(pSourceTaskList, false, pStream->uid, pDownstreamTask, pMnode, plan, pVgroup, pEpset,
skey, pVerList, pStream->conf.fillHistory);
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pSdb, pVgroup);
terrno = code;
@ -530,8 +557,8 @@ static int32_t addSourceTasksForMultiLevelStream(SMnode* pMnode, SQueryPlan* pPl
}
if (pStream->conf.fillHistory) {
code = doAddSourceTask(pHSourceTaskList, true, pStream->hTaskUid, pHDownstreamTask, pMnode, plan, pVgroup, pEpset,
nextWindowSkey, pStream->conf.fillHistory);
code = addSourceTaskForMultiLevelStream(pHSourceTaskList, true, pStream->hTaskUid, pHDownstreamTask, pMnode, plan, pVgroup, pEpset,
skey, pVerList, pStream->conf.fillHistory);
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pSdb, pVgroup);
return code;
@ -580,7 +607,8 @@ static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpst
}
}
static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, int64_t nextWindowSkey, SEpSet* pEpset) {
static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan* pPlan, SEpSet* pEpset, int64_t skey,
SArray* pVerList) {
SSdb* pSdb = pMnode->pSdb;
int32_t numOfPlanLevel = LIST_LENGTH(pPlan->pSubplans);
@ -637,15 +665,15 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
}
// source level
return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, nextWindowSkey);
return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, skey, pVerList);
} else if (numOfPlanLevel == 1) {
return addSourceTasksForOneLevelStream(pMnode, pPlan, pStream, pEpset, hasExtraSink, nextWindowSkey);
return addSourceTasksForOneLevelStream(pMnode, pPlan, pStream, pEpset, hasExtraSink, skey, pVerList);
}
return 0;
}
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t nextWindowSkey) {
int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t skey, SArray* pVgVerList) {
SQueryPlan* pPlan = qStringToQueryPlan(pStream->physicalPlan);
if (pPlan == NULL) {
terrno = TSDB_CODE_QRY_INVALID_INPUT;
@ -655,7 +683,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream, int64_t nextWindo
SEpSet mnodeEpset = {0};
mndGetMnodeEpSet(pMnode, &mnodeEpset);
int32_t code = doScheduleStream(pStream, pMnode, pPlan, nextWindowSkey, &mnodeEpset);
int32_t code = doScheduleStream(pStream, pMnode, pPlan, &mnodeEpset, skey, pVgVerList);
qDestroyQueryPlan(pPlan);
return code;

6
source/dnode/mnode/impl/src/mndShow.c
View File

@ -123,6 +123,12 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_COMPACT;
} else if (strncasecmp(name, TSDB_INS_TABLE_COMPACT_DETAILS, len) == 0) {
type = TSDB_MGMT_TABLE_COMPACT_DETAIL;
} else if (strncasecmp(name, TSDB_INS_TABLE_GRANTS_FULL, len) == 0) {
type = TSDB_MGMT_TABLE_GRANTS_FULL;
} else if (strncasecmp(name, TSDB_INS_TABLE_GRANTS_LOGS, len) == 0) {
type = TSDB_MGMT_TABLE_GRANTS_LOGS;
} else if (strncasecmp(name, TSDB_INS_TABLE_MACHINES, len) == 0) {
type = TSDB_MGMT_TABLE_MACHINES;
} else {
mError("invalid show name:%s len:%d", name, len);
}

2
source/dnode/mnode/impl/src/mndSma.c
View File

@ -638,7 +638,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
if (mndSetCreateSmaVgroupCommitLogs(pMnode, pTrans, &streamObj.fixedSinkVg) != 0) goto _OVER;
if (mndSetUpdateSmaStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER;
if (mndSetCreateSmaVgroupRedoActions(pMnode, pTrans, pDb, &streamObj.fixedSinkVg, &smaObj) != 0) goto _OVER;
if (mndScheduleStream(pMnode, &streamObj, 1685959190000) != 0) goto _OVER;
if (mndScheduleStream(pMnode, &streamObj, 1685959190000, NULL) != 0) goto _OVER;
if (mndPersistStream(pTrans, &streamObj) != 0) goto _OVER;
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;

82
source/dnode/mnode/impl/src/mndStream.c
View File

@ -62,8 +62,6 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot);
static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len);
static void freeCheckpointCandEntry(void *);
static void freeTaskList(void *param);
static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw);
SSdbRaw *mndStreamSeqActionEncode(SStreamObj *pStream);
@ -121,17 +119,7 @@ int32_t mndInitStream(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);
taosThreadMutexInit(&execInfo.lock, NULL);
_hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR);
execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK);
execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK);
taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry);
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
mndInitExecInfo();
if (sdbSetTable(pMnode->pSdb, table) != 0) {
return -1;
@ -644,13 +632,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
int32_t sqlLen = 0;
terrno = TSDB_CODE_SUCCESS;
if(grantCheck(TSDB_GRANT_STREAMS) < 0){
terrno = TSDB_CODE_GRANT_STREAM_LIMITED;
return -1;
if ((terrno = grantCheck(TSDB_GRANT_STREAMS)) < 0) {
goto _OVER;
}
SCMCreateStreamReq createStreamReq = {0};
if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createStreamReq) != 0) {
SCMCreateStreamReq createReq = {0};
if (tDeserializeSCMCreateStreamReq(pReq->pCont, pReq->contLen, &createReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
@ -659,17 +646,17 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
terrno = TSDB_CODE_MND_INVALID_PLATFORM;
goto _OVER;
#endif
mInfo("stream:%s, start to create, sql:%s", createStreamReq.name, createStreamReq.sql);
mInfo("stream:%s, start to create, sql:%s", createReq.name, createReq.sql);
if (mndCheckCreateStreamReq(&createStreamReq) != 0) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
if (mndCheckCreateStreamReq(&createReq) != 0) {
mError("stream:%s, failed to create since %s", createReq.name, terrstr());
goto _OVER;
}
pStream = mndAcquireStream(pMnode, createStreamReq.name);
pStream = mndAcquireStream(pMnode, createReq.name);
if (pStream != NULL) {
if (createStreamReq.igExists) {
mInfo("stream:%s, already exist, ignore exist is set", createStreamReq.name);
if (createReq.igExists) {
mInfo("stream:%s, already exist, ignore exist is set", createReq.name);
goto _OVER;
} else {
terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST;
@ -679,16 +666,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
goto _OVER;
}
if (createStreamReq.sql != NULL) {
sqlLen = strlen(createStreamReq.sql);
if (createReq.sql != NULL) {
sqlLen = strlen(createReq.sql);
sql = taosMemoryMalloc(sqlLen + 1);
memset(sql, 0, sqlLen + 1);
memcpy(sql, createStreamReq.sql, sqlLen);
memcpy(sql, createReq.sql, sqlLen);
}
// build stream obj from request
if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createReq) < 0) {
mError("stream:%s, failed to create since %s", createReq.name, terrstr());
goto _OVER;
}
@ -702,23 +689,23 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
}
// create stb for stream
if (createStreamReq.createStb == STREAM_CREATE_STABLE_TRUE &&
if (createReq.createStb == STREAM_CREATE_STABLE_TRUE &&
mndCreateStbForStream(pMnode, pTrans, &streamObj, pReq->info.conn.user) < 0) {
mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createStreamReq.name, terrstr());
mError("trans:%d, failed to create stb for stream %s since %s", pTrans->id, createReq.name, terrstr());
mndTransDrop(pTrans);
goto _OVER;
}
// schedule stream task for stream obj
if (mndScheduleStream(pMnode, &streamObj, createStreamReq.lastTs) < 0) {
mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr());
if (mndScheduleStream(pMnode, &streamObj, createReq.lastTs, createReq.pVgroupVerList) < 0) {
mError("stream:%s, failed to schedule since %s", createReq.name, terrstr());
mndTransDrop(pTrans);
goto _OVER;
}
// add stream to trans
if (mndPersistStream(pTrans, &streamObj) < 0) {
mError("stream:%s, failed to schedule since %s", createStreamReq.name, terrstr());
mError("stream:%s, failed to schedule since %s", createReq.name, terrstr());
mndTransDrop(pTrans);
goto _OVER;
}
@ -749,10 +736,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
taosThreadMutexUnlock(&execInfo.lock);
SName dbname = {0};
tNameFromString(&dbname, createStreamReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
SName name = {0};
tNameFromString(&name, createStreamReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
// reuse this function for stream
if (sql != NULL && sqlLen > 0) {
@ -765,11 +752,11 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
_OVER:
if (terrno != TSDB_CODE_SUCCESS && terrno != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stream:%s, failed to create since %s", createStreamReq.name, terrstr());
mError("stream:%s, failed to create since %s", createReq.name, terrstr());
}
mndReleaseStream(pMnode, pStream);
tFreeSCMCreateStreamReq(&createStreamReq);
tFreeSCMCreateStreamReq(&createReq);
tFreeStreamObj(&streamObj);
if (sql != NULL) {
taosMemoryFreeClear(sql);
@ -1629,7 +1616,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node;
SStreamObj *pStream = NULL;
if(grantCheck(TSDB_GRANT_STREAMS) < 0){
if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){
terrno = TSDB_CODE_GRANT_EXPIRED;
return -1;
}
@ -1843,6 +1830,11 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
}
}
// no need to build the trans to handle the vgroup upddate
if (pTrans == NULL) {
return 0;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
@ -2113,16 +2105,6 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
}
void freeCheckpointCandEntry(void *param) {
SCheckpointCandEntry *pEntry = param;
taosMemoryFreeClear(pEntry->pName);
}
void freeTaskList(void* param) {
SArray** pList = (SArray **)param;
taosArrayDestroy(*pList);
}
static void doAddTaskId(SArray* pList, int32_t taskId, int64_t uid, int32_t numOfTotal) {
int32_t num = taosArrayGetSize(pList);
for(int32_t i = 0; i < num; ++i) {
@ -2198,4 +2180,4 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
taosThreadMutexUnlock(&execInfo.lock);
return 0;
}
}

14
source/dnode/mnode/impl/src/mndStreamHb.c
View File

@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SArray *pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo));
SArray *pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask));
if(grantCheck(TSDB_GRANT_STREAMS) < 0){
if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){
if(suspendAllStreams(pMnode, &pReq->info) < 0){
return -1;
}
@ -316,16 +316,20 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
// current checkpoint is failed, rollback from the checkpoint trans
// kill the checkpoint trans and then set all tasks status to be normal
if (taosArrayGetSize(pFailedTasks) > 0) {
bool allReady = true;
SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady);
taosArrayDestroy(p);
bool allReady = true;
if (pMnode != NULL) {
SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady);
taosArrayDestroy(p);
} else {
allReady = false;
}
if (allReady || snodeChanged) {
// if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
for(int32_t i = 0; i < taosArrayGetSize(pFailedTasks); ++i) {
SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedTasks, i);
mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status",
pInfo->checkpointId, pInfo->transId);
pInfo->checkpointId, pInfo->transId);
mndResetStatusFromCheckpoint(pMnode, pInfo->streamUid, pInfo->transId);
}

24
source/dnode/mnode/impl/src/mndStreamUtil.c
View File

@ -543,3 +543,27 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *
taosWUnLockLatch(&pStream->lock);
return 0;
}
static void freeCheckpointCandEntry(void *param) {
SCheckpointCandEntry *pEntry = param;
taosMemoryFreeClear(pEntry->pName);
}
static void freeTaskList(void* param) {
SArray** pList = (SArray **)param;
taosArrayDestroy(*pList);
}
void mndInitExecInfo() {
taosThreadMutexInit(&execInfo.lock, NULL);
_hash_fn_t fn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR);
execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK);
execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.transMgmt.pWaitingList = taosHashInit(32, fn, true, HASH_NO_LOCK);
execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK);
taosHashSetFreeFp(execInfo.transMgmt.pWaitingList, freeCheckpointCandEntry);
taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList);
}

4
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -599,6 +599,8 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnConflct conflict,
pTrans->originRpcType = pReq->msgType;
}
mInfo("trans:%d, create transaction:%s, origin:%s", pTrans->id, pTrans->opername, opername);
mTrace("trans:%d, local object is created, data:%p", pTrans->id, pTrans);
return pTrans;
}
@ -845,6 +847,8 @@ int32_t mndTransCheckConflict(SMnode *pMnode, STrans *pTrans) {
}
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
if(pTrans == NULL) return -1;
if (mndTransCheckConflict(pMnode, pTrans) != 0) {
return -1;
}

2
source/dnode/mnode/impl/test/CMakeLists.txt
View File

@ -4,7 +4,7 @@ add_subdirectory(acct)
#add_subdirectory(db)
#add_subdirectory(dnode)
add_subdirectory(func)
#add_subdirectory(mnode)
add_subdirectory(stream)
add_subdirectory(profile)
add_subdirectory(qnode)
add_subdirectory(sdb)

13
source/dnode/mnode/impl/test/stream/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
SET(CMAKE_CXX_STANDARD 11)
aux_source_directory(. MNODE_STREAM_TEST_SRC)
add_executable(streamTest ${MNODE_STREAM_TEST_SRC})
target_link_libraries(
streamTest
PRIVATE dnode gtest
)
add_test(
NAME streamTest
COMMAND streamTest
)

155
source/dnode/mnode/impl/test/stream/stream.cpp Normal file
View File

@ -0,0 +1,155 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <iostream>
#include <gtest/gtest.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <libs/stream/tstream.h>
#include <libs/transport/trpc.h>
#include "../../inc/mndStream.h"
namespace {
SRpcMsg buildHbReq() {
SStreamHbMsg msg = {0};
msg.vgId = 1;
msg.numOfTasks = 5;
msg.pTaskStatus = taosArrayInit(4, sizeof(STaskStatusEntry));
for (int32_t i = 0; i < 4; ++i) {
STaskStatusEntry entry = {0};
entry.nodeId = i + 1;
entry.stage = 1;
entry.id.taskId = i + 1;
entry.id.streamId = 999;
if (i == 0) {
entry.stage = 4;
}
taosArrayPush(msg.pTaskStatus, &entry);
}
// (p->checkpointId != 0) && p->checkpointFailed
// add failed checkpoint info
{
STaskStatusEntry entry = {0};
entry.nodeId = 5;
entry.stage = 1;
entry.id.taskId = 5;
entry.id.streamId = 999;
entry.checkpointId = 1;
entry.checkpointFailed = true;
taosArrayPush(msg.pTaskStatus, &entry);
}
int32_t tlen = 0;
int32_t code = 0;
SEncoder encoder;
void* buf = NULL;
SRpcMsg msg1 = {0};
msg1.info.noResp = 1;
tEncodeSize(tEncodeStreamHbMsg, &msg, tlen, code);
if (code < 0) {
goto _end;
}
buf = rpcMallocCont(tlen);
if (buf == NULL) {
goto _end;
}
tEncoderInit(&encoder, (uint8_t*)buf, tlen);
if ((code = tEncodeStreamHbMsg(&encoder, &msg)) < 0) {
rpcFreeCont(buf);
goto _end;
}
tEncoderClear(&encoder);
initRpcMsg(&msg1, TDMT_MND_STREAM_HEARTBEAT, buf, tlen);
taosArrayDestroy(msg.pTaskStatus);
return msg1;
_end:
return msg1;
}
void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskId) {
SStreamExecInfo* pExecNode = &execInfo;
pTask->id.streamId = streamId;
pTask->id.taskId = taskId;
pTask->info.nodeId = nodeId;
STaskId id;
id.streamId = pTask->id.streamId;
id.taskId = pTask->id.taskId;
STaskStatusEntry entry;
streamTaskStatusInit(&entry, pTask);
entry.stage = 1;
entry.status = TASK_STATUS__READY;
taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry));
taosArrayPush(pExecNode->pTaskList, &id);
}
void initStreamExecInfo() {
SStreamExecInfo* pExecNode = &execInfo;
SStreamTask task = {0};
setTask(&task, 1, 999, 1);
setTask(&task, 1, 999, 2);
setTask(&task, 1, 999, 3);
setTask(&task, 1, 999, 4);
setTask(&task, 2, 999, 5);
}
void initNodeInfo() {
execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry));
SNodeEntry entry = {0};
entry.nodeId = 2;
entry.stageUpdated = true;
taosArrayPush(execInfo.pNodeList, &entry);
}
} // namespace
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(mndHbTest, handle_error_in_hb) {
mndInitExecInfo();
initStreamExecInfo();
initNodeInfo();
SRpcMsg msg = buildHbReq();
int32_t code = mndProcessStreamHb(&msg);
rpcFreeCont(msg.pCont);
}
#pragma GCC diagnostic pop

3
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -152,7 +152,8 @@ typedef enum {
SDB_STREAM_SEQ = 23,
SDB_COMPACT = 24,
SDB_COMPACT_DETAIL = 25,
SDB_MAX = 26
SDB_GRANT = 26, // grant log
SDB_MAX = 27
} ESdbType;
typedef struct SSdbRaw {

4
source/dnode/vnode/inc/vnode.h
View File

@ -90,6 +90,8 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num);
int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num);
int32_t vnodeGetAllCtbNum(SVnode *pVnode, int64_t *num);
int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
int32_t vnodeGetLoadLite(SVnode *pVnode, SVnodeLoadLite *pLoad);
@ -180,7 +182,6 @@ int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList,
int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, const int32_t *dstSlotIds,
SArray *pTableUids);
void *tsdbCacherowsReaderClose(void *pReader);
int32_t tsdbGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid);
void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity);
size_t tsdbCacheGetCapacity(SVnode *pVnode);
@ -233,6 +234,7 @@ int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, i
bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids);
int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr);
int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet);
int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished);
// sma
int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days);

7
source/dnode/vnode/src/inc/tq.h
View File

@ -97,7 +97,6 @@ typedef struct {
struct STQ {
SVnode* pVnode;
char* path;
int64_t walLogLastVer;
SRWLatch lock;
SHashObj* pPushMgr; // subKey -> STqHandle
SHashObj* pHandle; // subKey -> STqHandle
@ -153,14 +152,14 @@ char* tqOffsetBuildFName(const char* path, int32_t fVer);
int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
// tq util
int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type);
int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg);
int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId,
int32_t type, int64_t sver, int64_t ever);
int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset);
void tqUpdateNodeStage(STQ* pTq, bool isLeader);
int32_t setDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id);
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id);
int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,

3
source/dnode/vnode/src/inc/tsdb.h
View File

@ -279,6 +279,7 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx);
// tsdbRead.c ==============================================================================================
int32_t tsdbTakeReadSnap2(STsdbReader *pReader, _query_reseek_func_t reseek, STsdbReadSnap **ppSnap);
void tsdbUntakeReadSnap2(STsdbReader *pReader, STsdbReadSnap *pSnap, bool proactive);
int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid);
// tsdbMerge.c ==============================================================================================
typedef struct {
@ -970,8 +971,6 @@ static FORCE_INLINE TSDBROW *tsdbTbDataIterGet(STbDataIter *pIter) {
return pIter->pRow;
}
int32_t tRowInfoCmprFn(const void *p1, const void *p2);
typedef struct {
int64_t suid;
int64_t uid;

2
source/dnode/vnode/src/meta/metaIdx.c
View File

@ -62,6 +62,7 @@ int metaOpenIdx(SMeta *pMeta) {
return 0;
}
#ifdef BUILD_NO_CALL
void metaCloseIdx(SMeta *pMeta) { /* TODO */
#if 0
if (pMeta->pIdx) {
@ -114,3 +115,4 @@ int metaRemoveTableFromIdx(SMeta *pMeta, tb_uid_t uid) {
// TODO
return 0;
}
#endif

2
source/dnode/vnode/src/meta/metaOpen.c
View File

@ -273,7 +273,9 @@ static void metaCleanup(SMeta **ppMeta) {
if (pMeta) {
if (pMeta->pEnv) metaAbort(pMeta);
if (pMeta->pCache) metaCacheClose(pMeta);
#ifdef BUILD_NO_CALL
if (pMeta->pIdx) metaCloseIdx(pMeta);
#endif
if (pMeta->pStreamDb) tdbTbClose(pMeta->pStreamDb);
if (pMeta->pNcolIdx) tdbTbClose(pMeta->pNcolIdx);
if (pMeta->pBtimeIdx) tdbTbClose(pMeta->pBtimeIdx);

2
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -1554,7 +1554,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA
}
_resume_delete:
version = RSMA_EXEC_MSG_VER(msg);
if ((terrno = extractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version,
if ((terrno = tqExtractDelDataBlock(RSMA_EXEC_MSG_BODY(msg), RSMA_EXEC_MSG_LEN(msg), version,
&packData.pDataBlock, 1))) {
taosFreeQitem(msg);
goto _err;

4
source/dnode/vnode/src/sma/smaTimeRange.c
View File

@ -203,7 +203,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
if (index == NULL) { // no data yet, append it
code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
if (code != TSDB_CODE_SUCCESS) {
continue;
}
@ -213,7 +213,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1;
taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
} else {
code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
if (code != TSDB_CODE_SUCCESS) {
continue;
}

23
source/dnode/vnode/src/tq/tq.c
View File

@ -66,7 +66,6 @@ STQ* tqOpen(const char* path, SVnode* pVnode) {
pTq->path = taosStrdup(path);
pTq->pVnode = pVnode;
pTq->walLogLastVer = pVnode->pWal->vers.lastVer;
pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK);
taosHashSetFreeFp(pTq->pHandle, tqDestroyTqHandle);
@ -738,10 +737,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
if (pTask->pState == NULL) {
tqError("s-task:%s (vgId:%d) failed to open state for task", pTask->id.idStr, vgId);
return -1;
} else {
tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
}
tqDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState);
if (pTask->info.fillHistory) {
restoreStreamTaskId(pTask, &taskId);
}
@ -846,17 +844,19 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
if (pTask->info.fillHistory) {
tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64
" ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
(int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam, nextProcessVer);
} else {
tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64
" ms, inputVer:%" PRId64,
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
(int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
(int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam, nextProcessVer);
}
return 0;
@ -876,12 +876,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
}
static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) {
const char* id = pTask->id.idStr;
int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer;
// if it's an source task, extract the last version in wal.
const char* id = pTask->id.idStr;
int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer;
SVersionRange* pRange = &pTask->dataRange.range;
// if it's an source task, extract the last version in wal.
bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
pTask->execInfo.step2Start = taosGetTimestampMs();
@ -1055,7 +1054,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode));
// let's continue scan data in the wal files
if(code == 0 && pReq->reqType >= 0){
if (code == 0 && (pReq->reqType >= 0 || pReq->reqType == STREAM_EXEC_T_RESUME_TASK)) {
tqScanWalAsync(pTq, false);
}

2
source/dnode/vnode/src/tq/tqRead.c
View File

@ -344,7 +344,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con
void* pBody = POINTER_SHIFT(pCont->body, sizeof(SMsgHead));
int32_t len = pCont->bodyLen - sizeof(SMsgHead);
code = extractDelDataBlock(pBody, len, ver, (void**)pItem, 0);
code = tqExtractDelDataBlock(pBody, len, ver, (void**)pItem, 0);
if (code == TSDB_CODE_SUCCESS) {
if (*pItem == NULL) {
tqDebug("s-task:%s empty delete msg, discard it, len:%d, ver:%" PRId64, id, len, ver);

8
source/dnode/vnode/src/tq/tqSink.c
View File

@ -746,7 +746,7 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
return TDB_CODE_SUCCESS;
}
int32_t setDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id) {
int32_t numOfRows = pDataBlock->info.rows;
@ -821,7 +821,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
continue;
}
code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
if (code != TSDB_CODE_SUCCESS) {
continue;
}
@ -868,7 +868,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
continue;
}
code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
if (code != TSDB_CODE_SUCCESS) {
continue;
}
@ -878,7 +878,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
} else {
code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
code = tqSetDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
if (code != TSDB_CODE_SUCCESS) {
continue;
}

72
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -388,7 +388,7 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
return 0;
}
int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) {
int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type) {
SDecoder* pCoder = &(SDecoder){0};
SDeleteRes* pRes = &(SDeleteRes){0};
@ -449,3 +449,73 @@ int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void**
return TSDB_CODE_SUCCESS;
}
int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished) {
SStreamMeta* pMeta = pVnode->pTq->pStreamMeta;
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
int32_t code = TSDB_CODE_SUCCESS;
if (pDelay != NULL) {
*pDelay = 0;
}
*fhFinished = false;
if (numOfTasks <= 0) {
return code;
}
// extract the required source task for a given stream, identified by streamId
for (int32_t i = 0; i < numOfTasks; ++i) {
STaskId* pId = taosArrayGet(pMeta->pTaskList, i);
if (pId->streamId != streamId) {
continue;
}
SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId));
if (ppTask == NULL) {
tqError("vgId:%d failed to acquire task:0x%" PRIx64 " in retrieving progress", pMeta->vgId, pId->taskId);
continue;
}
if ((*ppTask)->info.taskLevel != TASK_LEVEL__SOURCE) {
continue;
}
// here we get the required stream source task
SStreamTask* pTask = *ppTask;
*fhFinished = !HAS_RELATED_FILLHISTORY_TASK(pTask);
int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader);
SVersionRange verRange = {0};
walReaderValidVersionRange(pTask->exec.pWalReader, &verRange.minVer, &verRange.maxVer);
SWalReader* pReader = walOpenReader(pTask->exec.pWalReader->pWal, NULL, 0);
if (pReader == NULL) {
tqError("failed to open wal reader to extract exec progress, vgId:%d", pMeta->vgId);
continue;
}
int64_t cur = 0;
int64_t latest = 0;
code = walFetchHead(pReader, ver);
if (code != TSDB_CODE_SUCCESS) {
cur = pReader->pHead->head.ingestTs;
}
code = walFetchHead(pReader, verRange.maxVer);
if (code != TSDB_CODE_SUCCESS) {
latest = pReader->pHead->head.ingestTs;
}
if (pDelay != NULL) { // delay in ms
*pDelay = (latest - cur) / 1000;
}
walCloseReader(pReader);
}
return TSDB_CODE_SUCCESS;
}

9
source/dnode/vnode/src/tqCommon/tqCommon.c
View File

@ -49,7 +49,7 @@ int32_t tqStreamTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, bool restart) {
return 0;
}
int32_t tqStreamOneTaskStartAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId) {
int32_t tqStreamStartOneTaskAsync(SStreamMeta* pMeta, SMsgCb* cb, int64_t streamId, int32_t taskId) {
int32_t vgId = pMeta->vgId;
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
@ -547,7 +547,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
streamMetaWUnLock(pMeta);
if (code < 0) {
tqError("failed to add s-task:0x%x into vgId:%d meta, total:%d, code:%s", vgId, taskId, numOfTasks,
tqError("failed to add s-task:0x%x into vgId:%d meta, existed:%d, code:%s", vgId, taskId, numOfTasks,
tstrerror(code));
tFreeStreamTask(pTask);
return code;
@ -562,9 +562,10 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sve
if (restored) {
SStreamTask* p = streamMetaAcquireTask(pMeta, streamId, taskId);
if (p != NULL && (p->info.fillHistory == 0)) {
tqStreamOneTaskStartAsync(pMeta, cb, streamId, taskId);
if ((p != NULL) && (p->info.fillHistory == 0)) {
tqStreamStartOneTaskAsync(pMeta, cb, streamId, taskId);
}
if (p != NULL) {
streamMetaReleaseTask(pMeta, p);
}

132
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -2294,6 +2294,7 @@ void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInf
pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
pResBlock->info.dataLoad = 1;
pResBlock->info.version = pReader->info.verRange.maxVer;
blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]);
setComposedBlockFlag(pReader, true);
@ -2627,6 +2628,58 @@ static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) {
return (pStatus->pProcMemTableIter != NULL);
}
static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) {
SReaderStatus* pStatus = &pReader->status;
SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader;
SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
SDataBlockInfo* pInfo = &pResBlock->info;
blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt);
pInfo->rows = pScanInfo->numOfRowsInStt;
pInfo->id.uid = pScanInfo->uid;
pInfo->dataLoad = 1;
pInfo->window = pScanInfo->sttWindow;
setComposedBlockFlag(pReader, true);
pScanInfo->sttKeyInfo.nextProcKey = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1;
pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
pScanInfo->lastProcKey = asc ? pScanInfo->sttWindow.ekey : pScanInfo->sttWindow.skey;
pScanInfo->sttBlockReturned = true;
pSttBlockReader->mergeTree.pIter = NULL;
tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s",
pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pResBlock->info.rows, pReader->idStr);
}
static void buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo,
SFileDataBlockInfo* pBlockInfo, int32_t blockIndex) {
// whole block is required, return it directly
SReaderStatus* pStatus = &pReader->status;
SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
pInfo->rows = pBlockInfo->numRow;
pInfo->id.uid = pScanInfo->uid;
pInfo->dataLoad = 0;
pInfo->version = pReader->info.verRange.maxVer;
pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey};
setComposedBlockFlag(pReader, false);
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);
// update the last key for the corresponding table
pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey;
tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, "
"table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
pReader, pScanInfo->uid, blockIndex, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, pBlockInfo->firstKey,
pBlockInfo->lastKey, pReader->idStr);
}
static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
SReaderStatus* pStatus = &pReader->status;
SSttBlockReader* pSttBlockReader = pStatus->fileIter.pSttBlockReader;
@ -2679,28 +2732,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
// if only require the total rows, no need to load data from stt file if it is clean stt blocks
if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) {
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
SDataBlockInfo* pInfo = &pResBlock->info;
blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt);
pInfo->rows = pScanInfo->numOfRowsInStt;
pInfo->id.uid = pScanInfo->uid;
pInfo->dataLoad = 1;
pInfo->window = pScanInfo->sttWindow;
setComposedBlockFlag(pReader, true);
pScanInfo->sttKeyInfo.nextProcKey = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1;
pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
pScanInfo->lastProcKey = asc ? pScanInfo->sttWindow.ekey : pScanInfo->sttWindow.skey;
pScanInfo->sttBlockReturned = true;
pSttBlockReader->mergeTree.pIter = NULL;
tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s",
pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pResBlock->info.rows, pReader->idStr);
buildCleanBlockFromSttFiles(pReader, pScanInfo);
return TSDB_CODE_SUCCESS;
}
@ -2740,10 +2772,11 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
}
}
static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) {
// current active data block not overlap with the stt-files/stt-blocks
static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) {
ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT);
if (pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) {
if ((!hasDataInSttBlock(pScanInfo)) || (pScanInfo->cleanSttBlocks == true)) {
return true;
} else {
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
@ -2793,23 +2826,32 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pScanInfo, pReader->info.order);
code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
} else {
if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) {
// whole block is required, return it directly
SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
pInfo->rows = pBlockInfo->numRow;
pInfo->id.uid = pScanInfo->uid;
pInfo->dataLoad = 0;
pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey};
setComposedBlockFlag(pReader, false);
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);
if (notOverlapWithFiles(pBlockInfo, pScanInfo, asc)) {
int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey;
// update the last key for the corresponding table
pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey;
tsdbDebug("%p uid:%" PRIu64
" clean file block retrieved from file, global index:%d, "
"table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
pReader, pScanInfo->uid, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->numRow,
pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr);
if ((!hasDataInSttBlock(pScanInfo)) || (asc && pBlockInfo->lastKey < keyInStt) ||
(!asc && pBlockInfo->firstKey > keyInStt)) {
if (pScanInfo->cleanSttBlocks && hasDataInSttBlock(pScanInfo)) {
if (asc) { // file block is located before the stt block
ASSERT(pScanInfo->sttWindow.skey > pBlockInfo->lastKey);
} else { // stt block is before the file block
ASSERT(pScanInfo->sttWindow.ekey < pBlockInfo->firstKey);
}
}
buildCleanBlockFromDataFiles(pReader, pScanInfo, pBlockInfo, pBlockIter->index);
} else { // clean stt block
if (asc) {
ASSERT(pScanInfo->sttWindow.ekey < pBlockInfo->firstKey);
} else {
ASSERT(pScanInfo->sttWindow.skey > pBlockInfo->lastKey);
}
// return the stt file block
ASSERT(pReader->info.execMode == READER_EXEC_ROWS && pSttBlockReader->mergeTree.pIter == NULL);
buildCleanBlockFromSttFiles(pReader, pScanInfo);
return TSDB_CODE_SUCCESS;
}
} else {
SBlockData* pBData = &pReader->status.fileBlockData;
tBlockDataReset(pBData);
@ -2820,7 +2862,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
int64_t st = taosGetTimestampUs();
// let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files
pScanInfo->cleanSttBlocks = false;
initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
// no data in stt block, no need to proceed.
@ -2838,8 +2879,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
// data in stt now overlaps with current active file data block, need to composed with file data block.
int64_t lastKeyInStt = getCurrentKeyInSttBlock(pSttBlockReader);
if ((lastKeyInStt >= pBlockInfo->firstKey && asc) ||
(lastKeyInStt <= pBlockInfo->lastKey && (!asc))) {
if ((lastKeyInStt >= pBlockInfo->firstKey && asc) || (lastKeyInStt <= pBlockInfo->lastKey && (!asc))) {
tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
lastKeyInStt, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr);
break;
@ -4993,9 +5033,9 @@ int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader) {
return rows;
}
int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) {
int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid) {
SMetaReader mr = {0};
metaReaderDoInit(&mr, ((SVnode*)pVnode)->pMeta, 0);
metaReaderDoInit(&mr, pMeta, 0);
int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TDB_INVALID_TABLE_ID;
@ -5025,7 +5065,7 @@ int32_t tsdbGetTableSchema(void* pVnode, int64_t uid, STSchema** pSchema, int64_
metaReaderClear(&mr);
// get the newest table schema version
code = metaGetTbTSchemaEx(((SVnode*)pVnode)->pMeta, *suid, uid, -1, pSchema);
code = metaGetTbTSchemaEx(pMeta, *suid, uid, -1, pSchema);
return code;
}

6
source/dnode/vnode/src/vnd/vnodeInitApi.c
View File

@ -91,7 +91,7 @@ void initMetadataAPI(SStoreMeta* pMeta) {
pMeta->getTableTypeByName = metaGetTableTypeByName;
pMeta->getTableNameByUid = metaGetTableNameByUid;
pMeta->getTableSchema = tsdbGetTableSchema; // todo refactor
pMeta->getTableSchema = vnodeGetTableSchema;
pMeta->storeGetTableList = vnodeGetTableList;
pMeta->getCachedTableList = metaGetCachedTableUidList;
@ -135,7 +135,9 @@ void initTqAPI(SStoreTqReader* pTq) {
pTq->tqReaderNextBlockFilterOut = tqNextDataBlockFilterOut;
pTq->tqGetResultBlockTime = tqGetResultBlockTime;
}
pTq->tqGetStreamExecProgress = tqGetStreamExecInfo;
}
void initStateStoreAPI(SStateStore* pStore) {
pStore->streamFileStateInit = streamFileStateInit;

5
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -14,6 +14,7 @@
*/
#include "vnd.h"
#include "tsdb.h"
#define VNODE_GET_LOAD_RESET_VALS(pVar, oVal, vType, tags) \
do { \
@ -703,3 +704,7 @@ void *vnodeGetIvtIdx(void *pVnode) {
}
return metaGetIvtIdx(((SVnode *)pVnode)->pMeta);
}
int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) {
return tsdbGetTableSchema(((SVnode*)pVnode)->pMeta, uid, pSchema, suid);
}

5
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -238,6 +238,11 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int
TSDB_CHECK_CODE(code, lino, _exit);
}
if (submitTbData.flags & SUBMIT_REQ_FROM_FILE) {
code = grantCheck(TSDB_GRANT_CSV);
TSDB_CHECK_CODE(code, lino, _exit);
}
int64_t uid;
if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) {
code = vnodePreprocessCreateTableReq(pVnode, pCoder, btimeMs, &uid);

4
source/libs/catalog/src/ctgRent.c
View File

@ -300,7 +300,3 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName,
return TSDB_CODE_SUCCESS;
}

2
source/libs/command/src/command.c
View File

@ -886,7 +886,7 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) {
for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) {
SConfigItem* pItem = taosArrayGet(tsCfg->array, i);
GRANT_CFG_SKIP;
// GRANT_CFG_SKIP;
char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE);

5
source/libs/executor/inc/executorInt.h
View File

@ -279,7 +279,6 @@ typedef struct STableScanInfo {
int8_t assignBlockUid;
uint8_t countState; // empty table count state
bool hasGroupByTag;
bool countOnly;
bool filesetDelimited;
bool needCountEmptyTable;
} STableScanInfo;
@ -757,10 +756,12 @@ extern void doDestroyExchangeOperatorInfo(void* param);
int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, const char* idStr, STableMetaCacheInfo* pCache);
int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name);
void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId);
void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset);
void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput);

4
source/libs/executor/src/cachescanoperator.c
View File

@ -250,7 +250,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pInfo->pseudoExprSup;
int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes,
pRes->info.rows, GET_TASKID(pTaskInfo), NULL);
pRes->info.rows, pTaskInfo, NULL);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
return NULL;
@ -313,7 +313,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
if (taosArrayGetSize(pInfo->pUidList) > 0) {
pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0);
code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes,
pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), NULL);
pInfo->pRes->info.rows, pTaskInfo, NULL);
if (code != TSDB_CODE_SUCCESS) {
pTaskInfo->code = code;
return NULL;

5
source/libs/executor/src/eventwindowoperator.c
View File

@ -220,7 +220,6 @@ static int32_t setSingleOutputTupleBufv1(SResultRowInfo* pResultRowInfo, STimeWi
(*pResult)->win = *win;
clearResultRowInitFlag(pExprSup->pCtx, pExprSup->numOfExprs);
setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset);
return TSDB_CODE_SUCCESS;
}
@ -262,6 +261,7 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p
} else if (pInfo->groupId != gid) {
// this is a new group, reset the info
pInfo->inWindow = false;
pInfo->groupId = gid;
}
SFilterColumnParam param1 = {.numOfCols = taosArrayGetSize(pBlock->pDataBlock), .pDataBlock = pBlock->pDataBlock};
@ -319,6 +319,9 @@ int32_t eventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* p
doKeepNewWindowStartInfo(pRowSup, tsList, rowIndex, gid);
pInfo->inWindow = true;
startIndex = rowIndex;
if (pInfo->pRow != NULL) {
clearResultRowInitFlag(pSup->pCtx, pSup->numOfExprs);
}
break;
}
}

2
source/libs/executor/src/executorInt.c
View File

@ -311,6 +311,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
pInput->totalRows = pBlock->info.rows;
pInput->numOfRows = pBlock->info.rows;
pInput->startRowIndex = 0;
pInput->blankFill = pBlock->info.blankFill;
// NOTE: the last parameter is the primary timestamp column
// todo: refactor this
@ -325,6 +326,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
pInput->totalRows = pBlock->info.rows;
pInput->numOfRows = pBlock->info.rows;
pInput->startRowIndex = 0;
pInput->blankFill = pBlock->info.blankFill;
code = doCreateConstantValColumnInfo(pInput, pFuncParam, j, pBlock->info.rows);
if (code != TSDB_CODE_SUCCESS) {

15
source/libs/executor/src/groupoperator.c
View File

@ -287,7 +287,6 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
terrno = TSDB_CODE_SUCCESS;
int32_t num = 0;
uint64_t groupId = 0;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
// Compare with the previous row of this column, and do not set the output buffer again if they are identical.
if (!pInfo->isInit) {
@ -478,20 +477,6 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
pOperator->status = OP_RES_TO_RETURN;
#if 0
if(pOperator->fpSet.encodeResultRow){
char *result = NULL;
int32_t length = 0;
pOperator->fpSet.encodeResultRow(pOperator, &result, &length);
SAggSupporter* pSup = &pInfo->aggSup;
taosHashClear(pSup->pResultRowHashTable);
pInfo->binfo.resultRowInfo.size = 0;
pOperator->fpSet.decodeResultRow(pOperator, result);
if(result){
taosMemoryFree(result);
}
}
#endif
// initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0);
if (pGroupResInfo->pRows != NULL) {
taosArrayDestroy(pGroupResInfo->pRows);

89
source/libs/executor/src/scanoperator.c
View File

@ -21,7 +21,6 @@
#include "querynodes.h"
#include "systable.h"
#include "tname.h"
#include "ttime.h"
#include "tdatablock.h"
#include "tmsg.h"
@ -252,7 +251,7 @@ static void doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* pBlo
SExprSupp* pSup = &pTableScanInfo->pseudoSup;
int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, rows,
GET_TASKID(pTaskInfo), &pTableScanInfo->metaCache);
pTaskInfo, &pTableScanInfo->metaCache);
// ignore the table not exists error, since this table may have been dropped during the scan procedure.
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_PAR_TABLE_NOT_EXIST) {
T_LONG_JMP(pTaskInfo->env, code);
@ -295,8 +294,8 @@ bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo
static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
uint32_t* status) {
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SStorageAPI* pAPI = &pTaskInfo->storageAPI;
SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
@ -482,24 +481,25 @@ static void doSetNullValue(SSDataBlock* pBlock, const SExprInfo* pExpr, int32_t
}
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, const char* idStr, STableMetaCacheInfo* pCache) {
int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache) {
int32_t code = 0;
bool freeReader = false;
LRUHandle* h = NULL;
STableCachedVal val = {0};
SMetaReader mr = {0};
const char* idStr = pTask->id.str;
// currently only the tbname pseudo column
if (numOfExpr <= 0) {
return TSDB_CODE_SUCCESS;
}
int32_t code = 0;
bool freeReader = false;
// todo: opt if only require the vgId and the vgVer;
// backup the rows
int32_t backupRows = pBlock->info.rows;
pBlock->info.rows = rows;
STableCachedVal val = {0};
SMetaReader mr = {0};
LRUHandle* h = NULL;
// todo refactor: extract method
// the handling of the null data should be packed in the extracted method
@ -586,7 +586,14 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
// this is to handle the tbname
if (fmIsScanPseudoColumnFunc(functionId)) {
setTbNameColData(pBlock, pColInfoData, functionId, val.pName);
int32_t fType = pExpr1->pExpr->_function.functionType;
if (fType == FUNCTION_TYPE_TBNAME) {
setTbNameColData(pBlock, pColInfoData, functionId, val.pName);
} else if (fType == FUNCTION_TYPE_VGID) {
setVgIdColData(pBlock, pColInfoData, functionId, pTask->id.vgId);
} else if (fType == FUNCTION_TYPE_VGVER) {
setVgVerColData(pBlock, pColInfoData, functionId, pBlock->info.version);
}
} else { // these are tags
STagVal tagVal = {0};
tagVal.cid = pExpr1->base.pParam[0].pCol->colId;
@ -655,6 +662,47 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData,
colDataDestroy(&infoData);
}
void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId) {
struct SScalarFuncExecFuncs fpSet = {0};
fmGetScalarFuncExecFuncs(functionId, &fpSet);
SColumnInfoData infoData = createColumnInfoData(pColInfoData->info.type, pColInfoData->info.bytes, 1);
colInfoDataEnsureCapacity(&infoData, 1, false);
colDataSetVal(&infoData, 0, (const char*)&vgId, false);
SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .columnData = &infoData};
SScalarParam param = {.columnData = pColInfoData};
if (fpSet.process != NULL) {
fpSet.process(&srcParam, 1, &param);
} else {
qError("failed to get the corresponding callback function, functionId:%d", functionId);
}
colDataDestroy(&infoData);
}
void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer) {
struct SScalarFuncExecFuncs fpSet = {0};
fmGetScalarFuncExecFuncs(functionId, &fpSet);
SColumnInfoData infoData = createColumnInfoData(pColInfoData->info.type, pColInfoData->info.bytes, 1);
colInfoDataEnsureCapacity(&infoData, 1, false);
colDataSetVal(&infoData, 0, (const char*)&vgVer, false);
SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .columnData = &infoData};
SScalarParam param = {.columnData = pColInfoData};
if (fpSet.process != NULL) {
fpSet.process(&srcParam, 1, &param);
} else {
qError("failed to get the corresponding callback function, functionId:%d", functionId);
}
colDataDestroy(&infoData);
}
static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) {
tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, pKeyInfo, size);
@ -663,6 +711,8 @@ static void initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, i
pInfo->tableEndIndex = (pInfo->tableStartIndex + (*size) - 1);
pInfo->pResBlock->info.blankFill = false;
if (!pInfo->needCountEmptyTable) {
pInfo->countState = TABLE_COUNT_STATE_END;
} else {
@ -687,6 +737,7 @@ static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBas
pBlock->info.rows = 1;
pBlock->info.id.uid = tbInfo->uid;
pBlock->info.id.groupId = tbInfo->groupId;
pBlock->info.blankFill = true;
// only one row: set all col data to null & hasNull
int32_t col_num = blockDataGetNumOfCols(pBlock);
@ -696,7 +747,7 @@ static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBas
}
// set tag/tbname
doSetTagColumnData(pBase, pBlock, pTaskInfo, pBlock->info.rows);
doSetTagColumnData(pBase, pBlock, pTaskInfo, 1);
return pBlock;
}
@ -963,13 +1014,11 @@ static SSDataBlock* groupSeqTableScan(SOperatorInfo* pOperator) {
STableKeyInfo* pList = NULL;
if (pInfo->currentGroupId == -1) {
int32_t numOfTables = tableListGetSize(pInfo->base.pTableListInfo);
if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
setOperatorCompleted(pOperator);
return NULL;
}
initNextGroupScan(pInfo, &pList, &num);
ASSERT(pInfo->base.dataReader == NULL);
@ -1172,10 +1221,6 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
goto _error;
}
if (scanDebug) {
pInfo->countOnly = true;
}
pInfo->filesetDelimited = pTableScanNode->filesetDelimited;
taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false);
@ -1901,7 +1946,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
// currently only the tbname pseudo column
if (pInfo->numOfPseudoExpr > 0) {
int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes,
pBlockInfo->rows, id, &pTableScanInfo->base.metaCache);
pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache);
// ignore the table not exists error, since this table may have been dropped during the scan procedure.
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_PAR_TABLE_NOT_EXIST) {
blockDataFreeRes((SSDataBlock*)pBlock);
@ -2134,7 +2179,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
pTSInfo->base.cond.endVersion = pStreamInfo->fillHistoryVer.maxVer;
pTSInfo->base.cond.twindows = pStreamInfo->fillHistoryWindow;
qDebug("stream recover step1, verRange:%" PRId64 "-%" PRId64 " window:%"PRId64"-%"PRId64", %s", pTSInfo->base.cond.startVersion,
qDebug("stream scan step1, verRange:%" PRId64 "-%" PRId64 " window:%"PRId64"-%"PRId64", %s", pTSInfo->base.cond.startVersion,
pTSInfo->base.cond.endVersion, pTSInfo->base.cond.twindows.skey, pTSInfo->base.cond.twindows.ekey, id);
pStreamInfo->recoverStep = STREAM_RECOVER_STEP__SCAN1;
pStreamInfo->recoverScanFinished = false;

2
source/libs/executor/src/streameventwindowoperator.c
View File

@ -349,6 +349,7 @@ static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
curWin.winInfo.pStatePos->beUpdated = true;
SSessionKey key = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &key);
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));
@ -725,7 +726,6 @@ SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhys
}
if (pInfo->isHistoryOp) {
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pAllUpdated = tSimpleHashInit(64, hashFn);
} else {
pInfo->pAllUpdated = NULL;

6
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -2083,6 +2083,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData
}
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
winInfo.pStatePos->beUpdated = true;
SSessionKey key = {0};
getSessionHashKey(&winInfo.sessionWin, &key);
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo));
@ -2286,6 +2287,10 @@ int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated) {
int32_t iter = 0;
while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) {
SResultWindowInfo* pWinInfo = pIte;
if (!pWinInfo->pStatePos->beUpdated) {
continue;
}
pWinInfo->pStatePos->beUpdated = false;
saveResult(*pWinInfo, pStUpdated);
}
return TSDB_CODE_SUCCESS;
@ -3425,6 +3430,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl
}
if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) {
curWin.winInfo.pStatePos->beUpdated = true;
SSessionKey key = {0};
getSessionHashKey(&curWin.winInfo.sessionWin, &key);
tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo));

1
source/libs/executor/src/tsort.c
View File

@ -1334,7 +1334,6 @@ static bool tsortOpenForBufMergeSort(SSortHandle* pHandle) {
int32_t tsortClose(SSortHandle* pHandle) {
atomic_val_compare_exchange_8(&pHandle->closed, 0, 1);
taosMsleep(10);
return TSDB_CODE_SUCCESS;
}

34
source/libs/function/src/builtins.c
View File

@ -707,18 +707,20 @@ static int32_t translateTbnameColumn(SFunctionNode* pFunc, char* pErrBuf, int32_
static int32_t translateTbUidColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS;
}
static int32_t translateVgIdColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType =
(SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT};
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes, .type = TSDB_DATA_TYPE_INT};
return TSDB_CODE_SUCCESS;
}
static int32_t translateVgVerColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT};
return TSDB_CODE_SUCCESS;
}
static int32_t translateTopBot(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
@ -3453,7 +3455,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.translateFunc = translateTbnameColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = qTbnameFunction,
.sprocessFunc = qPseudoTagFunction,
.finalizeFunc = NULL
},
{
@ -3740,11 +3742,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.translateFunc = translateTbUidColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
#ifdef BUILD_NO_CALL
.sprocessFunc = qTbUidFunction,
#else
.sprocessFunc = NULL,
#endif
.sprocessFunc = qPseudoTagFunction,
.finalizeFunc = NULL
},
{
@ -3754,11 +3752,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.translateFunc = translateVgIdColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
#ifdef BUILD_NO_CALL
.sprocessFunc = qVgIdFunction,
#else
.sprocessFunc = NULL,
#endif
.sprocessFunc = qPseudoTagFunction,
.finalizeFunc = NULL
},
{
@ -3781,6 +3775,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = toCharFunction,
.finalizeFunc = NULL
},
{
.name = "_vgver",
.type = FUNCTION_TYPE_VGVER,
.classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC,
.translateFunc = translateVgVerColumn,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = qPseudoTagFunction,
.finalizeFunc = NULL
}
};
// clang-format on

5
source/libs/function/src/builtinsimpl.c
View File

@ -499,6 +499,9 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) {
*/
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pInputCol = pInput->pData[0];
if(1 == pInput->numOfRows && pInput->blankFill) {
return 0;
}
if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) {
numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull;
} else {
@ -6022,7 +6025,7 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) {
goto _group_key_over;
}
if (colDataIsNull_s(pInputCol, startIndex)) {
if (pInputCol->pData == NULL || colDataIsNull_s(pInputCol, startIndex)) {
pInfo->isNull = true;
pInfo->hasResult = true;
goto _group_key_over;

77
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -193,6 +193,8 @@ const char* nodesNodeName(ENodeType type) {
return "GrantStmt";
case QUERY_NODE_REVOKE_STMT:
return "RevokeStmt";
case QUERY_NODE_ALTER_CLUSTER_STMT:
return "AlterClusterStmt";
case QUERY_NODE_SHOW_DNODES_STMT:
return "ShowDnodesStmt";
case QUERY_NODE_SHOW_MNODES_STMT:
@ -262,7 +264,13 @@ const char* nodesNodeName(ENodeType type) {
case QUERY_NODE_SHOW_COMPACTS_STMT:
return "ShowCompactsStmt";
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
return "ShowCompactDetailsStmt";
return "ShowCompactDetailsStmt";
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
return "ShowGrantsFullStmt";
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
return "ShowGrantsLogsStmt";
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return "ShowClusterMachinesStmt";
case QUERY_NODE_DELETE_STMT:
return "DeleteStmt";
case QUERY_NODE_INSERT_STMT:
@ -3286,6 +3294,8 @@ static const char* jkSubplanTagIndexCond = "TagIndexCond";
static const char* jkSubplanShowRewrite = "ShowRewrite";
static const char* jkSubplanRowsThreshold = "RowThreshold";
static const char* jkSubplanDynamicRowsThreshold = "DyRowThreshold";
static const char* jkSubplanIsView = "IsView";
static const char* jkSubplanIsAudit = "IsAudit";
static int32_t subplanToJson(const void* pObj, SJson* pJson) {
const SSubplan* pNode = (const SSubplan*)pObj;
@ -3324,6 +3334,12 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSubplanIsView, pNode->isView);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkSubplanIsAudit, pNode->isAudit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold);
}
@ -3371,6 +3387,12 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSubplanIsView, &pNode->isView);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkSubplanIsAudit, &pNode->isAudit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold);
}
@ -6162,6 +6184,31 @@ static int32_t jsonToDropConsumerGroupStmt(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkAlterClusterStmtConfig = "Config";
static const char* jkAlterClusterStmtValue = "Value";
static int32_t alterClusterStmtToJson(const void* pObj, SJson* pJson) {
const SAlterClusterStmt* pNode = (const SAlterClusterStmt*)pObj;
int32_t code = tjsonAddStringToObject(pJson, jkAlterClusterStmtConfig, pNode->config);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkAlterClusterStmtValue, pNode->value);
}
return code;
}
static int32_t jsonToAlterClusterStmt(const SJson* pJson, void* pObj) {
SAlterClusterStmt* pNode = (SAlterClusterStmt*)pObj;
int32_t code = tjsonGetStringValue(pJson, jkAlterClusterStmtConfig, pNode->config);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkAlterClusterStmtValue, pNode->value);
}
return code;
}
static const char* jkAlterLocalStmtConfig = "Config";
static const char* jkAlterLocalStmtValue = "Value";
@ -6559,6 +6606,18 @@ static int32_t showVariablesStmtToJson(const void* pObj, SJson* pJson) { return
static int32_t jsonToShowVariablesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t showGrantsFullStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowGrantsFullStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t showGrantsLogsStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t showClusterMachinesStmtToJson(const void* pObj, SJson* pJson) { return showStmtToJson(pObj, pJson); }
static int32_t jsonToShowGrantsLogsStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static int32_t jsonToShowClusterMachinesStmt(const SJson* pJson, void* pObj) { return jsonToShowStmt(pJson, pObj); }
static const char* jkShowDnodeVariablesStmtDnodeId = "DnodeId";
static const char* jkShowDnodeVariablesStmtLikePattern = "LikePattern";
@ -7014,6 +7073,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return grantStmtToJson(pObj, pJson);
case QUERY_NODE_REVOKE_STMT:
return revokeStmtToJson(pObj, pJson);
case QUERY_NODE_ALTER_CLUSTER_STMT:
return alterClusterStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_DNODES_STMT:
return showDnodesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_MNODES_STMT:
@ -7044,6 +7105,12 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return showConsumersStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_VARIABLES_STMT:
return showVariablesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
return showGrantsFullStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
return showGrantsLogsStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return showClusterMachinesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT:
return showDnodeVariablesStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
@ -7337,6 +7404,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToGrantStmt(pJson, pObj);
case QUERY_NODE_REVOKE_STMT:
return jsonToRevokeStmt(pJson, pObj);
case QUERY_NODE_ALTER_CLUSTER_STMT:
return jsonToAlterClusterStmt(pJson, pObj);
case QUERY_NODE_SHOW_DNODES_STMT:
return jsonToShowDnodesStmt(pJson, pObj);
case QUERY_NODE_SHOW_MNODES_STMT:
@ -7367,6 +7436,12 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToShowConsumersStmt(pJson, pObj);
case QUERY_NODE_SHOW_VARIABLES_STMT:
return jsonToShowVariablesStmt(pJson, pObj);
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
return jsonToShowGrantsFullStmt(pJson, pObj);
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
return jsonToShowGrantsLogsStmt(pJson, pObj);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return jsonToShowClusterMachinesStmt(pJson, pObj);
case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT:
return jsonToShowDnodeVariablesStmt(pJson, pObj);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT:

13
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -3930,6 +3930,12 @@ static int32_t subplanInlineToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueBool(pEncoder, pNode->dynamicRowThreshold);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueBool(pEncoder, pNode->isView);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeValueBool(pEncoder, pNode->isAudit);
}
return code;
}
@ -3985,7 +3991,12 @@ static int32_t msgToSubplanInline(STlvDecoder* pDecoder, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueBool(pDecoder, &pNode->dynamicRowThreshold);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueBool(pDecoder, &pNode->isView);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvDecodeValueBool(pDecoder, &pNode->isAudit);
}
return code;
}

16
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -31,7 +31,7 @@ typedef struct SNodeMemChunk {
struct SNodeMemChunk* pNext;
} SNodeMemChunk;
typedef struct SNodeAllocator {
struct SNodeAllocator {
int64_t self;
int64_t queryId;
int32_t chunkSize;
@ -39,7 +39,7 @@ typedef struct SNodeAllocator {
SNodeMemChunk* pCurrChunk;
SNodeMemChunk* pChunks;
TdThreadMutex mutex;
} SNodeAllocator;
};
static threadlocal SNodeAllocator* g_pNodeAllocator;
static int32_t g_allocatorReqRefPool = -1;
@ -409,6 +409,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SGrantStmt));
case QUERY_NODE_REVOKE_STMT:
return makeNode(type, sizeof(SRevokeStmt));
case QUERY_NODE_ALTER_CLUSTER_STMT:
return makeNode(type, sizeof(SAlterClusterStmt));
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
@ -439,6 +441,9 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
case QUERY_NODE_SHOW_VIEWS_STMT:
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return makeNode(type, sizeof(SShowTableTagsStmt));
@ -1046,6 +1051,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_REVOKE_STMT:
nodesDestroyNode(((SRevokeStmt*)pNode)->pTagCond);
break;
case QUERY_NODE_ALTER_CLUSTER_STMT: // no pointer field
break;
case QUERY_NODE_SHOW_DNODES_STMT:
case QUERY_NODE_SHOW_MNODES_STMT:
case QUERY_NODE_SHOW_MODULES_STMT:
@ -1075,7 +1082,10 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
case QUERY_NODE_SHOW_VIEWS_STMT: {
case QUERY_NODE_SHOW_VIEWS_STMT:
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName);
nodesDestroyNode(pStmt->pTbName);

1
source/libs/parser/inc/parAst.h
View File

@ -225,6 +225,7 @@ SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists,
int8_t withMeta, SNode* pWhere);
SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pTopicName);
SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pCGroupId, SToken* pTopicName);
SNode* createAlterClusterStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue);
SNode* createDefaultExplainOptions(SAstCreateContext* pCxt);
SNode* setExplainVerbose(SAstCreateContext* pCxt, SNode* pOptions, const SToken* pVal);

4
source/libs/parser/inc/parInt.h
View File

@ -35,8 +35,8 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache*
int32_t translate(SParseContext* pParseCxt, SQuery* pQuery, SParseMetaCache* pMetaCache);
int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema);
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placeholderNo, SArray** pPlaceholderValues);
int32_t translateTable(STranslateContext* pCxt, SNode** pTable);
int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput);

7
source/libs/parser/inc/sql.y
View File

@ -172,6 +172,10 @@ force_opt(A) ::= FORCE.
%destructor unsafe_opt { }
unsafe_opt(A) ::= UNSAFE. { A = true; }
/************************************************ alter cluster *********************************************************/
cmd ::= ALTER CLUSTER NK_STRING(A). { pCxt->pRootNode = createAlterClusterStmt(pCxt, &A, NULL); }
cmd ::= ALTER CLUSTER NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterClusterStmt(pCxt, &A, &B); }
/************************************************ alter local *********************************************************/
cmd ::= ALTER LOCAL NK_STRING(A). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, NULL); }
cmd ::= ALTER LOCAL NK_STRING(A) NK_STRING(B). { pCxt->pRootNode = createAlterLocalStmt(pCxt, &A, &B); }
@ -483,6 +487,9 @@ cmd ::= SHOW APPS.
cmd ::= SHOW CONNECTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); }
cmd ::= SHOW LICENCES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); }
cmd ::= SHOW GRANTS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); }
cmd ::= SHOW GRANTS FULL. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_GRANTS_FULL_STMT); }
cmd ::= SHOW GRANTS LOGS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_GRANTS_LOGS_STMT); }
cmd ::= SHOW CLUSTER MACHINES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT); }
cmd ::= SHOW CREATE DATABASE db_name(A). { pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &A); }
cmd ::= SHOW CREATE TABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, A); }
cmd ::= SHOW CREATE STABLE full_table_name(A). { pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, A); }

11
source/libs/parser/src/parAstCreater.c
View File

@ -2187,6 +2187,17 @@ SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToke
return (SNode*)pStmt;
}
SNode* createAlterClusterStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) {
CHECK_PARSER_STATUS(pCxt);
SAlterClusterStmt* pStmt = (SAlterClusterStmt*)nodesMakeNode(QUERY_NODE_ALTER_CLUSTER_STMT);
CHECK_OUT_OF_MEM(pStmt);
trimString(pConfig->z, pConfig->n, pStmt->config, sizeof(pStmt->config));
if (NULL != pValue) {
trimString(pValue->z, pValue->n, pStmt->value, sizeof(pStmt->value));
}
return (SNode*)pStmt;
}
SNode* createAlterLocalStmt(SAstCreateContext* pCxt, const SToken* pConfig, const SToken* pValue) {
CHECK_PARSER_STATUS(pCxt);
SAlterLocalStmt* pStmt = (SAlterLocalStmt*)nodesMakeNode(QUERY_NODE_ALTER_LOCAL_STMT);

21
source/libs/parser/src/parAstParser.c
View File

@ -619,6 +619,21 @@ static int32_t collectMetaKeyFromShowCompactDetails(SCollectMetaKeyCxt* pCxt, SS
return code;
}
static int32_t collectMetaKeyFromShowGrantsFull(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_GRANTS_FULL,
pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromShowGrantsLogs(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_GRANTS_LOGS,
pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromShowClusterMachines(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MACHINES,
pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) {
return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
}
@ -839,6 +854,12 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowCompacts(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_COMPACT_DETAILS_STMT:
return collectMetaKeyFromShowCompactDetails(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
return collectMetaKeyFromShowGrantsFull(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
return collectMetaKeyFromShowGrantsLogs(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return collectMetaKeyFromShowClusterMachines(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_TABLE_STMT:

3
source/libs/parser/src/parAuthenticator.c
View File

@ -349,6 +349,9 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
case QUERY_NODE_SHOW_VNODES_STMT:
case QUERY_NODE_SHOW_SCORES_STMT:
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
return !pCxt->pParseCxt->enableSysInfo ? TSDB_CODE_PAR_PERMISSION_DENIED : TSDB_CODE_SUCCESS;
case QUERY_NODE_SHOW_TABLES_STMT:
case QUERY_NODE_SHOW_STABLES_STMT:

4
source/libs/parser/src/parInsertSql.c
View File

@ -2144,13 +2144,15 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt
pStmt->pTableCxtHashObj =
taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
}
int32_t numOfRows = 0;
int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows);
if (TSDB_CODE_SUCCESS == code) {
pStmt->totalRowsNum += numOfRows;
pStmt->totalTbNum += 1;
TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT);
if (rowsDataCxt.pTableDataCxt && rowsDataCxt.pTableDataCxt->pData) {
rowsDataCxt.pTableDataCxt->pData->flags |= SUBMIT_REQ_FROM_FILE;
}
if (!pStmt->fileProcessing) {
taosCloseFile(&pStmt->fp);
} else {

3
source/libs/parser/src/parTokenizer.c
View File

@ -109,6 +109,9 @@ static SKeyword keywordTable[] = {
{"GEOMETRY", TK_GEOMETRY},
{"GRANT", TK_GRANT},
{"GRANTS", TK_GRANTS},
{"FULL", TK_FULL},
{"LOGS", TK_LOGS},
{"MACHINES", TK_MACHINES},
{"GROUP", TK_GROUP},
{"HAVING", TK_HAVING},
{"HOST", TK_HOST},

Some files were not shown because too many files have changed in this diff Show More