xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TD-21898

This commit is contained in:
liuyao 2023-11-01 16:50:46 +08:00
parent b98d79add5 f76417535c
commit 505c44d9aa
156 changed files with 13027 additions and 8307 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile2
View File

@ -424,7 +424,7 @@ pipeline {
echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}" echo "${WKDIR}/restore.sh -p ${BRANCH_NAME} -n ${BUILD_ID} -c {container name}"
} }
catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
timeout(time: 130, unit: 'MINUTES'){ timeout(time: 150, unit: 'MINUTES'){
pre_test() pre_test()
script { script {
sh ''' sh '''

1
include/common/systable.h
View File

@ -49,6 +49,7 @@ extern "C" {
#define TSDB_INS_TABLE_STREAMS "ins_streams" #define TSDB_INS_TABLE_STREAMS "ins_streams"
#define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks" #define TSDB_INS_TABLE_STREAM_TASKS "ins_stream_tasks"
#define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges" #define TSDB_INS_TABLE_USER_PRIVILEGES "ins_user_privileges"
#define TSDB_INS_TABLE_VIEWS "ins_views"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas" #define TSDB_PERFS_TABLE_SMAS "perf_smas"

3
include/common/taosdef.h
View File

@ -42,7 +42,8 @@ typedef enum {
TSDB_TEMP_TABLE = 4, // temp table created by nest query TSDB_TEMP_TABLE = 4, // temp table created by nest query
TSDB_SYSTEM_TABLE = 5, TSDB_SYSTEM_TABLE = 5,
TSDB_TSMA_TABLE = 6, // time-range-wise sma TSDB_TSMA_TABLE = 6, // time-range-wise sma
TSDB_TABLE_MAX = 7 TSDB_VIEW_TABLE = 7,
TSDB_TABLE_MAX = 8
} ETableType; } ETableType;
typedef enum { typedef enum {

6
include/common/tglobal.h
View File

@ -195,7 +195,8 @@ extern int64_t tsWalFsyncDataSizeLimit;
// internal // internal
extern int32_t tsTransPullupInterval; extern int32_t tsTransPullupInterval;
extern int32_t tsMqRebalanceInterval; extern int32_t tsMqRebalanceInterval;
extern int32_t tsStreamCheckpointTickInterval; extern int32_t tsStreamCheckpointInterval;
extern float tsSinkDataRate;
extern int32_t tsStreamNodeCheckInterval; extern int32_t tsStreamNodeCheckInterval;
extern int32_t tsTtlUnit; extern int32_t tsTtlUnit;
extern int32_t tsTtlPushIntervalSec; extern int32_t tsTtlPushIntervalSec;
@ -204,9 +205,6 @@ extern int32_t tsTrimVDbIntervalSec;
extern int32_t tsGrantHBInterval; extern int32_t tsGrantHBInterval;
extern int32_t tsUptimeInterval; extern int32_t tsUptimeInterval;
extern int32_t tsRpcRetryLimit;
extern int32_t tsRpcRetryInterval;
extern bool tsDisableStream; extern bool tsDisableStream;
extern int64_t tsStreamBufferSize; extern int64_t tsStreamBufferSize;
extern bool tsFilterScalarMode; extern bool tsFilterScalarMode;

107
include/common/tmsg.h
View File

@ -107,6 +107,8 @@ enum {
HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_DBINFO,
HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_STBINFO,
HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_TMQ,
HEARTBEAT_KEY_DYN_VIEW,
HEARTBEAT_KEY_VIEWINFO,
}; };
typedef enum _mgmt_table { typedef enum _mgmt_table {
@ -141,6 +143,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_APPS, TSDB_MGMT_TABLE_APPS,
TSDB_MGMT_TABLE_STREAM_TASKS, TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES, TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_MAX, TSDB_MGMT_TABLE_MAX,
} EShowType; } EShowType;
@ -168,26 +171,12 @@ typedef enum _mgmt_table {
#define TSDB_ALTER_USER_PASSWD 0x1 #define TSDB_ALTER_USER_PASSWD 0x1
#define TSDB_ALTER_USER_SUPERUSER 0x2 #define TSDB_ALTER_USER_SUPERUSER 0x2
#define TSDB_ALTER_USER_ADD_READ_DB 0x3 #define TSDB_ALTER_USER_ENABLE 0x3
#define TSDB_ALTER_USER_REMOVE_READ_DB 0x4 #define TSDB_ALTER_USER_SYSINFO 0x4
#define TSDB_ALTER_USER_ADD_WRITE_DB 0x5 #define TSDB_ALTER_USER_ADD_PRIVILEGES 0x5
#define TSDB_ALTER_USER_REMOVE_WRITE_DB 0x6 #define TSDB_ALTER_USER_DEL_PRIVILEGES 0x6
#define TSDB_ALTER_USER_ADD_ALL_DB 0x7 #define TSDB_ALTER_USER_ADD_WHITE_LIST 0x7
#define TSDB_ALTER_USER_REMOVE_ALL_DB 0x8 #define TSDB_ALTER_USER_DROP_WHITE_LIST 0x8
#define TSDB_ALTER_USER_ENABLE 0x9
#define TSDB_ALTER_USER_SYSINFO 0xA
#define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC 0xB
#define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC
#define TSDB_ALTER_USER_ADD_READ_TABLE 0xD
#define TSDB_ALTER_USER_REMOVE_READ_TABLE 0xE
#define TSDB_ALTER_USER_ADD_WRITE_TABLE 0xF
#define TSDB_ALTER_USER_REMOVE_WRITE_TABLE 0x10
#define TSDB_ALTER_USER_ADD_ALL_TABLE 0x11
#define TSDB_ALTER_USER_REMOVE_ALL_TABLE 0x12
#define TSDB_ALTER_USER_ADD_WHITE_LIST 0x13
#define TSDB_ALTER_USER_DROP_WHITE_LIST 0x14
#define TSDB_ALTER_USER_PRIVILEGES 0x2
#define TSDB_KILL_MSG_LEN 30 #define TSDB_KILL_MSG_LEN 30
@ -251,6 +240,7 @@ typedef enum ENodeType {
QUERY_NODE_CASE_WHEN, QUERY_NODE_CASE_WHEN,
QUERY_NODE_EVENT_WINDOW, QUERY_NODE_EVENT_WINDOW,
QUERY_NODE_HINT, QUERY_NODE_HINT,
QUERY_NODE_VIEW,
// Statement nodes are used in parser and planner module. // Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100, QUERY_NODE_SET_OPERATOR = 100,
@ -333,6 +323,8 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT,
QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_VNODES_STMT,
QUERY_NODE_SHOW_USER_PRIVILEGES_STMT, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT,
QUERY_NODE_SHOW_VIEWS_STMT,
QUERY_NODE_SHOW_CREATE_VIEW_STMT,
QUERY_NODE_SHOW_CREATE_DATABASE_STMT, QUERY_NODE_SHOW_CREATE_DATABASE_STMT,
QUERY_NODE_SHOW_CREATE_TABLE_STMT, QUERY_NODE_SHOW_CREATE_TABLE_STMT,
QUERY_NODE_SHOW_CREATE_STABLE_STMT, QUERY_NODE_SHOW_CREATE_STABLE_STMT,
@ -354,7 +346,9 @@ typedef enum ENodeType {
QUERY_NODE_RESTORE_MNODE_STMT, QUERY_NODE_RESTORE_MNODE_STMT,
QUERY_NODE_RESTORE_VNODE_STMT, QUERY_NODE_RESTORE_VNODE_STMT,
QUERY_NODE_PAUSE_STREAM_STMT, QUERY_NODE_PAUSE_STREAM_STMT,
QUERY_NODE_RESUME_STREAM_STMT, QUERY_NODE_RESUME_STREAM_STMT,
QUERY_NODE_CREATE_VIEW_STMT,
QUERY_NODE_DROP_VIEW_STMT,
// logic plan node // logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000, QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -451,7 +445,7 @@ typedef struct SRetention {
int8_t keepUnit; int8_t keepUnit;
} SRetention; } SRetention;
#define RETENTION_VALID(r) (((r)->freq > 0) && ((r)->keep > 0)) #define RETENTION_VALID(l, r) ((((l) == 0 && (r)->freq >= 0) || ((r)->freq > 0)) && ((r)->keep > 0))
#pragma pack(push, 1) #pragma pack(push, 1)
@ -943,6 +937,7 @@ typedef struct {
int8_t superUser; int8_t superUser;
int8_t sysInfo; int8_t sysInfo;
int8_t enable; int8_t enable;
int8_t isView;
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char pass[TSDB_USET_PASSWORD_LEN]; char pass[TSDB_USET_PASSWORD_LEN];
char objname[TSDB_DB_FNAME_LEN]; // db or topic char objname[TSDB_DB_FNAME_LEN]; // db or topic
@ -951,6 +946,7 @@ typedef struct {
int32_t tagCondLen; int32_t tagCondLen;
int32_t numIpRanges; int32_t numIpRanges;
SIpV4Range* pIpRanges; SIpV4Range* pIpRanges;
int64_t privileges;
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
} SAlterUserReq; } SAlterUserReq;
@ -979,6 +975,10 @@ typedef struct {
SHashObj* writeDbs; SHashObj* writeDbs;
SHashObj* readTbs; SHashObj* readTbs;
SHashObj* writeTbs; SHashObj* writeTbs;
SHashObj* alterTbs;
SHashObj* readViews;
SHashObj* writeViews;
SHashObj* alterViews;
SHashObj* useDbs; SHashObj* useDbs;
int64_t whiteListVer; int64_t whiteListVer;
} SGetUserAuthRsp; } SGetUserAuthRsp;
@ -1568,9 +1568,7 @@ typedef struct {
typedef struct { typedef struct {
int32_t id; int32_t id;
int8_t isMnode; int8_t isMnode;
#ifdef TD_GRANT_HB_OPTIMIZE
int8_t offlineReason; int8_t offlineReason;
#endif
SEp ep; SEp ep;
char active[TSDB_ACTIVE_KEY_LEN]; char active[TSDB_ACTIVE_KEY_LEN];
char connActive[TSDB_CONN_ACTIVE_KEY_LEN]; char connActive[TSDB_CONN_ACTIVE_KEY_LEN];
@ -1811,6 +1809,15 @@ int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp);
int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp);
void tFreeSSTbHbRsp(SSTbHbRsp* pRsp); void tFreeSSTbHbRsp(SSTbHbRsp* pRsp);
typedef struct {
SArray* pViewRsp; // Array of SViewMetaRsp*;
} SViewHbRsp;
int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp);
int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp);
void tFreeSViewHbRsp(SViewHbRsp* pRsp);
typedef struct { typedef struct {
int32_t numOfTables; int32_t numOfTables;
int32_t numOfVgroup; int32_t numOfVgroup;
@ -3883,6 +3890,58 @@ typedef struct {
}; };
} SPackedData; } SPackedData;
typedef struct {
char fullname[TSDB_VIEW_FNAME_LEN];
char name[TSDB_VIEW_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
char* querySql;
char* sql;
int8_t orReplace;
int8_t precision;
int32_t numOfCols;
SSchema* pSchema;
} SCMCreateViewReq;
int32_t tSerializeSCMCreateViewReq(void* buf, int32_t bufLen, const SCMCreateViewReq* pReq);
int32_t tDeserializeSCMCreateViewReq(void* buf, int32_t bufLen, SCMCreateViewReq* pReq);
void tFreeSCMCreateViewReq(SCMCreateViewReq* pReq);
typedef struct {
char fullname[TSDB_VIEW_FNAME_LEN];
char name[TSDB_VIEW_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
char* sql;
int8_t igNotExists;
} SCMDropViewReq;
int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq* pReq);
int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pReq);
void tFreeSCMDropViewReq(SCMDropViewReq* pReq);
typedef struct {
char fullname[TSDB_VIEW_FNAME_LEN];
} SViewMetaReq;
int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq);
int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq);
typedef struct {
char name[TSDB_VIEW_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
char* user;
uint64_t dbId;
uint64_t viewId;
char* querySql;
int8_t precision;
int8_t type;
int32_t version;
int32_t numOfCols;
SSchema* pSchema;
} SViewMetaRsp;
int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp);
int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
void tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
#pragma pack(pop) #pragma pack(pop)
#ifdef __cplusplus #ifdef __cplusplus

3
include/common/tmsgdef.h
View File

@ -189,6 +189,9 @@ enum { // WARN: new msg should be appended to segment tail
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_VIEW, "create-view", SCMCreateViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_VIEW, "drop-view", SCMDropViewReq, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_VIEW_META, "view-meta", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_NEW_MSG_SEG(TDMT_VND_MSG)

351
include/common/ttokendef.h
View File

@ -187,180 +187,183 @@
#define TK_SUBSCRIPTIONS 168 #define TK_SUBSCRIPTIONS 168
#define TK_VNODES 169 #define TK_VNODES 169
#define TK_ALIVE 170 #define TK_ALIVE 170
#define TK_NORMAL 171 #define TK_VIEWS 171
#define TK_CHILD 172 #define TK_VIEW 172
#define TK_LIKE 173 #define TK_NORMAL 173
#define TK_TBNAME 174 #define TK_CHILD 174
#define TK_QTAGS 175 #define TK_LIKE 175
#define TK_AS 176 #define TK_TBNAME 176
#define TK_SYSTEM 177 #define TK_QTAGS 177
#define TK_INDEX 178 #define TK_AS 178
#define TK_FUNCTION 179 #define TK_SYSTEM 179
#define TK_INTERVAL 180 #define TK_INDEX 180
#define TK_COUNT 181 #define TK_FUNCTION 181
#define TK_LAST_ROW 182 #define TK_INTERVAL 182
#define TK_META 183 #define TK_COUNT 183
#define TK_ONLY 184 #define TK_LAST_ROW 184
#define TK_TOPIC 185 #define TK_META 185
#define TK_CONSUMER 186 #define TK_ONLY 186
#define TK_GROUP 187 #define TK_TOPIC 187
#define TK_DESC 188 #define TK_CONSUMER 188
#define TK_DESCRIBE 189 #define TK_GROUP 189
#define TK_RESET 190 #define TK_DESC 190
#define TK_QUERY 191 #define TK_DESCRIBE 191
#define TK_CACHE 192 #define TK_RESET 192
#define TK_EXPLAIN 193 #define TK_QUERY 193
#define TK_ANALYZE 194 #define TK_CACHE 194
#define TK_VERBOSE 195 #define TK_EXPLAIN 195
#define TK_NK_BOOL 196 #define TK_ANALYZE 196
#define TK_RATIO 197 #define TK_VERBOSE 197
#define TK_NK_FLOAT 198 #define TK_NK_BOOL 198
#define TK_OUTPUTTYPE 199 #define TK_RATIO 199
#define TK_AGGREGATE 200 #define TK_NK_FLOAT 200
#define TK_BUFSIZE 201 #define TK_OUTPUTTYPE 201
#define TK_LANGUAGE 202 #define TK_AGGREGATE 202
#define TK_REPLACE 203 #define TK_BUFSIZE 203
#define TK_STREAM 204 #define TK_LANGUAGE 204
#define TK_INTO 205 #define TK_REPLACE 205
#define TK_PAUSE 206 #define TK_STREAM 206
#define TK_RESUME 207 #define TK_INTO 207
#define TK_TRIGGER 208 #define TK_PAUSE 208
#define TK_AT_ONCE 209 #define TK_RESUME 209
#define TK_WINDOW_CLOSE 210 #define TK_TRIGGER 210
#define TK_IGNORE 211 #define TK_AT_ONCE 211
#define TK_EXPIRED 212 #define TK_WINDOW_CLOSE 212
#define TK_FILL_HISTORY 213 #define TK_IGNORE 213
#define TK_UPDATE 214 #define TK_EXPIRED 214
#define TK_SUBTABLE 215 #define TK_FILL_HISTORY 215
#define TK_UNTREATED 216 #define TK_UPDATE 216
#define TK_KILL 217 #define TK_SUBTABLE 217
#define TK_CONNECTION 218 #define TK_UNTREATED 218
#define TK_TRANSACTION 219 #define TK_KILL 219
#define TK_BALANCE 220 #define TK_CONNECTION 220
#define TK_VGROUP 221 #define TK_TRANSACTION 221
#define TK_LEADER 222 #define TK_BALANCE 222
#define TK_MERGE 223 #define TK_VGROUP 223
#define TK_REDISTRIBUTE 224 #define TK_LEADER 224
#define TK_SPLIT 225 #define TK_MERGE 225
#define TK_DELETE 226 #define TK_REDISTRIBUTE 226
#define TK_INSERT 227 #define TK_SPLIT 227
#define TK_NULL 228 #define TK_DELETE 228
#define TK_NK_QUESTION 229 #define TK_INSERT 229
#define TK_NK_ALIAS 230 #define TK_NULL 230
#define TK_NK_ARROW 231 #define TK_NK_QUESTION 231
#define TK_ROWTS 232 #define TK_NK_ALIAS 232
#define TK_QSTART 233 #define TK_NK_ARROW 233
#define TK_QEND 234 #define TK_ROWTS 234
#define TK_QDURATION 235 #define TK_QSTART 235
#define TK_WSTART 236 #define TK_QEND 236
#define TK_WEND 237 #define TK_QDURATION 237
#define TK_WDURATION 238 #define TK_WSTART 238
#define TK_IROWTS 239 #define TK_WEND 239
#define TK_ISFILLED 240 #define TK_WDURATION 240
#define TK_CAST 241 #define TK_IROWTS 241
#define TK_NOW 242 #define TK_ISFILLED 242
#define TK_TODAY 243 #define TK_CAST 243
#define TK_TIMEZONE 244 #define TK_NOW 244
#define TK_CLIENT_VERSION 245 #define TK_TODAY 245
#define TK_SERVER_VERSION 246 #define TK_TIMEZONE 246
#define TK_SERVER_STATUS 247 #define TK_CLIENT_VERSION 247
#define TK_CURRENT_USER 248 #define TK_SERVER_VERSION 248
#define TK_CASE 249 #define TK_SERVER_STATUS 249
#define TK_WHEN 250 #define TK_CURRENT_USER 250
#define TK_THEN 251 #define TK_CASE 251
#define TK_ELSE 252 #define TK_WHEN 252
#define TK_BETWEEN 253 #define TK_THEN 253
#define TK_IS 254 #define TK_ELSE 254
#define TK_NK_LT 255 #define TK_BETWEEN 255
#define TK_NK_GT 256 #define TK_IS 256
#define TK_NK_LE 257 #define TK_NK_LT 257
#define TK_NK_GE 258 #define TK_NK_GT 258
#define TK_NK_NE 259 #define TK_NK_LE 259
#define TK_MATCH 260 #define TK_NK_GE 260
#define TK_NMATCH 261 #define TK_NK_NE 261
#define TK_CONTAINS 262 #define TK_MATCH 262
#define TK_IN 263 #define TK_NMATCH 263
#define TK_JOIN 264 #define TK_CONTAINS 264
#define TK_INNER 265 #define TK_IN 265
#define TK_SELECT 266 #define TK_JOIN 266
#define TK_NK_HINT 267 #define TK_INNER 267
#define TK_DISTINCT 268 #define TK_SELECT 268
#define TK_WHERE 269 #define TK_NK_HINT 269
#define TK_PARTITION 270 #define TK_DISTINCT 270
#define TK_BY 271 #define TK_WHERE 271
#define TK_SESSION 272 #define TK_PARTITION 272
#define TK_STATE_WINDOW 273 #define TK_BY 273
#define TK_EVENT_WINDOW 274 #define TK_SESSION 274
#define TK_SLIDING 275 #define TK_STATE_WINDOW 275
#define TK_FILL 276 #define TK_EVENT_WINDOW 276
#define TK_VALUE 277 #define TK_SLIDING 277
#define TK_VALUE_F 278 #define TK_FILL 278
#define TK_NONE 279 #define TK_VALUE 279
#define TK_PREV 280 #define TK_VALUE_F 280
#define TK_NULL_F 281 #define TK_NONE 281
#define TK_LINEAR 282 #define TK_PREV 282
#define TK_NEXT 283 #define TK_NULL_F 283
#define TK_HAVING 284 #define TK_LINEAR 284
#define TK_RANGE 285 #define TK_NEXT 285
#define TK_EVERY 286 #define TK_HAVING 286
#define TK_ORDER 287 #define TK_RANGE 287
#define TK_SLIMIT 288 #define TK_EVERY 288
#define TK_SOFFSET 289 #define TK_ORDER 289
#define TK_LIMIT 290 #define TK_SLIMIT 290
#define TK_OFFSET 291 #define TK_SOFFSET 291
#define TK_ASC 292 #define TK_LIMIT 292
#define TK_NULLS 293 #define TK_OFFSET 293
#define TK_ABORT 294 #define TK_ASC 294
#define TK_AFTER 295 #define TK_NULLS 295
#define TK_ATTACH 296 #define TK_ABORT 296
#define TK_BEFORE 297 #define TK_AFTER 297
#define TK_BEGIN 298 #define TK_ATTACH 298
#define TK_BITAND 299 #define TK_BEFORE 299
#define TK_BITNOT 300 #define TK_BEGIN 300
#define TK_BITOR 301 #define TK_BITAND 301
#define TK_BLOCKS 302 #define TK_BITNOT 302
#define TK_CHANGE 303 #define TK_BITOR 303
#define TK_COMMA 304 #define TK_BLOCKS 304
#define TK_CONCAT 305 #define TK_CHANGE 305
#define TK_CONFLICT 306 #define TK_COMMA 306
#define TK_COPY 307 #define TK_CONCAT 307
#define TK_DEFERRED 308 #define TK_CONFLICT 308
#define TK_DELIMITERS 309 #define TK_COPY 309
#define TK_DETACH 310 #define TK_DEFERRED 310
#define TK_DIVIDE 311 #define TK_DELIMITERS 311
#define TK_DOT 312 #define TK_DETACH 312
#define TK_EACH 313 #define TK_DIVIDE 313
#define TK_FAIL 314 #define TK_DOT 314
#define TK_FILE 315 #define TK_EACH 315
#define TK_FOR 316 #define TK_FAIL 316
#define TK_GLOB 317 #define TK_FILE 317
#define TK_ID 318 #define TK_FOR 318
#define TK_IMMEDIATE 319 #define TK_GLOB 319
#define TK_IMPORT 320 #define TK_ID 320
#define TK_INITIALLY 321 #define TK_IMMEDIATE 321
#define TK_INSTEAD 322 #define TK_IMPORT 322
#define TK_ISNULL 323 #define TK_INITIALLY 323
#define TK_KEY 324 #define TK_INSTEAD 324
#define TK_MODULES 325 #define TK_ISNULL 325
#define TK_NK_BITNOT 326 #define TK_KEY 326
#define TK_NK_SEMI 327 #define TK_MODULES 327
#define TK_NOTNULL 328 #define TK_NK_BITNOT 328
#define TK_OF 329 #define TK_NK_SEMI 329
#define TK_PLUS 330 #define TK_NOTNULL 330
#define TK_PRIVILEGE 331 #define TK_OF 331
#define TK_RAISE 332 #define TK_PLUS 332
#define TK_RESTRICT 333 #define TK_PRIVILEGE 333
#define TK_ROW 334 #define TK_RAISE 334
#define TK_SEMI 335 #define TK_RESTRICT 335
#define TK_STAR 336 #define TK_ROW 336
#define TK_STATEMENT 337 #define TK_SEMI 337
#define TK_STRICT 338 #define TK_STAR 338
#define TK_STRING 339 #define TK_STATEMENT 339
#define TK_TIMES 340 #define TK_STRICT 340
#define TK_VALUES 341 #define TK_STRING 341
#define TK_VARIABLE 342 #define TK_TIMES 342
#define TK_VIEW 343 #define TK_VALUES 343
#define TK_WAL 344 #define TK_VARIABLE 344
#define TK_WAL 345

47
include/libs/catalog/catalog.h
View File

@ -37,13 +37,16 @@ enum {
CTG_DBG_DB_NUM = 1, CTG_DBG_DB_NUM = 1,
CTG_DBG_META_NUM, CTG_DBG_META_NUM,
CTG_DBG_STB_NUM, CTG_DBG_STB_NUM,
CTG_DBG_VIEW_NUM,
CTG_DBG_DB_RENT_NUM, CTG_DBG_DB_RENT_NUM,
CTG_DBG_STB_RENT_NUM, CTG_DBG_STB_RENT_NUM,
CTG_DBG_VIEW_RENT_NUM,
}; };
typedef enum { typedef enum {
AUTH_TYPE_READ = 1, AUTH_TYPE_READ = 1,
AUTH_TYPE_WRITE, AUTH_TYPE_WRITE,
AUTH_TYPE_ALTER,
AUTH_TYPE_OTHER, AUTH_TYPE_OTHER,
AUTH_TYPE_READ_OR_WRITE, AUTH_TYPE_READ_OR_WRITE,
} AUTH_TYPE; } AUTH_TYPE;
@ -51,12 +54,19 @@ typedef enum {
typedef struct SUserAuthInfo { typedef struct SUserAuthInfo {
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
SName tbName; SName tbName;
bool isView;
AUTH_TYPE type; AUTH_TYPE type;
} SUserAuthInfo; } SUserAuthInfo;
typedef enum {
AUTH_RES_BASIC = 0,
AUTH_RES_VIEW,
AUTH_RES_MAX_VALUE
} AUTH_RES_TYPE;
typedef struct SUserAuthRes { typedef struct SUserAuthRes {
bool pass; bool pass[AUTH_RES_MAX_VALUE];
SNode* pCond; SNode* pCond[AUTH_RES_MAX_VALUE];
} SUserAuthRes; } SUserAuthRes;
typedef struct SDbInfo { typedef struct SDbInfo {
@ -83,6 +93,7 @@ typedef struct SCatalogReq {
SArray* pTableIndex; // element is SNAME SArray* pTableIndex; // element is SNAME
SArray* pTableCfg; // element is SNAME SArray* pTableCfg; // element is SNAME
SArray* pTableTag; // element is SNAME SArray* pTableTag; // element is SNAME
SArray* pView; // element is STablesReq
bool qNodeRequired; // valid qnode bool qNodeRequired; // valid qnode
bool dNodeRequired; // valid dnode bool dNodeRequired; // valid dnode
bool svrVerRequired; bool svrVerRequired;
@ -96,6 +107,7 @@ typedef struct SMetaRes {
} SMetaRes; } SMetaRes;
typedef struct SMetaData { typedef struct SMetaData {
bool ctgFree; // need to freed by catalog module
SArray* pDbVgroup; // pRes = SArray<SVgroupInfo>* SArray* pDbVgroup; // pRes = SArray<SVgroupInfo>*
SArray* pDbCfg; // pRes = SDbCfgInfo* SArray* pDbCfg; // pRes = SDbCfgInfo*
SArray* pDbInfo; // pRes = SDbInfo* SArray* pDbInfo; // pRes = SDbInfo*
@ -109,21 +121,24 @@ typedef struct SMetaData {
SArray* pTableCfg; // pRes = STableCfg* SArray* pTableCfg; // pRes = STableCfg*
SArray* pTableTag; // pRes = SArray<STagVal>* SArray* pTableTag; // pRes = SArray<STagVal>*
SArray* pDnodeList; // pRes = SArray<SEpSet>* SArray* pDnodeList; // pRes = SArray<SEpSet>*
SArray* pView; // pRes = SViewMeta*
SMetaRes* pSvrVer; // pRes = char* SMetaRes* pSvrVer; // pRes = char*
} SMetaData; } SMetaData;
typedef struct SCatalogCfg { typedef struct SCatalogCfg {
uint32_t maxTblCacheNum; uint32_t maxTblCacheNum;
uint32_t maxViewCacheNum;
uint32_t maxDBCacheNum; uint32_t maxDBCacheNum;
uint32_t maxUserCacheNum; uint32_t maxUserCacheNum;
uint32_t dbRentSec; uint32_t dbRentSec;
uint32_t stbRentSec; uint32_t stbRentSec;
uint32_t viewRentSec;
} SCatalogCfg; } SCatalogCfg;
typedef struct SSTableVersion { typedef struct SSTableVersion {
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
char stbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN];
uint64_t dbId; int64_t dbId;
uint64_t suid; uint64_t suid;
int32_t sversion; int32_t sversion;
int32_t tversion; int32_t tversion;
@ -139,6 +154,20 @@ typedef struct SDbCacheInfo {
int64_t stateTs; int64_t stateTs;
} SDbCacheInfo; } SDbCacheInfo;
typedef struct SDynViewVersion {
int64_t svrBootTs;
uint64_t dynViewVer;
} SDynViewVersion;
typedef struct SViewVersion {
char dbFName[TSDB_DB_FNAME_LEN];
char viewName[TSDB_VIEW_NAME_LEN];
int64_t dbId;
uint64_t viewId;
int32_t version;
} SViewVersion;
typedef struct STbSVersion { typedef struct STbSVersion {
char* tbFName; char* tbFName;
int32_t sver; int32_t sver;
@ -307,6 +336,8 @@ int32_t catalogGetDnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray*
int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, uint32_t* num); int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, uint32_t* num);
int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion);
int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num);
int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num);
@ -343,6 +374,16 @@ SMetaData* catalogCloneMetaData(SMetaData* pData);
void catalogFreeMetaData(SMetaData* pData); void catalogFreeMetaData(SMetaData* pData);
int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* viewName, uint64_t viewId);
int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer);
int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg);
int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg);
int32_t catalogGetViewMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pViewName, STableMeta** pTableMeta);
int32_t ctgdEnableDebug(char* option, bool enable); int32_t ctgdEnableDebug(char* option, bool enable);
int32_t ctgdHandleDbgCommand(char* command); int32_t ctgdHandleDbgCommand(char* command);

32
include/libs/nodes/cmdnodes.h
View File

@ -26,7 +26,7 @@ extern "C" {
#define DESCRIBE_RESULT_COLS 4 #define DESCRIBE_RESULT_COLS 4
#define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_FIELD_LEN (TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE)
#define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_NOTE_LEN (16 + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_DB_RESULT_COLS 2 #define SHOW_CREATE_DB_RESULT_COLS 2
#define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE)
@ -36,6 +36,11 @@ extern "C" {
#define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) #define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_TB_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN * 3) #define SHOW_CREATE_TB_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN * 3)
#define SHOW_CREATE_VIEW_RESULT_COLS 2
#define SHOW_CREATE_VIEW_RESULT_FIELD1_LEN (TSDB_VIEW_FNAME_LEN + 4 + VARSTR_HEADER_SIZE)
#define SHOW_CREATE_VIEW_RESULT_FIELD2_LEN (TSDB_MAX_ALLOWED_SQL_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_COLS 3 #define SHOW_LOCAL_VARIABLES_RESULT_COLS 3
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE)
#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) #define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE)
@ -51,6 +56,7 @@ extern "C" {
#define PRIVILEGE_TYPE_READ BIT_FLAG_MASK(1) #define PRIVILEGE_TYPE_READ BIT_FLAG_MASK(1)
#define PRIVILEGE_TYPE_WRITE BIT_FLAG_MASK(2) #define PRIVILEGE_TYPE_WRITE BIT_FLAG_MASK(2)
#define PRIVILEGE_TYPE_SUBSCRIBE BIT_FLAG_MASK(3) #define PRIVILEGE_TYPE_SUBSCRIBE BIT_FLAG_MASK(3)
#define PRIVILEGE_TYPE_ALTER BIT_FLAG_MASK(4)
typedef struct SDatabaseOptions { typedef struct SDatabaseOptions {
ENodeType type; ENodeType type;
@ -297,6 +303,13 @@ typedef struct SShowCreateTableStmt {
void* pTableCfg; // STableCfg void* pTableCfg; // STableCfg
} SShowCreateTableStmt; } SShowCreateTableStmt;
typedef struct SShowCreateViewStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char viewName[TSDB_VIEW_NAME_LEN];
void* pViewMeta;
} SShowCreateViewStmt;
typedef struct SShowTableDistributedStmt { typedef struct SShowTableDistributedStmt {
ENodeType type; ENodeType type;
char dbName[TSDB_DB_NAME_LEN]; char dbName[TSDB_DB_NAME_LEN];
@ -490,6 +503,23 @@ typedef struct SDropFunctionStmt {
bool ignoreNotExists; bool ignoreNotExists;
} SDropFunctionStmt; } SDropFunctionStmt;
typedef struct SCreateViewStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char viewName[TSDB_VIEW_NAME_LEN];
char* pQuerySql;
bool orReplace;
SNode* pQuery;
SCMCreateViewReq createReq;
} SCreateViewStmt;
typedef struct SDropViewStmt {
ENodeType type;
char dbName[TSDB_DB_NAME_LEN];
char viewName[TSDB_VIEW_NAME_LEN];
bool ignoreNotExists;
} SDropViewStmt;
typedef struct SGrantStmt { typedef struct SGrantStmt {
ENodeType type; ENodeType type;
char userName[TSDB_USER_LEN]; char userName[TSDB_USER_LEN];

15
include/libs/nodes/querynodes.h
View File

@ -30,6 +30,11 @@ extern "C" {
#define VGROUPS_INFO_SIZE(pInfo) \ #define VGROUPS_INFO_SIZE(pInfo) \
(NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo))) (NULL == (pInfo) ? 0 : (sizeof(SVgroupsInfo) + (pInfo)->numOfVgroups * sizeof(SVgroupInfo)))
typedef struct SAssociationNode {
SNode** pPlace;
SNode* pAssociationNode;
} SAssociationNode;
typedef struct SRawExprNode { typedef struct SRawExprNode {
ENodeType nodeType; ENodeType nodeType;
char* p; char* p;
@ -182,6 +187,16 @@ typedef struct STempTableNode {
SNode* pSubquery; SNode* pSubquery;
} STempTableNode; } STempTableNode;
typedef struct SViewNode {
STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta;
SVgroupsInfo* pVgroupList;
char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES
double ratio;
SArray* pSmaIndexes;
int8_t cacheLastMode;
} SViewNode;
typedef enum EJoinType { typedef enum EJoinType {
JOIN_TYPE_INNER = 1, JOIN_TYPE_INNER = 1,
JOIN_TYPE_LEFT, JOIN_TYPE_LEFT,

38
include/libs/parser/parser.h
View File

@ -22,9 +22,7 @@ extern "C" {
#include "query.h" #include "query.h"
#include "querynodes.h" #include "querynodes.h"
#include "catalog.h"
struct SCatalogReq;
struct SMetaData;
typedef struct SStmtCallback { typedef struct SStmtCallback {
TAOS_STMT* pStmt; TAOS_STMT* pStmt;
@ -33,6 +31,33 @@ typedef struct SStmtCallback {
int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**); int32_t (*getExecInfoFn)(TAOS_STMT*, SHashObj**, SHashObj**);
} SStmtCallback; } SStmtCallback;
typedef enum {
PARSE_SQL_RES_QUERY = 1,
PARSE_SQL_RES_SCHEMA,
} SParseResType;
typedef struct SParseSchemaRes {
int8_t precision;
int32_t numOfCols;
SSchema* pSchema;
} SParseSchemaRes;
typedef struct SParseQueryRes {
SNode* pQuery;
SCatalogReq* pCatalogReq;
SMetaData meta;
} SParseQueryRes;
typedef struct SParseSqlRes {
SParseResType resType;
union {
SParseSchemaRes schemaRes;
SParseQueryRes queryRes;
};
} SParseSqlRes;
typedef int32_t (*parseSqlFn)(void*, const char*, const char*, bool, const char*, SParseSqlRes*);
typedef struct SParseCsvCxt { typedef struct SParseCsvCxt {
TdFilePtr fp; // last parsed file TdFilePtr fp; // last parsed file
int32_t tableNo; // last parsed table int32_t tableNo; // last parsed table
@ -55,6 +80,8 @@ typedef struct SParseContext {
struct SCatalog* pCatalog; struct SCatalog* pCatalog;
SStmtCallback* pStmtCb; SStmtCallback* pStmtCb;
const char* pUser; const char* pUser;
const char* pEffectiveUser;
bool parseOnly;
bool isSuperUser; bool isSuperUser;
bool enableSysInfo; bool enableSysInfo;
bool async; bool async;
@ -64,7 +91,10 @@ typedef struct SParseContext {
SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableMetaPos; // sql table pos => catalog data pos
SArray* pTableVgroupPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos
int64_t allocatorId; int64_t allocatorId;
parseSqlFn parseSqlFp;
void* parseSqlParam;
int8_t biMode; int8_t biMode;
SArray* pSubMetaList;
} SParseContext; } SParseContext;
int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery);
@ -125,6 +155,8 @@ int rawBlockBindData(SQuery *query, STableMeta* pTableMeta, void* data, SVCr
int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap); SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap); SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
void destoryCatalogReq(SCatalogReq *pCatalogReq);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

24
include/libs/qcom/query.h
View File

@ -119,6 +119,17 @@ typedef struct STableMeta {
} STableMeta; } STableMeta;
#pragma pack(pop) #pragma pack(pop)
typedef struct SViewMeta {
uint64_t viewId;
char* user;
char* querySql;
int8_t precision;
int8_t type;
int32_t version;
int32_t numOfCols;
SSchema* pSchema;
} SViewMeta;
typedef struct SDBVgInfo { typedef struct SDBVgInfo {
int32_t vgVersion; int32_t vgVersion;
int16_t hashPrefix; int16_t hashPrefix;
@ -148,6 +159,15 @@ typedef struct STableMetaOutput {
STableMeta* tbMeta; STableMeta* tbMeta;
} STableMetaOutput; } STableMetaOutput;
typedef struct SViewMetaOutput {
char name[TSDB_VIEW_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
char* querySql;
int8_t precision;
int32_t numOfCols;
SSchema* pSchema;
} SViewMetaOutput;
typedef struct SDataBuf { typedef struct SDataBuf {
int32_t msgType; int32_t msgType;
void* pData; void* pData;
@ -300,9 +320,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
(NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \ (NO_RET_REDIRECT_ERROR(_code) || SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || \
SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code)) SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || SYNC_OTHER_LEADER_REDIRECT_ERROR(_code))
#define IS_VIEW_REQUEST(_type) ((_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW)
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ #define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
(_type) == TDMT_MND_DROP_STB) (_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW)
#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \ #define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
(SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \ (SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \

172
include/libs/stream/tstream.h
View File

@ -38,16 +38,25 @@ extern "C" {
#define TASK_DOWNSTREAM_READY 0x0 #define TASK_DOWNSTREAM_READY 0x0
#define TASK_DOWNSTREAM_NOT_READY 0x1 #define TASK_DOWNSTREAM_NOT_READY 0x1
#define TASK_DOWNSTREAM_NOT_LEADER 0x2 #define TASK_DOWNSTREAM_NOT_LEADER 0x2
#define TASK_SELF_NEW_STAGE 0x3 #define TASK_UPSTREAM_NEW_STAGE 0x3
#define NODE_ROLE_UNINIT 0x1 #define NODE_ROLE_UNINIT 0x1
#define NODE_ROLE_LEADER 0x2 #define NODE_ROLE_LEADER 0x2
#define NODE_ROLE_FOLLOWER 0x3 #define NODE_ROLE_FOLLOWER 0x3
typedef struct SStreamTask SStreamTask; #define HAS_RELATED_FILLHISTORY_TASK(_t) ((_t)->hTaskInfo.id.taskId != 0)
typedef struct SStreamQueue SStreamQueue; #define CLEAR_RELATED_FILLHISTORY_TASK(_t) \
do { \
(_t)->hTaskInfo.id.taskId = 0; \
(_t)->hTaskInfo.id.streamId = 0; \
} while (0)
typedef struct SStreamTask SStreamTask;
typedef struct SStreamQueue SStreamQueue;
typedef struct SStreamTaskSM SStreamTaskSM;
#define SSTREAM_TASK_VER 2 #define SSTREAM_TASK_VER 2
enum { enum {
STREAM_STATUS__NORMAL = 0, STREAM_STATUS__NORMAL = 0,
STREAM_STATUS__STOP, STREAM_STATUS__STOP,
@ -58,7 +67,7 @@ enum {
}; };
typedef enum ETaskStatus { typedef enum ETaskStatus {
TASK_STATUS__NORMAL = 0, TASK_STATUS__READY = 0,
TASK_STATUS__DROPPING, TASK_STATUS__DROPPING,
TASK_STATUS__UNINIT, // not used, an placeholder TASK_STATUS__UNINIT, // not used, an placeholder
TASK_STATUS__STOP, TASK_STATUS__STOP,
@ -66,6 +75,7 @@ typedef enum ETaskStatus {
TASK_STATUS__HALT, // pause, but not be manipulated by user command TASK_STATUS__HALT, // pause, but not be manipulated by user command
TASK_STATUS__PAUSE, // pause TASK_STATUS__PAUSE, // pause
TASK_STATUS__CK, // stream task is in checkpoint status, no data are allowed to put into inputQ anymore TASK_STATUS__CK, // stream task is in checkpoint status, no data are allowed to put into inputQ anymore
TASK_STATUS__STREAM_SCAN_HISTORY,
} ETaskStatus; } ETaskStatus;
enum { enum {
@ -118,6 +128,22 @@ enum {
STREAM_META_OK_TO_STOP = 2, STREAM_META_OK_TO_STOP = 2,
}; };
typedef enum EStreamTaskEvent {
TASK_EVENT_INIT = 0x1,
TASK_EVENT_INIT_SCANHIST = 0x2,
TASK_EVENT_INIT_STREAM_SCANHIST = 0x3,
TASK_EVENT_SCANHIST_DONE = 0x4,
TASK_EVENT_STOP = 0x5,
TASK_EVENT_GEN_CHECKPOINT = 0x6,
TASK_EVENT_CHECKPOINT_DONE = 0x7,
TASK_EVENT_PAUSE = 0x8,
TASK_EVENT_RESUME = 0x9,
TASK_EVENT_HALT = 0xA,
TASK_EVENT_DROPPING = 0xB,
TASK_EVENT_SCAN_TSDB = 0xC,
TASK_EVENT_SCAN_WAL = 0xD,
} EStreamTaskEvent;
typedef struct { typedef struct {
int8_t type; int8_t type;
} SStreamQueueItem; } SStreamQueueItem;
@ -155,11 +181,6 @@ typedef struct {
SSDataBlock* pBlock; SSDataBlock* pBlock;
} SStreamRefDataBlock; } SStreamRefDataBlock;
typedef struct {
int8_t type;
SSDataBlock* pBlock;
} SStreamTrigger;
typedef struct SStreamQueueNode SStreamQueueNode; typedef struct SStreamQueueNode SStreamQueueNode;
struct SStreamQueueNode { struct SStreamQueueNode {
@ -215,6 +236,11 @@ typedef struct {
SUseDbRsp dbInfo; SUseDbRsp dbInfo;
} STaskDispatcherShuffle; } STaskDispatcherShuffle;
typedef struct {
int32_t nodeId;
SEpSet epset;
} SDownstreamTaskEpset;
typedef struct { typedef struct {
int64_t stbUid; int64_t stbUid;
char stbFullName[TSDB_TABLE_FNAME_LEN]; char stbFullName[TSDB_TABLE_FNAME_LEN];
@ -265,14 +291,15 @@ typedef struct SCheckpointInfo {
} SCheckpointInfo; } SCheckpointInfo;
typedef struct SStreamStatus { typedef struct SStreamStatus {
int8_t taskStatus; SStreamTaskSM* pSM;
int8_t downstreamReady; // downstream tasks are all ready now, if this flag is set int8_t taskStatus;
int8_t schedStatus; int8_t downstreamReady; // downstream tasks are all ready now, if this flag is set
int8_t keepTaskStatus; int8_t schedStatus;
bool appendTranstateBlock; // has append the transfer state data block already, todo: remove it int8_t keepTaskStatus;
int8_t pauseAllowed; // allowed task status to be set to be paused bool appendTranstateBlock; // has append the transfer state data block already, todo: remove it
int32_t timerActive; // timer is active int8_t pauseAllowed; // allowed task status to be set to be paused
int32_t inScanHistorySentinel; int32_t timerActive; // timer is active
int32_t inScanHistorySentinel;
} SStreamStatus; } SStreamStatus;
typedef struct SDataRange { typedef struct SDataRange {
@ -305,15 +332,10 @@ typedef struct SDispatchMsgInfo {
void* pTimer; // used to dispatch data after a given time duration void* pTimer; // used to dispatch data after a given time duration
} SDispatchMsgInfo; } SDispatchMsgInfo;
typedef struct STaskOutputQueue { typedef struct STaskQueue {
int8_t status; int8_t status;
SStreamQueue* queue; SStreamQueue* queue;
} STaskOutputQueue; } STaskQueue;
typedef struct STaskInputInfo {
int8_t status;
SStreamQueue* queue;
} STaskInputInfo;
typedef struct STaskSchedInfo { typedef struct STaskSchedInfo {
int8_t status; int8_t status;
@ -349,6 +371,7 @@ typedef struct SHistoryTaskInfo {
int32_t tickCount; int32_t tickCount;
int32_t retryTimes; int32_t retryTimes;
int32_t waitInterval; int32_t waitInterval;
int64_t haltVer; // offset in wal when halt the stream task
} SHistoryTaskInfo; } SHistoryTaskInfo;
typedef struct STaskOutputInfo { typedef struct STaskOutputInfo {
@ -361,6 +384,7 @@ typedef struct STaskOutputInfo {
}; };
int8_t type; int8_t type;
STokenBucket* pTokenBucket; STokenBucket* pTokenBucket;
SArray* pDownstreamUpdateList;
} STaskOutputInfo; } STaskOutputInfo;
typedef struct SUpstreamInfo { typedef struct SUpstreamInfo {
@ -372,8 +396,8 @@ struct SStreamTask {
int64_t ver; int64_t ver;
SStreamTaskId id; SStreamTaskId id;
SSTaskBasicInfo info; SSTaskBasicInfo info;
STaskOutputQueue outputq; STaskQueue outputq;
STaskInputInfo inputInfo; STaskQueue inputq;
STaskSchedInfo schedInfo; STaskSchedInfo schedInfo;
STaskOutputInfo outputInfo; STaskOutputInfo outputInfo;
SDispatchMsgInfo msgInfo; SDispatchMsgInfo msgInfo;
@ -408,38 +432,43 @@ struct SStreamTask {
typedef struct STaskStartInfo { typedef struct STaskStartInfo {
int64_t startTs; int64_t startTs;
int64_t readyTs; int64_t readyTs;
int32_t startedAfterNodeUpdate; int32_t startAllTasksFlag;
SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing
int32_t elapsedTime; int32_t elapsedTime;
} STaskStartInfo; } STaskStartInfo;
typedef struct STaskUpdateInfo {
SHashObj* pTasks;
int32_t transId;
} STaskUpdateInfo;
// meta // meta
typedef struct SStreamMeta { typedef struct SStreamMeta {
char* path; char* path;
TDB* db; TDB* db;
TTB* pTaskDb; TTB* pTaskDb;
TTB* pCheckpointDb; TTB* pCheckpointDb;
SHashObj* pTasksMap; SHashObj* pTasksMap;
SArray* pTaskList; // SArray<STaskId*> SArray* pTaskList; // SArray<STaskId*>
void* ahandle; void* ahandle;
TXN* txn; TXN* txn;
FTaskExpand* expandFunc; FTaskExpand* expandFunc;
int32_t vgId; int32_t vgId;
int64_t stage; int64_t stage;
int32_t role; int32_t role;
STaskStartInfo startInfo; STaskStartInfo startInfo;
SRWLatch lock; SRWLatch lock;
int32_t walScanCounter; int32_t walScanCounter;
void* streamBackend; void* streamBackend;
int64_t streamBackendRid; int64_t streamBackendRid;
SHashObj* pTaskBackendUnique; SHashObj* pTaskBackendUnique;
TdThreadMutex backendMutex; TdThreadMutex backendMutex;
SMetaHbInfo* pHbInfo; SMetaHbInfo* pHbInfo;
SHashObj* pUpdateTaskSet; STaskUpdateInfo updateInfo;
int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta int32_t numOfStreamTasks; // this value should be increased when a new task is added into the meta
int32_t numOfPausedTasks; int32_t numOfPausedTasks;
int32_t chkptNotReadyTasks; int32_t chkptNotReadyTasks;
int64_t rid; int64_t rid;
int64_t chkpId; int64_t chkpId;
SArray* chkpSaved; SArray* chkpSaved;
@ -617,7 +646,8 @@ typedef struct STaskStatusEntry {
typedef struct SStreamHbMsg { typedef struct SStreamHbMsg {
int32_t vgId; int32_t vgId;
int32_t numOfTasks; int32_t numOfTasks;
SArray* pTaskStatus; // SArray<SStreamTaskStatusEntry> SArray* pTaskStatus; // SArray<STaskStatusEntry>
SArray* pUpdateNodes; // SArray<int32_t>, needs update the epsets in stream tasks for those nodes.
} SStreamHbMsg; } SStreamHbMsg;
int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp); int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pRsp);
@ -641,6 +671,7 @@ typedef struct SNodeUpdateInfo {
} SNodeUpdateInfo; } SNodeUpdateInfo;
typedef struct SStreamTaskNodeUpdateMsg { typedef struct SStreamTaskNodeUpdateMsg {
int32_t transId; // to identify the msg
int64_t streamId; int64_t streamId;
int32_t taskId; int32_t taskId;
SArray* pNodeList; // SArray<SNodeUpdateInfo> SArray* pNodeList; // SArray<SNodeUpdateInfo>
@ -686,24 +717,36 @@ SStreamChildEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t
void streamTaskInputFail(SStreamTask* pTask); void streamTaskInputFail(SStreamTask* pTask);
int32_t streamExecTask(SStreamTask* pTask); int32_t streamExecTask(SStreamTask* pTask);
int32_t streamSchedExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask);
bool streamTaskShouldStop(const SStreamStatus* pStatus); bool streamTaskShouldStop(const SStreamTask* pStatus);
bool streamTaskShouldPause(const SStreamStatus* pStatus); bool streamTaskShouldPause(const SStreamTask* pStatus);
bool streamTaskIsIdle(const SStreamTask* pTask); bool streamTaskIsIdle(const SStreamTask* pTask);
bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus);
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
ETaskStatus streamTaskGetStatus(const SStreamTask* pTask, char** pStr);
const char* streamTaskGetStatusStr(ETaskStatus status);
void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask);
void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen);
char* createStreamTaskIdStr(int64_t streamId, int32_t taskId);
// recover and fill history // recover and fill history
void streamTaskCheckDownstream(SStreamTask* pTask); void streamTaskCheckDownstream(SStreamTask* pTask);
int32_t streamTaskStartScanHistory(SStreamTask* pTask); int32_t onNormalTaskReady(SStreamTask* pTask);
int32_t onScanhistoryTaskReady(SStreamTask* pTask);
int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage); int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage);
int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask);
bool streamTaskAllUpstreamClosed(SStreamTask* pTask); bool streamTaskAllUpstreamClosed(SStreamTask* pTask);
bool streamTaskSetSchedStatusWait(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask);
int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask);
int8_t streamTaskSetSchedStatusInActive(SStreamTask* pTask); int8_t streamTaskSetSchedStatusInactive(SStreamTask* pTask);
int32_t streamTaskClearHTaskAttr(SStreamTask* pTask);
int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event);
int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent event);
void streamTaskRestoreStatus(SStreamTask* pTask);
int32_t streamTaskStop(SStreamTask* pTask); int32_t streamTaskStop(SStreamTask* pTask);
int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp, int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq* pReq, SStreamTaskCheckRsp* pRsp,
@ -714,16 +757,11 @@ int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask);
int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated); int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);
int32_t streamQueueGetAvailableSpace(const SStreamQueue* pQueue, int32_t* availNum, double* availSize);
// common // common
int32_t streamRestoreParam(SStreamTask* pTask); int32_t streamRestoreParam(SStreamTask* pTask);
int32_t streamSetStatusNormal(SStreamTask* pTask);
int32_t streamSetStatusUnint(SStreamTask* pTask);
const char* streamGetTaskStatusStr(int32_t status);
void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta); void streamTaskPause(SStreamTask* pTask, SStreamMeta* pMeta);
void streamTaskResume(SStreamTask* pTask, SStreamMeta* pMeta); void streamTaskResume(SStreamTask* pTask);
void streamTaskResumeFromHalt(SStreamTask* pTask);
void streamTaskDisablePause(SStreamTask* pTask); void streamTaskDisablePause(SStreamTask* pTask);
void streamTaskEnablePause(SStreamTask* pTask); void streamTaskEnablePause(SStreamTask* pTask);
int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask); int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask);
@ -759,7 +797,6 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey);
int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded);
int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta);
int32_t streamMetaGetNumOfStreamTasks(SStreamMeta* pMeta);
SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId);
void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask);
int32_t streamMetaReopen(SStreamMeta* pMeta); int32_t streamMetaReopen(SStreamMeta* pMeta);
@ -768,12 +805,13 @@ int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta);
void streamMetaNotifyClose(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta);
void streamMetaStartHb(SStreamMeta* pMeta); void streamMetaStartHb(SStreamMeta* pMeta);
void streamMetaInitForSnode(SStreamMeta* pMeta); void streamMetaInitForSnode(SStreamMeta* pMeta);
bool streamMetaTaskInTimer(SStreamMeta* pMeta);
int32_t streamMetaUpdateTaskReadyInfo(SStreamTask* pTask);
// checkpoint // checkpoint
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq);
int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask); int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask);
void streamTaskClearCheckInfo(SStreamTask* pTask); void streamTaskClearCheckInfo(SStreamTask* pTask);
int32_t streamAlignTransferState(SStreamTask* pTask); int32_t streamAlignTransferState(SStreamTask* pTask);
int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId); int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId);
int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask, int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask,

28
include/util/taoserror.h
View File

@ -332,6 +332,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_REPLICA TAOS_DEF_ERROR_CODE(0, 0x03B7) #define TSDB_CODE_MND_INVALID_REPLICA TAOS_DEF_ERROR_CODE(0, 0x03B7)
#define TSDB_CODE_MND_DNODE_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x03B8) #define TSDB_CODE_MND_DNODE_IN_CREATING TAOS_DEF_ERROR_CODE(0, 0x03B8)
#define TSDB_CODE_MND_DNODE_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x03B9) #define TSDB_CODE_MND_DNODE_IN_DROPPING TAOS_DEF_ERROR_CODE(0, 0x03B9)
#define TSDB_CODE_MND_NO_ENOUGH_VNODES TAOS_DEF_ERROR_CODE(0, 0x03BA)
// mnode-stable-part2 // mnode-stable-part2
#define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03C0) #define TSDB_CODE_MND_NAME_CONFLICT_WITH_TOPIC TAOS_DEF_ERROR_CODE(0, 0x03C0)
@ -383,15 +384,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_TOO_MANY_STREAMS TAOS_DEF_ERROR_CODE(0, 0x03F6) #define TSDB_CODE_MND_TOO_MANY_STREAMS TAOS_DEF_ERROR_CODE(0, 0x03F6)
#define TSDB_CODE_MND_INVALID_TARGET_TABLE TAOS_DEF_ERROR_CODE(0, 0x03F7) #define TSDB_CODE_MND_INVALID_TARGET_TABLE TAOS_DEF_ERROR_CODE(0, 0x03F7)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481)
#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482)
// mnode-tag-indxe
#define TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0483)
#define TSDB_CODE_MND_TAG_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0484)
// dnode // dnode
// #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x // #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x
@ -418,6 +411,22 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal #define TSDB_CODE_MNODE_NO_NEED_RESTORE TAOS_DEF_ERROR_CODE(0, 0x0415) // internal
#define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416) #define TSDB_CODE_DNODE_ONLY_USE_WHEN_OFFLINE TAOS_DEF_ERROR_CODE(0, 0x0416)
// mnode-sma
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481)
#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482)
// mnode-tag-indxe
#define TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0483)
#define TSDB_CODE_MND_TAG_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0484)
// mnode-view
#define TSDB_CODE_MND_VIEW_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A0)
#define TSDB_CODE_MND_VIEW_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04A1)
// vnode // vnode
// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x
// #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x // #define TSDB_CODE_VND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0501) // 2.x
@ -725,6 +734,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669) #define TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2669)
#define TSDB_CODE_PAR_INVALID_VARBINARY TAOS_DEF_ERROR_CODE(0, 0x266A) #define TSDB_CODE_PAR_INVALID_VARBINARY TAOS_DEF_ERROR_CODE(0, 0x266A)
#define TSDB_CODE_PAR_INVALID_IP_RANGE TAOS_DEF_ERROR_CODE(0, 0x266B) #define TSDB_CODE_PAR_INVALID_IP_RANGE TAOS_DEF_ERROR_CODE(0, 0x266B)
#define TSDB_CODE_PAR_INVALID_VIEW_QUERY TAOS_DEF_ERROR_CODE(0, 0x266C)
#define TSDB_CODE_PAR_COL_QUERY_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x266D)
#define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF) #define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner //planner

2
include/util/tconfig.h
View File

@ -65,7 +65,6 @@ typedef struct SConfigItem {
union { union {
bool bval; bool bval;
float fval; float fval;
double dval;
int32_t i32; int32_t i32;
int64_t i64; int64_t i64;
char *str; char *str;
@ -104,7 +103,6 @@ int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scop
int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope);
int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope); int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope);
int32_t cfgAddDouble(SConfig *pCfg, const char *name, double defaultVal, double minval, double maxval, int8_t scope);
int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);
int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope);

5
include/util/tdef.h
View File

@ -240,6 +240,9 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_SQL_SHOW_LEN 1024 #define TSDB_MAX_SQL_SHOW_LEN 1024
#define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb #define TSDB_MAX_ALLOWED_SQL_LEN (1 * 1024 * 1024u) // sql length should be less than 1mb
#define TSDB_VIEW_NAME_LEN 193
#define TSDB_VIEW_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_VIEW_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_APP_NAME_LEN TSDB_UNI_LEN #define TSDB_APP_NAME_LEN TSDB_UNI_LEN
#define TSDB_TB_COMMENT_LEN 1025 #define TSDB_TB_COMMENT_LEN 1025
@ -327,7 +330,7 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440)
#define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440) #define TSDB_DEFAULT_DAYS_PER_FILE (10 * 1440)
#define TSDB_MIN_DURATION_PER_FILE 60 // unit minute #define TSDB_MIN_DURATION_PER_FILE 60 // unit minute
#define TSDB_MAX_DURATION_PER_FILE (3650 * 1440) #define TSDB_MAX_DURATION_PER_FILE (90 * 1440)
#define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440) #define TSDB_DEFAULT_DURATION_PER_FILE (10 * 1440)
#define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute #define TSDB_MIN_KEEP (1 * 1440) // data in db to be reserved. unit minute
#define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved. #define TSDB_MAX_KEEP (365000 * 1440) // data in db to be reserved.

8
include/util/tqueue.h
View File

@ -101,6 +101,9 @@ struct STaosQall {
STaosQnode *current; STaosQnode *current;
STaosQnode *start; STaosQnode *start;
int32_t numOfItems; int32_t numOfItems;
int64_t memOfItems;
int32_t unAccessedNumOfItems;
int64_t unAccessMemOfItems;
}; };
STaosQueue *taosOpenQueue(); STaosQueue *taosOpenQueue();
@ -123,6 +126,9 @@ int32_t taosReadAllQitems(STaosQueue *queue, STaosQall *qall);
int32_t taosGetQitem(STaosQall *qall, void **ppItem); int32_t taosGetQitem(STaosQall *qall, void **ppItem);
void taosResetQitems(STaosQall *qall); void taosResetQitems(STaosQall *qall);
int32_t taosQallItemSize(STaosQall *qall); int32_t taosQallItemSize(STaosQall *qall);
int64_t taosQallMemSize(STaosQall *qll);
int64_t taosQallUnAccessedItemSize(STaosQall *qall);
int64_t taosQallUnAccessedMemSize(STaosQall *qall);
STaosQset *taosOpenQset(); STaosQset *taosOpenQset();
void taosCloseQset(STaosQset *qset); void taosCloseQset(STaosQset *qset);
@ -135,8 +141,6 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, SQueueInfo *qinfo)
int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, SQueueInfo *qinfo);
void taosResetQsetThread(STaosQset *qset, void *pItem); void taosResetQsetThread(STaosQset *qset, void *pItem);
extern int64_t tsRpcQueueMemoryAllowed;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

4
source/client/CMakeLists.txt
View File

@ -1,5 +1,9 @@
aux_source_directory(src CLIENT_SRC) aux_source_directory(src CLIENT_SRC)
IF (TD_ENTERPRISE)
LIST(APPEND CLIENT_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/clientView.c)
ENDIF ()
if(TD_WINDOWS) if(TD_WINDOWS)
add_library(taos SHARED ${CLIENT_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/taos.rc.in) add_library(taos SHARED ${CLIENT_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/src/taos.rc.in)
else() else()

12
source/client/inc/clientInt.h
View File

@ -269,6 +269,7 @@ typedef struct SRequestObj {
bool syncQuery; // todo refactor: async query object bool syncQuery; // todo refactor: async query object
bool stableQuery; // todo refactor bool stableQuery; // todo refactor
bool validateOnly; // todo refactor bool validateOnly; // todo refactor
bool parseOnly;
bool killed; bool killed;
bool inRetry; bool inRetry;
bool isSubReq; bool isSubReq;
@ -279,6 +280,8 @@ typedef struct SRequestObj {
void* pPostPlan; void* pPostPlan;
SReqRelInfo relation; SReqRelInfo relation;
void* pWrapper; void* pWrapper;
SMetaData parseMeta;
char* effectiveUser;
} SRequestObj; } SRequestObj;
typedef struct SSyncQueryParam { typedef struct SSyncQueryParam {
@ -305,6 +308,8 @@ void taosAsyncQueryImpl(uint64_t connId, const char* sql, __taos_async_fn_t fp,
void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly, void taosAsyncQueryImplWithReqid(uint64_t connId, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly,
int64_t reqid); int64_t reqid);
void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param); void taosAsyncFetchImpl(SRequestObj *pRequest, __taos_async_fn_t fp, void *param);
int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes);
void syncQueryFn(void* param, void* res, int32_t code);
int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols); int32_t getVersion1BlockMetaSize(const char* p, int32_t numOfCols);
@ -403,7 +408,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResu
int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest);
int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList);
void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta); void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta);
int32_t removeMeta(STscObj* pTscObj, SArray* tbList); int32_t removeMeta(STscObj* pTscObj, SArray* tbList, bool isView);
int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);
int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog); int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog);
bool qnodeRequired(SRequestObj* pRequest); bool qnodeRequired(SRequestObj* pRequest);
@ -415,6 +420,11 @@ int32_t buildPreviousRequest(SRequestObj *pRequest, const char* sql, SRequestObj
int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce); int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *pRequest, bool updateMetaForce);
void returnToUser(SRequestObj* pRequest); void returnToUser(SRequestObj* pRequest);
void stopAllQueries(SRequestObj *pRequest); void stopAllQueries(SRequestObj *pRequest);
void freeQueryParam(SSyncQueryParam* param);
#ifdef TD_ENTERPRISE
int32_t clientParseSqlImpl(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effeciveUser, SParseSqlRes* pRes);
#endif
#ifdef __cplusplus #ifdef __cplusplus
} }

1
source/client/src/clientEnv.c
View File

@ -447,6 +447,7 @@ void doDestroyRequest(void *p) {
qDestroyQuery(pRequest->pQuery); qDestroyQuery(pRequest->pQuery);
nodesDestroyAllocator(pRequest->allocatorRefId); nodesDestroyAllocator(pRequest->allocatorRefId);
taosMemoryFreeClear(pRequest->effectiveUser);
taosMemoryFreeClear(pRequest->sqlstr); taosMemoryFreeClear(pRequest->sqlstr);
taosMemoryFree(pRequest); taosMemoryFree(pRequest);
tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest); tscTrace("end to destroy request %" PRIx64 " p:%p", reqId, pRequest);

248
source/client/src/clientHb.c
View File

@ -53,7 +53,7 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC
int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray);
for (int32_t i = 0; i < numOfBatchs; ++i) { for (int32_t i = 0; i < numOfBatchs; ++i) {
SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i); SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i);
tscDebug("hb user auth rsp, user:%s, version:%d", rsp->user, rsp->version); tscDebug("hb to update user auth, user:%s, version:%d", rsp->user, rsp->version);
catalogUpdateUserAuthInfo(pCatalog, rsp); catalogUpdateUserAuthInfo(pCatalog, rsp);
} }
@ -205,6 +205,7 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
rsp->useDbRsp->db, rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid); rsp->useDbRsp->db, rsp->useDbRsp->vgVersion, rsp->useDbRsp->stateTs, rsp->useDbRsp->uid);
if (rsp->useDbRsp->vgVersion < 0) { if (rsp->useDbRsp->vgVersion < 0) {
tscDebug("hb to remove db, db:%s", rsp->useDbRsp->db);
code = catalogRemoveDB(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid); code = catalogRemoveDB(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid);
} else { } else {
SDBVgInfo *vgInfo = NULL; SDBVgInfo *vgInfo = NULL;
@ -213,6 +214,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
goto _return; goto _return;
} }
tscDebug("hb to update db vgInfo, db:%s", rsp->useDbRsp->db);
catalogUpdateDBVgInfo(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid, vgInfo); catalogUpdateDBVgInfo(pCatalog, rsp->useDbRsp->db, rsp->useDbRsp->uid, vgInfo);
if (IS_SYS_DBNAME(rsp->useDbRsp->db)) { if (IS_SYS_DBNAME(rsp->useDbRsp->db)) {
@ -253,10 +256,10 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i); STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i);
if (rsp->numOfColumns < 0) { if (rsp->numOfColumns < 0) {
tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); tscDebug("hb to remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid); catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid);
} else { } else {
tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); tscDebug("hb to update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName);
if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) {
tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId);
tFreeSSTbHbRsp(&hbRsp); tFreeSSTbHbRsp(&hbRsp);
@ -281,6 +284,108 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t hbProcessDynViewRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
return catalogUpdateDynViewVer(pCatalog, (SDynViewVersion*)value);
}
static void hbFreeSViewMetaInRsp(void* p) {
if (NULL == p || NULL == *(void**)p) {
return;
}
SViewMetaRsp *pRsp = *(SViewMetaRsp**)p;
tFreeSViewMetaRsp(pRsp);
taosMemoryFreeClear(pRsp);
}
static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) {
int32_t code = 0;
SViewHbRsp hbRsp = {0};
if (tDeserializeSViewHbRsp(value, valueLen, &hbRsp) != 0) {
taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp);
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp);
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i);
if (rsp->numOfCols < 0) {
tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name);
catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId);
tFreeSViewMetaRsp(rsp);
taosMemoryFreeClear(rsp);
} else {
tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name);
catalogUpdateViewMeta(pCatalog, rsp);
}
}
taosArrayDestroy(hbRsp.pViewRsp);
return TSDB_CODE_SUCCESS;
}
static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
for (int32_t i = 0; i < kvNum; ++i) {
SKv *kv = taosArrayGet(pKvs, i);
switch (kv->key) {
case HEARTBEAT_KEY_USER_AUTHINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog, pAppHbMgr);
break;
}
case HEARTBEAT_KEY_DBINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog);
break;
}
case HEARTBEAT_KEY_STBINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog);
break;
}
#ifdef TD_ENTERPRISE
case HEARTBEAT_KEY_DYN_VIEW: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid dyn view info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
hbProcessDynViewRsp(kv->value, kv->valueLen, pCatalog);
break;
}
case HEARTBEAT_KEY_VIEWINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid view info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog);
break;
}
#endif
default:
tscError("invalid hb key type:%d", kv->key);
break;
}
}
}
static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &pRsp->connKey, sizeof(SClientHbKey)); SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &pRsp->connKey, sizeof(SClientHbKey));
if (NULL == pReq) { if (NULL == pReq) {
@ -338,66 +443,16 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) {
tscDebug("hb got %d rsp kv", kvNum); tscDebug("hb got %d rsp kv", kvNum);
for (int32_t i = 0; i < kvNum; ++i) { if (kvNum > 0) {
SKv *kv = taosArrayGet(pRsp->info, i); struct SCatalog *pCatalog = NULL;
switch (kv->key) { int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
case HEARTBEAT_KEY_USER_AUTHINFO: { if (code != TSDB_CODE_SUCCESS) {
if (kv->valueLen <= 0 || NULL == kv->value) { tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value); } else {
break; hbProcessQueryRspKvs(kvNum, pRsp->info, pCatalog, pAppHbMgr);
}
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
break;
}
hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog, pAppHbMgr);
break;
}
case HEARTBEAT_KEY_DBINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
break;
}
hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog);
break;
}
case HEARTBEAT_KEY_STBINFO: {
if (kv->valueLen <= 0 || NULL == kv->value) {
tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value);
break;
}
struct SCatalog *pCatalog = NULL;
int32_t code = catalogGetHandle(pReq->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", pReq->clusterId, tstrerror(code));
break;
}
hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog);
break;
}
default:
tscError("invalid hb key type:%d", kv->key);
break;
} }
} }
taosHashRelease(pAppHbMgr->activeInfo, pReq); taosHashRelease(pAppHbMgr->activeInfo, pReq);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -740,8 +795,8 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
for (int32_t i = 0; i < stbNum; ++i) { for (int32_t i = 0; i < stbNum; ++i) {
SSTableVersion *stb = &stbs[i]; SSTableVersion *stb = &stbs[i];
stb->suid = htobe64(stb->suid); stb->suid = htobe64(stb->suid);
stb->sversion = htons(stb->sversion); stb->sversion = htonl(stb->sversion);
stb->tversion = htons(stb->tversion); stb->tversion = htonl(stb->tversion);
stb->smaVer = htonl(stb->smaVer); stb->smaVer = htonl(stb->smaVer);
} }
@ -762,6 +817,56 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) {
SViewVersion *views = NULL;
uint32_t viewNum = 0;
int32_t code = 0;
SDynViewVersion *pDynViewVer = NULL;
code = catalogGetExpiredViews(pCatalog, &views, &viewNum, &pDynViewVer);
if (TSDB_CODE_SUCCESS != code) {
taosMemoryFree(views);
taosMemoryFree(pDynViewVer);
return code;
}
if (viewNum <= 0) {
taosMemoryFree(views);
taosMemoryFree(pDynViewVer);
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < viewNum; ++i) {
SViewVersion *view = &views[i];
view->dbId = htobe64(view->dbId);
view->viewId = htobe64(view->viewId);
view->version = htonl(view->version);
}
tscDebug("hb got %d expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum);
if (NULL == req->info) {
req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
}
SKv kv = {
.key = HEARTBEAT_KEY_DYN_VIEW,
.valueLen = sizeof(SDynViewVersion),
.value = pDynViewVer,
};
taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
kv.key = HEARTBEAT_KEY_VIEWINFO;
kv.valueLen = sizeof(SViewVersion) * viewNum;
kv.value = views;
taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
return TSDB_CODE_SUCCESS;
}
int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) {
SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId));
if (NULL != pApp) { if (NULL != pApp) {
@ -781,19 +886,17 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
SHbParam *hbParam = (SHbParam *)param; SHbParam *hbParam = (SHbParam *)param;
SCatalog *pCatalog = NULL; SCatalog *pCatalog = NULL;
hbGetQueryBasicInfo(connKey, req);
if (hbParam->reqCnt == 0) { if (hbParam->reqCnt == 0) {
code = catalogGetHandle(hbParam->clusterId, &pCatalog); code = catalogGetHandle(hbParam->clusterId, &pCatalog);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code)); tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code));
return code; return code;
} }
}
hbGetAppInfo(hbParam->clusterId, req); hbGetAppInfo(hbParam->clusterId, req);
hbGetQueryBasicInfo(connKey, req);
if (hbParam->reqCnt == 0) {
if (!taosHashGet(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(hbParam->clusterId))) { if (!taosHashGet(clientHbMgr.appHbHash, &hbParam->clusterId, sizeof(hbParam->clusterId))) {
code = hbGetExpiredUserInfo(connKey, pCatalog, req); code = hbGetExpiredUserInfo(connKey, pCatalog, req);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -819,6 +922,15 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
} }
#ifdef TD_ENTERPRISE
code = hbGetExpiredViewInfo(connKey, pCatalog, req);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
#endif
} else {
req->app.appId = 0;
} }
++hbParam->reqCnt; // success to get catalog info ++hbParam->reqCnt; // success to get catalog info

87
source/client/src/clientImpl.c
View File

@ -1009,7 +1009,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
tscDebug("schedulerExecCb request type %s", TMSG_INFO(pRequest->type)); tscDebug("schedulerExecCb request type %s", TMSG_INFO(pRequest->type));
if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) { if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) {
removeMeta(pTscObj, pRequest->targetTableList); removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
} }
pRequest->metric.execCostUs = taosGetTimestampUs() - pRequest->metric.execStart; pRequest->metric.execCostUs = taosGetTimestampUs() - pRequest->metric.execStart;
@ -1097,7 +1097,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
} }
if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) { if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type) && NULL == pRequest->body.resInfo.execRes.res) {
removeMeta(pRequest->pTscObj, pRequest->targetTableList); removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
} }
handleQueryExecRsp(pRequest); handleQueryExecRsp(pRequest);
@ -1116,31 +1116,34 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue
static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta,
SSqlCallbackWrapper* pWrapper) { SSqlCallbackWrapper* pWrapper) {
int32_t code = TSDB_CODE_SUCCESS;
pRequest->type = pQuery->msgType; pRequest->type = pQuery->msgType;
SArray* pMnodeList = NULL;
SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); SQueryPlan* pDag = NULL;
SPlanContext cxt = {.queryId = pRequest->requestId,
.acctId = pRequest->pTscObj->acctId,
.mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pUser = pRequest->pTscObj->user,
.sysInfo = pRequest->pTscObj->sysInfo,
.allocatorId = pRequest->allocatorRefId};
SQueryPlan* pDag = NULL;
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
if (code) { if (!pRequest->parseOnly) {
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad));
pRequest->requestId);
} else { SPlanContext cxt = {.queryId = pRequest->requestId,
pRequest->body.subplanNum = pDag->numOfSubplans; .acctId = pRequest->pTscObj->acctId,
TSWAP(pRequest->pPostPlan, pDag->pPostPlan); .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp),
.pAstRoot = pQuery->pRoot,
.showRewrite = pQuery->showRewrite,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE,
.pUser = pRequest->pTscObj->user,
.sysInfo = pRequest->pTscObj->sysInfo,
.allocatorId = pRequest->allocatorRefId};
code = qCreateQueryPlan(&cxt, &pDag, pMnodeList);
if (code) {
tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId);
} else {
pRequest->body.subplanNum = pDag->numOfSubplans;
TSWAP(pRequest->pPostPlan, pDag->pPostPlan);
}
} }
pRequest->metric.execStart = taosGetTimestampUs(); pRequest->metric.execStart = taosGetTimestampUs();
@ -1173,6 +1176,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
code = schedulerExecJob(&req, &pRequest->body.queryJob); code = schedulerExecJob(&req, &pRequest->body.queryJob);
taosArrayDestroy(pNodeList); taosArrayDestroy(pNodeList);
} else { } else {
qDestroyQueryPlan(pDag);
tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), tscDebug("0x%" PRIx64 " plan not executed, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code),
pRequest->requestId); pRequest->requestId);
destorySqlCallbackWrapper(pWrapper); destorySqlCallbackWrapper(pWrapper);
@ -1193,6 +1197,11 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat
void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper) { void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta, SSqlCallbackWrapper* pWrapper) {
int32_t code = 0; int32_t code = 0;
if (pRequest->parseOnly) {
pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
return;
}
pRequest->body.execMode = pQuery->execMode; pRequest->body.execMode = pQuery->execMode;
if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) { if (QUERY_EXEC_MODE_SCHEDULE != pRequest->body.execMode) {
destorySqlCallbackWrapper(pWrapper); destorySqlCallbackWrapper(pWrapper);
@ -1226,6 +1235,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM
pRequest->body.queryFp(pRequest->body.param, pRequest, 0); pRequest->body.queryFp(pRequest->body.param, pRequest, 0);
break; break;
default: default:
tscError("0x%" PRIx64 " invalid execMode %d", pRequest->self, pQuery->execMode);
pRequest->body.queryFp(pRequest->body.param, pRequest, -1); pRequest->body.queryFp(pRequest->body.param, pRequest, -1);
break; break;
} }
@ -1272,7 +1282,7 @@ int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest) {
return code; return code;
} }
int32_t removeMeta(STscObj* pTscObj, SArray* tbList) { int32_t removeMeta(STscObj* pTscObj, SArray* tbList, bool isView) {
SCatalog* pCatalog = NULL; SCatalog* pCatalog = NULL;
int32_t tbNum = taosArrayGetSize(tbList); int32_t tbNum = taosArrayGetSize(tbList);
int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog);
@ -1280,9 +1290,18 @@ int32_t removeMeta(STscObj* pTscObj, SArray* tbList) {
return code; return code;
} }
for (int32_t i = 0; i < tbNum; ++i) { if (isView) {
SName* pTbName = taosArrayGet(tbList, i); for (int32_t i = 0; i < tbNum; ++i) {
catalogRemoveTableMeta(pCatalog, pTbName); SName* pViewName = taosArrayGet(tbList, i);
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pViewName, dbFName);
catalogRemoveViewMeta(pCatalog, dbFName, 0, pViewName->tname, 0);
}
} else {
for (int32_t i = 0; i < tbNum; ++i) {
SName* pTbName = taosArrayGet(tbList, i);
catalogRemoveTableMeta(pCatalog, pTbName);
}
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2604,3 +2623,13 @@ void taosAsyncFetchImpl(SRequestObj* pRequest, __taos_async_fn_t fp, void* param
schedulerFetchRows(pRequest->body.queryJob, &req); schedulerFetchRows(pRequest->body.queryJob, &req);
} }
int32_t clientParseSql(void* param, const char* dbName, const char* sql, bool parseOnly, const char* effectiveUser, SParseSqlRes* pRes) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_SUCCESS;
#else
return clientParseSqlImpl(param, dbName, sql, parseOnly, effectiveUser, pRes);
#endif
}

51
source/client/src/clientMain.c
View File

@ -32,7 +32,7 @@
#define TSC_VAR_RELEASED 0 #define TSC_VAR_RELEASED 0
static int32_t sentinel = TSC_VAR_NOT_RELEASE; static int32_t sentinel = TSC_VAR_NOT_RELEASE;
static int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt); static int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SSqlCallbackWrapper *pWrapper);
int taos_options(TSDB_OPTION option, const void *arg, ...) { int taos_options(TSDB_OPTION option, const void *arg, ...) {
static int32_t lock = 0; static int32_t lock = 0;
@ -879,39 +879,12 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) {
return code; return code;
} }
static void destoryTablesReq(void *p) {
STablesReq *pRes = (STablesReq *)p;
taosArrayDestroy(pRes->pTables);
}
static void destoryCatalogReq(SCatalogReq *pCatalogReq) {
if (NULL == pCatalogReq) {
return;
}
taosArrayDestroy(pCatalogReq->pDbVgroup);
taosArrayDestroy(pCatalogReq->pDbCfg);
taosArrayDestroy(pCatalogReq->pDbInfo);
if (pCatalogReq->cloned) {
taosArrayDestroy(pCatalogReq->pTableMeta);
taosArrayDestroy(pCatalogReq->pTableHash);
} else {
taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq);
taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq);
}
taosArrayDestroy(pCatalogReq->pUdf);
taosArrayDestroy(pCatalogReq->pIndex);
taosArrayDestroy(pCatalogReq->pUser);
taosArrayDestroy(pCatalogReq->pTableIndex);
taosArrayDestroy(pCatalogReq->pTableCfg);
taosArrayDestroy(pCatalogReq->pTableTag);
taosMemoryFree(pCatalogReq);
}
void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) { void destorySqlCallbackWrapper(SSqlCallbackWrapper *pWrapper) {
if (NULL == pWrapper) { if (NULL == pWrapper) {
return; return;
} }
destoryCatalogReq(pWrapper->pCatalogReq); destoryCatalogReq(pWrapper->pCatalogReq);
taosMemoryFree(pWrapper->pCatalogReq);
qDestroyParseContext(pWrapper->pParseCtx); qDestroyParseContext(pWrapper->pParseCtx);
taosMemoryFree(pWrapper); taosMemoryFree(pWrapper);
} }
@ -933,6 +906,7 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
int64_t analyseStart = taosGetTimestampUs(); int64_t analyseStart = taosGetTimestampUs();
pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart; pRequest->metric.ctgCostUs = analyseStart - pRequest->metric.ctgStart;
pWrapper->pParseCtx->parseOnly = pRequest->parseOnly;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery); code = qAnalyseSqlSemantic(pWrapper->pParseCtx, pWrapper->pCatalogReq, pResultMeta, pQuery);
@ -940,6 +914,11 @@ static void doAsyncQueryFromAnalyse(SMetaData *pResultMeta, void *param, int32_t
pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart; pRequest->metric.analyseCostUs += taosGetTimestampUs() - analyseStart;
if (pRequest->parseOnly) {
memcpy(&pRequest->parseMeta, pResultMeta, sizeof(*pResultMeta));
memset(pResultMeta, 0, sizeof(*pResultMeta));
}
handleQueryAnslyseRes(pWrapper, pResultMeta, code); handleQueryAnslyseRes(pWrapper, pResultMeta, code);
} }
@ -960,6 +939,7 @@ int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) {
pTarget->pTableIndex = taosArrayDup(pSrc->pTableIndex, NULL); pTarget->pTableIndex = taosArrayDup(pSrc->pTableIndex, NULL);
pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL); pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL);
pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL); pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL);
pTarget->pView = taosArrayDup(pSrc->pView, NULL);
pTarget->qNodeRequired = pSrc->qNodeRequired; pTarget->qNodeRequired = pSrc->qNodeRequired;
pTarget->dNodeRequired = pSrc->dNodeRequired; pTarget->dNodeRequired = pSrc->dNodeRequired;
pTarget->svrVerRequired = pSrc->svrVerRequired; pTarget->svrVerRequired = pSrc->svrVerRequired;
@ -1147,7 +1127,7 @@ void taos_query_a_with_reqid(TAOS *taos, const char *sql, __taos_async_fn_t fp,
taosAsyncQueryImplWithReqid(connId, sql, fp, param, false, reqid); taosAsyncQueryImplWithReqid(connId, sql, fp, param, false, reqid);
} }
int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) { int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt, SSqlCallbackWrapper *pWrapper) {
const STscObj *pTscObj = pRequest->pTscObj; const STscObj *pTscObj = pRequest->pTscObj;
*pCxt = taosMemoryCalloc(1, sizeof(SParseContext)); *pCxt = taosMemoryCalloc(1, sizeof(SParseContext));
@ -1167,12 +1147,15 @@ int32_t createParseContext(const SRequestObj *pRequest, SParseContext **pCxt) {
.pTransporter = pTscObj->pAppInfo->pTransporter, .pTransporter = pTscObj->pAppInfo->pTransporter,
.pStmtCb = NULL, .pStmtCb = NULL,
.pUser = pTscObj->user, .pUser = pTscObj->user,
.pEffectiveUser = pRequest->effectiveUser,
.isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)), .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER)),
.enableSysInfo = pTscObj->sysInfo, .enableSysInfo = pTscObj->sysInfo,
.async = true, .async = true,
.svrVer = pTscObj->sVer, .svrVer = pTscObj->sVer,
.nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes), .nodeOffline = (pTscObj->pAppInfo->onlineDnodes < pTscObj->pAppInfo->totalDnodes),
.allocatorId = pRequest->allocatorRefId}; .allocatorId = pRequest->allocatorRefId,
.parseSqlFp = clientParseSql,
.parseSqlParam = pWrapper};
int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode); int8_t biMode = atomic_load_8(&((STscObj *)pTscObj)->biMode);
(*pCxt)->biMode = biMode; (*pCxt)->biMode = biMode;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1191,7 +1174,7 @@ int32_t prepareAndParseSqlSyntax(SSqlCallbackWrapper **ppWrapper, SRequestObj *p
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createParseContext(pRequest, &pWrapper->pParseCtx); code = createParseContext(pRequest, &pWrapper->pParseCtx, pWrapper);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1570,7 +1553,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
tsem_wait(&pParam->sem); tsem_wait(&pParam->sem);
_return: _return:
taosArrayDestroyEx(catalogReq.pTableMeta, destoryTablesReq); destoryCatalogReq(&catalogReq);
destroyRequest(pRequest); destroyRequest(pRequest);
return code; return code;
} }
@ -1860,4 +1843,4 @@ int taos_set_conn_mode(TAOS* taos, int mode, int value) {
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
return 0; return 0;
} }

2
source/client/src/clientMsgHandler.c
View File

@ -35,7 +35,7 @@ int32_t genericRspCallback(void* param, SDataBuf* pMsg, int32_t code) {
setErrno(pRequest, code); setErrno(pRequest, code);
if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) { if (NEED_CLIENT_RM_TBLMETA_REQ(pRequest->type)) {
removeMeta(pRequest->pTscObj, pRequest->targetTableList); removeMeta(pRequest->pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
} }
taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pMsg->pEpSet);

351
source/client/src/clientRawBlockWrite.c
View File

File diff suppressed because it is too large Load Diff

4
source/client/src/clientSml.c
View File

@ -104,9 +104,9 @@ static int32_t smlCheckAuth(SSmlHandle *info, SRequestConnInfo* conn, const cha
SUserAuthRes authRes = {0}; SUserAuthRes authRes = {0};
code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes); code = catalogChkAuth(info->pCatalog, conn, &pAuth, &authRes);
nodesDestroyNode(authRes.pCond); nodesDestroyNode(authRes.pCond[AUTH_RES_BASIC]);
return (code == TSDB_CODE_SUCCESS) ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; return (code == TSDB_CODE_SUCCESS) ? (authRes.pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
} }
inline bool smlDoubleToInt64OverFlow(double num) { inline bool smlDoubleToInt64OverFlow(double num) {

2
source/client/src/clientTmq.c
View File

@ -1451,7 +1451,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp)
bool set = false; bool set = false;
int32_t topicNumGet = taosArrayGetSize(pRsp->topics); int32_t topicNumGet = taosArrayGetSize(pRsp->topics);
if (topicNumGet <= 0 && epoch <= tmq->epoch) { if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) {
tscInfo("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tscInfo("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d",
tmq->consumerId, tmq->epoch, epoch, topicNumGet); tmq->consumerId, tmq->epoch, epoch, topicNumGet);
return false; return false;

17
source/common/src/systable.c
View File

@ -24,6 +24,7 @@
#define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_TABLE_NAME_LEN ((TSDB_TABLE_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_DB_NAME_LEN ((TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE) #define SYSTABLE_SCH_COL_NAME_LEN ((TSDB_COL_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
#define SYSTABLE_SCH_VIEW_NAME_LEN ((TSDB_VIEW_NAME_LEN - 1) + VARSTR_HEADER_SIZE)
// clang-format off // clang-format off
static const SSysDbTableSchema dnodesSchema[] = { static const SSysDbTableSchema dnodesSchema[] = {
@ -315,8 +316,23 @@ static const SSysDbTableSchema userUserPrivilegesSchema[] = {
{.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "notes", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
}; };
static const SSysDbTableSchema userViewsSchema[] = {
{.name = "view_name", .bytes = SYSTABLE_SCH_VIEW_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "effective_user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "type", .bytes = 128 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "query_sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "parameters", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "default_values", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target_table", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
// {.name = "column_list", .bytes = 2048 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysTableMeta infosMeta[] = { static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true}, {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true},
{TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true}, {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true},
@ -343,6 +359,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false}, {TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false},
{TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true}, {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true}, {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
{TSDB_INS_TABLE_VIEWS, userViewsSchema, tListLen(userViewsSchema), false},
}; };
static const SSysDbTableSchema connectionsSchema[] = { static const SSysDbTableSchema connectionsSchema[] = {

53
source/common/src/tglobal.c
View File

@ -254,7 +254,8 @@ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch
// internal // internal
int32_t tsTransPullupInterval = 2; int32_t tsTransPullupInterval = 2;
int32_t tsMqRebalanceInterval = 2; int32_t tsMqRebalanceInterval = 2;
int32_t tsStreamCheckpointTickInterval = 300; int32_t tsStreamCheckpointInterval = 60;
float tsSinkDataRate = 2.0;
int32_t tsStreamNodeCheckInterval = 30; int32_t tsStreamNodeCheckInterval = 30;
int32_t tsTtlUnit = 86400; int32_t tsTtlUnit = 86400;
int32_t tsTtlPushIntervalSec = 10; int32_t tsTtlPushIntervalSec = 10;
@ -282,8 +283,6 @@ char tsS3Hostname[TSDB_FQDN_LEN] = "<hostname>";
int32_t tsS3BlockSize = 4096; // number of tsdb pages int32_t tsS3BlockSize = 4096; // number of tsdb pages
int32_t tsS3BlockCacheSize = 16; // number of blocks int32_t tsS3BlockCacheSize = 16; // number of blocks
int32_t tsCheckpointInterval = 300;
#ifndef _STORAGE #ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) { int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir"); SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -512,11 +511,11 @@ static int32_t taosAddSystemCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "SSE42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "AVX", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "AVX2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "FMA", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "SIMD-builtins", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1;
if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1;
@ -673,18 +672,18 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointTickInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1;
return -1;
if (cfgAddString(pCfg, "LossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddFloat(pCfg, "FPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddDouble(pCfg, "DPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "MaxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "CurRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "IfAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddString(pCfg, "Compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1;
@ -1046,7 +1045,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32;
tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64;
tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "SIMD-builtins")->bval; tsSIMDBuiltins = (bool)cfgGetItem(pCfg, "simdEnable")->bval;
tsTagFilterCache = (bool)cfgGetItem(pCfg, "tagFilterCache")->bval; tsTagFilterCache = (bool)cfgGetItem(pCfg, "tagFilterCache")->bval;
tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval;
@ -1105,16 +1104,18 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32; tsCacheLazyLoadThreshold = cfgGetItem(pCfg, "cacheLazyLoadThreshold")->i32;
tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "LossyColumns")->str, sizeof(tsLossyColumns)); tstrncpy(tsLossyColumns, cfgGetItem(pCfg, "lossyColumns")->str, sizeof(tsLossyColumns));
tsFPrecision = cfgGetItem(pCfg, "FPrecision")->fval; tsFPrecision = cfgGetItem(pCfg, "fPrecision")->fval;
tsDPrecision = cfgGetItem(pCfg, "DPrecision")->dval; tsDPrecision = cfgGetItem(pCfg, "dPrecision")->fval;
tsMaxRange = cfgGetItem(pCfg, "MaxRange")->i32; tsMaxRange = cfgGetItem(pCfg, "maxRange")->i32;
tsCurRange = cfgGetItem(pCfg, "CurRange")->i32; tsCurRange = cfgGetItem(pCfg, "curRange")->i32;
tsIfAdtFse = cfgGetItem(pCfg, "IfAdtFse")->bval; tsIfAdtFse = cfgGetItem(pCfg, "ifAdtFse")->bval;
tstrncpy(tsCompressor, cfgGetItem(pCfg, "Compressor")->str, sizeof(tsCompressor)); tstrncpy(tsCompressor, cfgGetItem(pCfg, "compressor")->str, sizeof(tsCompressor));
tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval; tsDisableStream = cfgGetItem(pCfg, "disableStream")->bval;
tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64; tsStreamBufferSize = cfgGetItem(pCfg, "streamBufferSize")->i64;
tsStreamCheckpointInterval = cfgGetItem(pCfg, "checkpointInterval")->i32;
tsSinkDataRate = cfgGetItem(pCfg, "streamSinkDataRate")->fval;
tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval; tsFilterScalarMode = cfgGetItem(pCfg, "filterScalarMode")->bval;
tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32; tsMaxStreamBackendCache = cfgGetItem(pCfg, "maxStreamBackendCache")->i32;

476
source/common/src/tmsg.c
View File

@ -1677,6 +1677,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
if (tEncodeI8(&encoder, pReq->superUser) < 0) return -1; if (tEncodeI8(&encoder, pReq->superUser) < 0) return -1;
if (tEncodeI8(&encoder, pReq->sysInfo) < 0) return -1; if (tEncodeI8(&encoder, pReq->sysInfo) < 0) return -1;
if (tEncodeI8(&encoder, pReq->enable) < 0) return -1; if (tEncodeI8(&encoder, pReq->enable) < 0) return -1;
if (tEncodeI8(&encoder, pReq->isView) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1; if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1;
@ -1691,6 +1692,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq)
if (tEncodeU32(&encoder, pReq->pIpRanges[i].ip) < 0) return -1; if (tEncodeU32(&encoder, pReq->pIpRanges[i].ip) < 0) return -1;
if (tEncodeU32(&encoder, pReq->pIpRanges[i].mask) < 0) return -1; if (tEncodeU32(&encoder, pReq->pIpRanges[i].mask) < 0) return -1;
} }
if (tEncodeI64(&encoder, pReq->privileges) < 0) return -1;
ENCODESQL(); ENCODESQL();
tEndEncode(&encoder); tEndEncode(&encoder);
@ -1708,6 +1710,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
if (tDecodeI8(&decoder, &pReq->superUser) < 0) return -1; if (tDecodeI8(&decoder, &pReq->superUser) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->sysInfo) < 0) return -1; if (tDecodeI8(&decoder, &pReq->sysInfo) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1; if (tDecodeI8(&decoder, &pReq->enable) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->isView) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1;
@ -1728,6 +1731,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq
if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].ip)) < 0) return -1; if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].ip)) < 0) return -1;
if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].mask)) < 0) return -1; if (tDecodeU32(&decoder, &(pReq->pIpRanges[i].mask)) < 0) return -1;
} }
if (tDecodeI64(&decoder, &pReq->privileges) < 0) return -1;
DECODESQL(); DECODESQL();
tEndDecode(&decoder); tEndDecode(&decoder);
@ -1802,10 +1806,18 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
int32_t numOfReadTbs = taosHashGetSize(pRsp->readTbs); int32_t numOfReadTbs = taosHashGetSize(pRsp->readTbs);
int32_t numOfWriteTbs = taosHashGetSize(pRsp->writeTbs); int32_t numOfWriteTbs = taosHashGetSize(pRsp->writeTbs);
int32_t numOfUseTbs = taosHashGetSize(pRsp->useDbs); int32_t numOfAlterTbs = taosHashGetSize(pRsp->alterTbs);
int32_t numOfReadViews = taosHashGetSize(pRsp->readViews);
int32_t numOfWriteViews = taosHashGetSize(pRsp->writeViews);
int32_t numOfAlterViews = taosHashGetSize(pRsp->alterViews);
int32_t numOfUseDbs = taosHashGetSize(pRsp->useDbs);
if (tEncodeI32(pEncoder, numOfReadTbs) < 0) return -1; if (tEncodeI32(pEncoder, numOfReadTbs) < 0) return -1;
if (tEncodeI32(pEncoder, numOfWriteTbs) < 0) return -1; if (tEncodeI32(pEncoder, numOfWriteTbs) < 0) return -1;
if (tEncodeI32(pEncoder, numOfUseTbs) < 0) return -1; if (tEncodeI32(pEncoder, numOfAlterTbs) < 0) return -1;
if (tEncodeI32(pEncoder, numOfReadViews) < 0) return -1;
if (tEncodeI32(pEncoder, numOfWriteViews) < 0) return -1;
if (tEncodeI32(pEncoder, numOfAlterViews) < 0) return -1;
if (tEncodeI32(pEncoder, numOfUseDbs) < 0) return -1;
char *tb = taosHashIterate(pRsp->readTbs, NULL); char *tb = taosHashIterate(pRsp->readTbs, NULL);
while (tb != NULL) { while (tb != NULL) {
@ -1837,6 +1849,66 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
tb = taosHashIterate(pRsp->writeTbs, tb); tb = taosHashIterate(pRsp->writeTbs, tb);
} }
tb = taosHashIterate(pRsp->alterTbs, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
size_t valueLen = 0;
valueLen = strlen(tb);
if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
if (tEncodeCStr(pEncoder, tb) < 0) return -1;
tb = taosHashIterate(pRsp->alterTbs, tb);
}
tb = taosHashIterate(pRsp->readViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
size_t valueLen = 0;
valueLen = strlen(tb);
if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
if (tEncodeCStr(pEncoder, tb) < 0) return -1;
tb = taosHashIterate(pRsp->readViews, tb);
}
tb = taosHashIterate(pRsp->writeViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
size_t valueLen = 0;
valueLen = strlen(tb);
if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
if (tEncodeCStr(pEncoder, tb) < 0) return -1;
tb = taosHashIterate(pRsp->writeViews, tb);
}
tb = taosHashIterate(pRsp->alterViews, NULL);
while (tb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(tb, &keyLen);
if (tEncodeI32(pEncoder, keyLen) < 0) return -1;
if (tEncodeCStr(pEncoder, key) < 0) return -1;
size_t valueLen = 0;
valueLen = strlen(tb);
if (tEncodeI32(pEncoder, valueLen) < 0) return -1;
if (tEncodeCStr(pEncoder, tb) < 0) return -1;
tb = taosHashIterate(pRsp->alterViews, tb);
}
int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL);
while (useDb != NULL) { while (useDb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
@ -1876,9 +1948,14 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->readTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->readTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->writeTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->writeTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->alterTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->readViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->writeViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->alterViews = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL || if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL ||
pRsp->writeTbs == NULL || pRsp->useDbs == NULL) { pRsp->writeTbs == NULL || pRsp->alterTbs == NULL || pRsp->readViews == NULL ||
pRsp->writeViews == NULL || pRsp->alterViews == NULL ||pRsp->useDbs == NULL) {
goto _err; goto _err;
} }
@ -1900,29 +1977,37 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; if (tDecodeCStrTo(pDecoder, db) < 0) goto _err;
int32_t len = strlen(db); int32_t len = strlen(db);
taosHashPut(pRsp->createdDbs, db, len, db, len + 1); taosHashPut(pRsp->createdDbs, db, len + 1, db, len + 1);
} }
for (int32_t i = 0; i < numOfReadDbs; ++i) { for (int32_t i = 0; i < numOfReadDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; if (tDecodeCStrTo(pDecoder, db) < 0) goto _err;
int32_t len = strlen(db); int32_t len = strlen(db);
taosHashPut(pRsp->readDbs, db, len, db, len + 1); taosHashPut(pRsp->readDbs, db, len + 1, db, len + 1);
} }
for (int32_t i = 0; i < numOfWriteDbs; ++i) { for (int32_t i = 0; i < numOfWriteDbs; ++i) {
char db[TSDB_DB_FNAME_LEN] = {0}; char db[TSDB_DB_FNAME_LEN] = {0};
if (tDecodeCStrTo(pDecoder, db) < 0) goto _err; if (tDecodeCStrTo(pDecoder, db) < 0) goto _err;
int32_t len = strlen(db); int32_t len = strlen(db);
taosHashPut(pRsp->writeDbs, db, len, db, len + 1); taosHashPut(pRsp->writeDbs, db, len + 1, db, len + 1);
} }
if (!tDecodeIsEnd(pDecoder)) { if (!tDecodeIsEnd(pDecoder)) {
int32_t numOfReadTbs = 0; int32_t numOfReadTbs = 0;
int32_t numOfWriteTbs = 0; int32_t numOfWriteTbs = 0;
int32_t numOfAlterTbs = 0;
int32_t numOfReadViews = 0;
int32_t numOfWriteViews = 0;
int32_t numOfAlterViews = 0;
int32_t numOfUseDbs = 0; int32_t numOfUseDbs = 0;
if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) goto _err; if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) goto _err;
if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) goto _err; if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) goto _err;
if (tDecodeI32(pDecoder, &numOfAlterTbs) < 0) goto _err;
if (tDecodeI32(pDecoder, &numOfReadViews) < 0) goto _err;
if (tDecodeI32(pDecoder, &numOfWriteViews) < 0) goto _err;
if (tDecodeI32(pDecoder, &numOfAlterViews) < 0) goto _err;
if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) goto _err; if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) goto _err;
for (int32_t i = 0; i < numOfReadTbs; ++i) { for (int32_t i = 0; i < numOfReadTbs; ++i) {
@ -1938,7 +2023,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
value = taosMemoryCalloc(valuelen + 1, sizeof(char)); value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) goto _err; if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
taosHashPut(pRsp->readTbs, key, strlen(key), value, valuelen + 1); taosHashPut(pRsp->readTbs, key, keyLen, value, valuelen + 1);
taosMemoryFreeClear(key); taosMemoryFreeClear(key);
taosMemoryFreeClear(value); taosMemoryFreeClear(value);
@ -1957,7 +2042,83 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
value = taosMemoryCalloc(valuelen + 1, sizeof(char)); value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) goto _err; if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
taosHashPut(pRsp->writeTbs, key, strlen(key), value, valuelen + 1); taosHashPut(pRsp->writeTbs, key, keyLen, value, valuelen + 1);
taosMemoryFreeClear(key);
taosMemoryFreeClear(value);
}
for (int32_t i = 0; i < numOfAlterTbs; ++i) {
int32_t keyLen = 0;
if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
key = taosMemoryCalloc(keyLen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
int32_t valuelen = 0;
if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
taosHashPut(pRsp->alterTbs, key, keyLen, value, valuelen + 1);
taosMemoryFreeClear(key);
taosMemoryFreeClear(value);
}
for (int32_t i = 0; i < numOfReadViews; ++i) {
int32_t keyLen = 0;
if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
key = taosMemoryCalloc(keyLen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
int32_t valuelen = 0;
if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
taosHashPut(pRsp->readViews, key, keyLen, value, valuelen + 1);
taosMemoryFreeClear(key);
taosMemoryFreeClear(value);
}
for (int32_t i = 0; i < numOfWriteViews; ++i) {
int32_t keyLen = 0;
if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
key = taosMemoryCalloc(keyLen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
int32_t valuelen = 0;
if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
taosHashPut(pRsp->writeViews, key, keyLen, value, valuelen + 1);
taosMemoryFreeClear(key);
taosMemoryFreeClear(value);
}
for (int32_t i = 0; i < numOfAlterViews; ++i) {
int32_t keyLen = 0;
if (tDecodeI32(pDecoder, &keyLen) < 0) goto _err;
key = taosMemoryCalloc(keyLen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, key) < 0) goto _err;
int32_t valuelen = 0;
if (tDecodeI32(pDecoder, &valuelen) < 0) goto _err;
value = taosMemoryCalloc(valuelen + 1, sizeof(char));
if (tDecodeCStrTo(pDecoder, value) < 0) goto _err;
taosHashPut(pRsp->alterViews, key, keyLen, value, valuelen + 1);
taosMemoryFreeClear(key); taosMemoryFreeClear(key);
taosMemoryFreeClear(value); taosMemoryFreeClear(value);
@ -1973,7 +2134,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
int32_t ref = 0; int32_t ref = 0;
if (tDecodeI32(pDecoder, &ref) < 0) goto _err; if (tDecodeI32(pDecoder, &ref) < 0) goto _err;
taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); taosHashPut(pRsp->useDbs, key, keyLen, &ref, sizeof(ref));
taosMemoryFreeClear(key); taosMemoryFreeClear(key);
} }
// since 3.0.7.0 // since 3.0.7.0
@ -1993,8 +2154,12 @@ _err:
taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->createdDbs);
taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->readDbs);
taosHashCleanup(pRsp->writeDbs); taosHashCleanup(pRsp->writeDbs);
taosHashCleanup(pRsp->writeTbs);
taosHashCleanup(pRsp->readTbs); taosHashCleanup(pRsp->readTbs);
taosHashCleanup(pRsp->writeTbs);
taosHashCleanup(pRsp->alterTbs);
taosHashCleanup(pRsp->readViews);
taosHashCleanup(pRsp->writeViews);
taosHashCleanup(pRsp->alterViews);
taosHashCleanup(pRsp->useDbs); taosHashCleanup(pRsp->useDbs);
taosMemoryFreeClear(key); taosMemoryFreeClear(key);
@ -2020,8 +2185,12 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) {
taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->createdDbs);
taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->readDbs);
taosHashCleanup(pRsp->writeDbs); taosHashCleanup(pRsp->writeDbs);
taosHashCleanup(pRsp->writeTbs);
taosHashCleanup(pRsp->readTbs); taosHashCleanup(pRsp->readTbs);
taosHashCleanup(pRsp->writeTbs);
taosHashCleanup(pRsp->alterTbs);
taosHashCleanup(pRsp->readViews);
taosHashCleanup(pRsp->writeViews);
taosHashCleanup(pRsp->alterViews);
taosHashCleanup(pRsp->useDbs); taosHashCleanup(pRsp->useDbs);
} }
@ -8405,7 +8574,7 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) {
} else { } else {
tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); tDestroySVCreateTbReq(pTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE);
} }
taosMemoryFree(pTbData->pCreateTbReq); taosMemoryFreeClear(pTbData->pCreateTbReq);
} }
if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { if (pTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
@ -8634,3 +8803,286 @@ void tDeleteMqSubTopicEp(SMqSubTopicEp *pSubTopicEp) {
pSubTopicEp->schema.nCols = 0; pSubTopicEp->schema.nCols = 0;
taosArrayDestroy(pSubTopicEp->vgs); taosArrayDestroy(pSubTopicEp->vgs);
} }
int32_t tSerializeSCMCreateViewReq(void *buf, int32_t bufLen, const SCMCreateViewReq *pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->fullname) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->querySql) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1;
if (tEncodeI8(&encoder, pReq->precision) < 0) return -1;
if (tEncodeI32(&encoder, pReq->numOfCols) < 0) return -1;
for (int32_t i = 0; i < pReq->numOfCols; ++i) {
SSchema *pSchema = &pReq->pSchema[i];
if (tEncodeSSchema(&encoder, pSchema) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCMCreateViewReq(void *buf, int32_t bufLen, SCMCreateViewReq *pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->fullname) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1;
if (tDecodeCStrAlloc(&decoder, &pReq->querySql) < 0) return -1;
if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->numOfCols) < 0) return -1;
if (pReq->numOfCols > 0) {
pReq->pSchema = taosMemoryCalloc(pReq->numOfCols, sizeof(SSchema));
if (pReq->pSchema == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < pReq->numOfCols; ++i) {
SSchema* pSchema = pReq->pSchema + i;
if (tDecodeSSchema(&decoder, pSchema) < 0) return -1;
}
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSCMCreateViewReq(SCMCreateViewReq* pReq) {
if (NULL == pReq) {
return;
}
taosMemoryFreeClear(pReq->querySql);
taosMemoryFreeClear(pReq->sql);
taosMemoryFreeClear(pReq->pSchema);
}
int32_t tSerializeSCMDropViewReq(void* buf, int32_t bufLen, const SCMDropViewReq* pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->fullname) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSCMDropViewReq(void* buf, int32_t bufLen, SCMDropViewReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->fullname) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1;
if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSCMDropViewReq(SCMDropViewReq* pReq) {
if (NULL == pReq) {
return;
}
taosMemoryFree(pReq->sql);
}
int32_t tSerializeSViewMetaReq(void* buf, int32_t bufLen, const SViewMetaReq* pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->fullname) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSViewMetaReq(void* buf, int32_t bufLen, SViewMetaReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->fullname) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
static int32_t tEncodeSViewMetaRsp(SEncoder *pEncoder, const SViewMetaRsp *pRsp) {
if (tEncodeCStr(pEncoder, pRsp->name) < 0) return -1;
if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1;
if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1;
if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1;
if (tEncodeU64(pEncoder, pRsp->viewId) < 0) return -1;
if (tEncodeCStr(pEncoder, pRsp->querySql) < 0) return -1;
if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1;
if (tEncodeI8(pEncoder, pRsp->type) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->numOfCols) < 0) return -1;
for (int32_t i = 0; i < pRsp->numOfCols; ++i) {
SSchema *pSchema = &pRsp->pSchema[i];
if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
}
return 0;
}
int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeSViewMetaRsp(&encoder, pRsp) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
static int32_t tDecodeSViewMetaRsp(SDecoder *pDecoder, SViewMetaRsp *pRsp) {
if (tDecodeCStrTo(pDecoder, pRsp->name) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pRsp->user) < 0) return -1;
if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1;
if (tDecodeU64(pDecoder, &pRsp->viewId) < 0) return -1;
if (tDecodeCStrAlloc(pDecoder, &pRsp->querySql) < 0) return -1;
if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1;
if (tDecodeI8(pDecoder, &pRsp->type) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->numOfCols) < 0) return -1;
if (pRsp->numOfCols > 0) {
pRsp->pSchema = taosMemoryCalloc(pRsp->numOfCols, sizeof(SSchema));
if (pRsp->pSchema == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < pRsp->numOfCols; ++i) {
SSchema* pSchema = pRsp->pSchema + i;
if (tDecodeSSchema(pDecoder, pSchema) < 0) return -1;
}
}
return 0;
}
int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeSViewMetaRsp(&decoder, pRsp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSViewMetaRsp(SViewMetaRsp* pRsp) {
if (NULL == pRsp) {
return;
}
taosMemoryFree(pRsp->user);
taosMemoryFree(pRsp->querySql);
taosMemoryFree(pRsp->pSchema);
}
int32_t tSerializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
int32_t numOfMeta = taosArrayGetSize(pRsp->pViewRsp);
if (tEncodeI32(&encoder, numOfMeta) < 0) return -1;
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp *pMetaRsp = taosArrayGetP(pRsp->pViewRsp, i);
if (tEncodeSViewMetaRsp(&encoder, pMetaRsp) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSViewHbRsp(void *buf, int32_t bufLen, SViewHbRsp *pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
int32_t numOfMeta = 0;
if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1;
pRsp->pViewRsp = taosArrayInit(numOfMeta, POINTER_BYTES);
if (pRsp->pViewRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp* metaRsp = taosMemoryCalloc(1, sizeof(SViewMetaRsp));
if (NULL == metaRsp) return -1;
if (tDecodeSViewMetaRsp(&decoder, metaRsp) < 0) return -1;
taosArrayPush(pRsp->pViewRsp, &metaRsp);
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSViewHbRsp(SViewHbRsp *pRsp) {
int32_t numOfMeta = taosArrayGetSize(pRsp->pViewRsp);
for (int32_t i = 0; i < numOfMeta; ++i) {
SViewMetaRsp *pMetaRsp = taosArrayGetP(pRsp->pViewRsp, i);
tFreeSViewMetaRsp(pMetaRsp);
taosMemoryFree(pMetaRsp);
}
taosArrayDestroy(pRsp->pViewRsp);
}

3
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -189,6 +189,9 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RESTORE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RESTORE_DNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_VIEW, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_VIEW_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER;

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -137,7 +137,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
SRetention *pRetention = &pCfg->tsdbCfg.retentions[i]; SRetention *pRetention = &pCfg->tsdbCfg.retentions[i];
memcpy(pRetention, taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention)); memcpy(pRetention, taosArrayGet(pCreate->pRetensions, i), sizeof(SRetention));
if (i == 0) { if (i == 0) {
if ((pRetention->freq > 0 && pRetention->keep > 0)) pCfg->isRsma = 1; if ((pRetention->freq >= 0 && pRetention->keep > 0)) pCfg->isRsma = 1;
} }
} }

8
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -97,11 +97,7 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper); void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode); int32_t dmInitVars(SDnode *pDnode);
void dmClearVars(SDnode *pDnode); void dmClearVars(SDnode *pDnode);
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers); int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitModule(SDnode *pDnode);
#endif
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper); bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper); SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
void dmSetStatus(SDnode *pDnode, EDndRunStatus stype); void dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
@ -123,11 +119,7 @@ int32_t dmInitStatusClient(SDnode *pDnode);
void dmCleanupClient(SDnode *pDnode); void dmCleanupClient(SDnode *pDnode);
void dmCleanupStatusClient(SDnode *pDnode); void dmCleanupStatusClient(SDnode *pDnode);
SMsgCb dmGetMsgcb(SDnode *pDnode); SMsgCb dmGetMsgcb(SDnode *pDnode);
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers); int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitMsgHandle(SDnode *pDnode);
#endif
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c // dmMonitor.c

8
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -66,15 +66,9 @@ int32_t dmInitDnode(SDnode *pDnode) {
goto _OVER; goto _OVER;
} }
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
if (dmInitModule(pDnode, pDnode->wrappers) != 0) { if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
goto _OVER; goto _OVER;
} }
#else
if (dmInitModule(pDnode) != 0) {
goto _OVER;
}
#endif
indexInit(tsNumOfCommitThreads); indexInit(tsNumOfCommitThreads);
streamMetaInit(); streamMetaInit();
@ -113,7 +107,6 @@ void dmCleanupDnode(SDnode *pDnode) {
dDebug("dnode is closed, ptr:%p", pDnode); dDebug("dnode is closed, ptr:%p", pDnode);
} }
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitVars(SDnode *pDnode) { int32_t dmInitVars(SDnode *pDnode) {
SDnodeData *pData = &pDnode->data; SDnodeData *pData = &pDnode->data;
pData->dnodeId = 0; pData->dnodeId = 0;
@ -182,7 +175,6 @@ void dmClearVars(SDnode *pDnode) {
taosThreadMutexDestroy(&pDnode->mutex); taosThreadMutexDestroy(&pDnode->mutex);
memset(&pDnode->mutex, 0, sizeof(pDnode->mutex)); memset(&pDnode->mutex, 0, sizeof(pDnode->mutex));
} }
#endif
void dmSetStatus(SDnode *pDnode, EDndRunStatus status) { void dmSetStatus(SDnode *pDnode, EDndRunStatus status) {
if (pDnode->status != status) { if (pDnode->status != status) {

28
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -251,7 +251,6 @@ _OVER:
dmReleaseWrapper(pWrapper); dmReleaseWrapper(pWrapper);
} }
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) { int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
SDnodeTrans *pTrans = &pDnode->trans; SDnodeTrans *pTrans = &pDnode->trans;
@ -277,33 +276,6 @@ int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
return 0; return 0;
} }
#else
int32_t dmInitMsgHandle(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
SArray *pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
}
if (!pMgmt->needCheckVgId) {
pHandle->defaultNtype = ntype;
}
pWrapper->msgFps[TMSG_INDEX(pMgmt->msgType)] = pMgmt->msgFp;
}
taosArrayDestroy(pArray);
}
return 0;
}
#endif
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance(); SDnode *pDnode = dmInstance();

1
source/dnode/mnode/impl/CMakeLists.txt
View File

@ -7,6 +7,7 @@ IF (TD_ENTERPRISE)
LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c)
LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndVgroup.c) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndVgroup.c)
LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDnode.c) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDnode.c)
LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/mndView.c)
ENDIF () ENDIF ()
add_library(mnode STATIC ${MNODE_SRC}) add_library(mnode STATIC ${MNODE_SRC})

34
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -74,6 +74,8 @@ typedef enum {
MND_OPER_SUBSCRIBE, MND_OPER_SUBSCRIBE,
MND_OPER_CREATE_TOPIC, MND_OPER_CREATE_TOPIC,
MND_OPER_DROP_TOPIC, MND_OPER_DROP_TOPIC,
MND_OPER_CREATE_VIEW,
MND_OPER_DROP_VIEW,
} EOperType; } EOperType;
typedef enum { typedef enum {
@ -297,6 +299,10 @@ typedef struct {
SHashObj* topics; SHashObj* topics;
SHashObj* readTbs; SHashObj* readTbs;
SHashObj* writeTbs; SHashObj* writeTbs;
SHashObj* alterTbs;
SHashObj* readViews;
SHashObj* writeViews;
SHashObj* alterViews;
SHashObj* useDbs; SHashObj* useDbs;
SRWLatch lock; SRWLatch lock;
} SUserObj; } SUserObj;
@ -704,6 +710,34 @@ void tFreeStreamObj(SStreamObj* pObj);
// SArray* childInfo; // SArray<SStreamChildEpInfo> // SArray* childInfo; // SArray<SStreamChildEpInfo>
// } SStreamCheckpointObj; // } SStreamCheckpointObj;
#define VIEW_TYPE_UPDATABLE (1 << 0)
#define VIEW_TYPE_MATERIALIZED (1 << 1)
typedef struct {
char fullname[TSDB_VIEW_FNAME_LEN];
char name[TSDB_VIEW_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
char user[TSDB_USER_LEN];
char* querySql;
char* parameters;
void** defaultValues;
char* targetTable;
uint64_t viewId;
uint64_t dbId;
int64_t createdTime;
int32_t version;
int8_t precision;
int8_t type;
int32_t numOfCols;
SSchema* pSchema;
SRWLatch lock;
} SViewObj;
int32_t tEncodeSViewObj(SEncoder* pEncoder, const SViewObj* pObj);
int32_t tDecodeSViewObj(SDecoder* pDecoder, SViewObj* pObj, int32_t sver);
void tFreeSViewObj(SViewObj* pObj);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

1
source/dnode/mnode/impl/inc/mndPrivilege.h
View File

@ -28,6 +28,7 @@ void mndCleanupPrivilege(SMnode *pMnode);
int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType);
int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb);
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname); int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname);
int32_t mndCheckViewPrivilege(SMnode *pMnode, const char *user, EOperType operType, const char *pViewFName);
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic); int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic);
int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName); int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName);
int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname); int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);

1
source/dnode/mnode/impl/inc/mndUser.h
View File

@ -41,6 +41,7 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int3
int32_t *pRspLen); int32_t *pRspLen);
int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db);
int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb); int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb);
int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view);
int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic);
int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew); int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew);

66
source/dnode/mnode/impl/inc/mndView.h Executable file
View File

@ -0,0 +1,66 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_MND_VIEW_H_
#define _TD_MND_VIEW_H_
#include "mndInt.h"
#ifdef __cplusplus
extern "C" {
#endif
int32_t mndInitView(SMnode *pMnode);
void mndCleanupView(SMnode *pMnode);
int32_t mndProcessCreateViewReq(SRpcMsg *pReq);
int32_t mndProcessDropViewReq(SRpcMsg *pReq);
int32_t mndProcessGetViewMetaReq(SRpcMsg *pReq);
#ifdef TD_ENTERPRISE
void initDynViewVersion(void);
SViewObj *mndAcquireView(SMnode *pMnode, char *viewName);
void mndReleaseView(SMnode *pMnode, SViewObj *pView);
SSdbRaw *mndViewActionEncode(SViewObj *pView);
SSdbRow *mndViewActionDecode(SSdbRaw *pRaw);
int32_t mndViewActionInsert(SSdb *pSdb, SViewObj *pView);
int32_t mndViewActionDelete(SSdb *pSdb, SViewObj *pView);
int32_t mndViewActionUpdate(SSdb *pSdb, SViewObj *pOldView, SViewObj *pNewView);
int32_t mndProcessCreateViewReqImpl(SCMCreateViewReq* pCreateView, SRpcMsg *pReq);
int32_t mndProcessDropViewReqImpl(SCMDropViewReq* pDropView, SRpcMsg *pReq);
int32_t mndProcessViewMetaReqImpl(SViewMetaReq* pMetaReq, SRpcMsg *pReq);
int32_t mndRetrieveViewImpl(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
int32_t mndDropViewByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndValidateDynViewVersion(SMnode *pMnode, SDynViewVersion* pReqVer, bool *needCheck, SDynViewVersion** ppRspVer);
int32_t mndValidateViewInfo(SMnode *pMnode, SViewVersion *pViewVersions, int32_t numOfViews, void **ppRsp,
int32_t *pRspLen);
#endif
int32_t mndRetrieveView(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
void mndCancelGetNextView(SMnode *pMnode, void *pIter);
#ifdef __cplusplus
}
#endif
#endif /*_TD_MND_VIEW_H_*/

4
source/dnode/mnode/impl/src/mndDb.c
View File

@ -28,6 +28,7 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#include "mndView.h"
#include "systable.h" #include "systable.h"
#include "tjson.h" #include "tjson.h"
#include "thttp.h" #include "thttp.h"
@ -1262,6 +1263,9 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) {
/*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
/*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/
if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
#ifdef TD_ENTERPRISE
if (mndDropViewByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
#endif
if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
if (mndDropIdxsByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropIdxsByDb(pMnode, pTrans, pDb) != 0) goto _OVER;
if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER;

2
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -397,9 +397,7 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
SDnodeInfo dInfo; SDnodeInfo dInfo;
dInfo.id = pDnode->id; dInfo.id = pDnode->id;
dInfo.ep.port = pDnode->port; dInfo.ep.port = pDnode->port;
#ifdef TD_GRANT_HB_OPTIMIZE
dInfo.offlineReason = pDnode->offlineReason; dInfo.offlineReason = pDnode->offlineReason;
#endif
tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN); tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN); tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN); tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);

4
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -78,7 +78,7 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName); mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }
@ -111,7 +111,7 @@ int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbN
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName); mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }

4
source/dnode/mnode/impl/src/mndMain.c
View File

@ -41,6 +41,7 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#include "mndView.h"
static inline int32_t mndAcquireRpc(SMnode *pMnode) { static inline int32_t mndAcquireRpc(SMnode *pMnode) {
int32_t code = 0; int32_t code = 0;
@ -281,7 +282,7 @@ static void *mndThreadFp(void *param) {
mndCalMqRebalance(pMnode); mndCalMqRebalance(pMnode);
} }
if (sec % tsStreamCheckpointTickInterval == 0) { if (sec % tsStreamCheckpointInterval == 0) {
mndStreamCheckpointTick(pMnode, sec); mndStreamCheckpointTick(pMnode, sec);
} }
@ -447,6 +448,7 @@ static int32_t mndInitSteps(SMnode *pMnode) {
if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1; if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1; if (mndAllocStep(pMnode, "mnode-func", mndInitFunc, mndCleanupFunc) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-view", mndInitView, mndCleanupView) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sdb", mndOpenSdb, NULL) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1; if (mndAllocStep(pMnode, "mnode-profile", mndInitProfile, mndCleanupProfile) != 0) return -1;
if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1; if (mndAllocStep(pMnode, "mnode-show", mndInitShow, mndCleanupShow) != 0) return -1;

4
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -75,7 +75,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == meta) { if (NULL == meta) {
mError("invalid performance schema table name:%s", tbName); mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }
@ -101,7 +101,7 @@ int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *t
STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == pMeta) { if (NULL == pMeta) {
mError("invalid performance schema table name:%s", tbName); mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1; return -1;
} }

46
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -23,6 +23,7 @@
#include "mndShow.h" #include "mndShow.h"
#include "mndStb.h" #include "mndStb.h"
#include "mndUser.h" #include "mndUser.h"
#include "mndView.h"
#include "tglobal.h" #include "tglobal.h"
#include "tversion.h" #include "tversion.h"
#include "audit.h" #include "audit.h"
@ -466,7 +467,9 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL};
SRpcConnInfo connInfo = pMsg->info.conn; SRpcConnInfo connInfo = pMsg->info.conn;
mndUpdateAppInfo(pMnode, pHbReq, &connInfo); if (0 != pHbReq->app.appId) {
mndUpdateAppInfo(pMnode, pHbReq, &connInfo);
}
if (pHbReq->query) { if (pHbReq->query) {
SQueryHbReqBasic *pBasic = pHbReq->query; SQueryHbReqBasic *pBasic = pHbReq->query;
@ -529,6 +532,28 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
return -1; return -1;
} }
#ifdef TD_ENTERPRISE
bool needCheck = true;
int32_t key = HEARTBEAT_KEY_DYN_VIEW;
SDynViewVersion* pDynViewVer = NULL;
SKv* pKv = taosHashGet(pHbReq->info, &key, sizeof(key));
if (NULL != pKv) {
pDynViewVer = pKv->value;
mTrace("recv view dyn ver, bootTs:%" PRId64 ", ver:%" PRIu64, pDynViewVer->svrBootTs, pDynViewVer->dynViewVer);
SDynViewVersion* pRspVer = NULL;
if (0 != mndValidateDynViewVersion(pMnode, pDynViewVer, &needCheck, &pRspVer)) {
return -1;
}
if (needCheck) {
SKv kv1 = {.key = HEARTBEAT_KEY_DYN_VIEW, .valueLen = sizeof(*pDynViewVer), .value = pRspVer};
taosArrayPush(hbRsp.info, &kv1);
mTrace("need to check view ver, lastest bootTs:%" PRId64 ", ver:%" PRIu64, pRspVer->svrBootTs, pRspVer->dynViewVer);
}
}
#endif
void *pIter = taosHashIterate(pHbReq->info, NULL); void *pIter = taosHashIterate(pHbReq->info, NULL);
while (pIter != NULL) { while (pIter != NULL) {
SKv *kv = pIter; SKv *kv = pIter;
@ -564,6 +589,25 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
} }
break; break;
} }
#ifdef TD_ENTERPRISE
case HEARTBEAT_KEY_DYN_VIEW: {
break;
}
case HEARTBEAT_KEY_VIEWINFO: {
if (!needCheck) {
break;
}
void *rspMsg = NULL;
int32_t rspLen = 0;
mndValidateViewInfo(pMnode, kv->value, kv->valueLen / sizeof(SViewVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv1 = {.key = HEARTBEAT_KEY_VIEWINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv1);
}
break;
}
#endif
default: default:
mError("invalid kv key:%d", kv->key); mError("invalid kv key:%d", kv->key);
hbRsp.status = TSDB_CODE_APP_ERROR; hbRsp.status = TSDB_CODE_APP_ERROR;

10
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -569,6 +569,10 @@ static int32_t addSinkTasks(SArray* pTasksList, SMnode* pMnode, SStreamObj* pStr
} }
static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) { static void setSinkTaskUpstreamInfo(SArray* pTasksList, const SStreamTask* pUpstreamTask) {
if (taosArrayGetSize(pTasksList) < SINK_NODE_LEVEL || pUpstreamTask == NULL) {
return;
}
SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL); SArray* pSinkTaskList = taosArrayGetP(pTasksList, SINK_NODE_LEVEL);
for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(pSinkTaskList); ++i) {
SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i); SStreamTask* pSinkTask = taosArrayGetP(pSinkTaskList, i);
@ -628,7 +632,9 @@ static int32_t doScheduleStream(SStreamObj* pStream, SMnode* pMnode, SQueryPlan*
} }
setSinkTaskUpstreamInfo(pStream->tasks, pAggTask); setSinkTaskUpstreamInfo(pStream->tasks, pAggTask);
setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask); if (pHAggTask != NULL) {
setSinkTaskUpstreamInfo(pStream->pHTasksList, pHAggTask);
}
// source level // source level
return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, nextWindowSkey); return addSourceTasksForMultiLevelStream(pMnode, pPlan, pStream, pAggTask, pHAggTask, pEpset, nextWindowSkey);
@ -722,7 +728,7 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
pVgEp->vgId = pVgroup->vgId; pVgEp->vgId = pVgroup->vgId;
taosArrayPush(pSub->unassignedVgs, &pVgEp); taosArrayPush(pSub->unassignedVgs, &pVgEp);
mDebug("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId); mInfo("init subscription %s for topic:%s assign vgId:%d", pSub->key, pTopic->name, pVgEp->vgId);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }

4
source/dnode/mnode/impl/src/mndShow.c
View File

@ -116,6 +116,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_STREAM_TASKS; type = TSDB_MGMT_TABLE_STREAM_TASKS;
} else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) { } else if (strncasecmp(name, TSDB_INS_TABLE_USER_PRIVILEGES, len) == 0) {
type = TSDB_MGMT_TABLE_PRIVILEGES; type = TSDB_MGMT_TABLE_PRIVILEGES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VIEWS, len) == 0) {
type = TSDB_MGMT_TABLE_VIEWS;
} else { } else {
mError("invalid show name:%s len:%d", name, len); mError("invalid show name:%s len:%d", name, len);
} }
@ -211,7 +213,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
if (pMeta == NULL) { if (pMeta == NULL) {
pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); pMeta = taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) { if (pMeta == NULL) {
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
return -1; return -1;
} }

84
source/dnode/mnode/impl/src/mndStb.c
View File

@ -1903,8 +1903,51 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName,
return 0; return 0;
} }
static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, static int32_t mndValidateStbVersion(SMnode *pMnode, SSTableVersion* pStbVer, bool* schema, bool* sma) {
int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(tbFName, sizeof(tbFName), "%s.%s", pStbVer->dbFName, pStbVer->stbName);
SDbObj *pDb = mndAcquireDb(pMnode, pStbVer->dbFName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
}
if (pDb->uid != pStbVer->dbId) {
mndReleaseDb(pMnode, pDb);
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
return -1;
}
SStbObj *pStb = mndAcquireStb(pMnode, tbFName);
if (pStb == NULL) {
mndReleaseDb(pMnode, pDb);
terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
return -1;
}
taosRLockLatch(&pStb->lock);
if (pStbVer->sversion != pStb->colVer || pStbVer->tversion != pStb->tagVer) {
*schema = true;
} else {
*schema = false;
}
if (pStbVer->smaVer && pStbVer->smaVer != pStb->smaVer) {
*sma = true;
} else {
*sma = false;
}
taosRUnLockLatch(&pStb->lock);
mndReleaseDb(pMnode, pDb);
mndReleaseStb(pMnode, pStb);
return TSDB_CODE_SUCCESS;
}
static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) {
char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; char tbFName[TSDB_TABLE_FNAME_LEN] = {0};
snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName);
@ -1921,10 +1964,6 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char
return -1; return -1;
} }
if (smaVer) {
*smaVer = pStb->smaVer;
}
int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp);
mndReleaseDb(pMnode, pDb); mndReleaseDb(pMnode, pDb);
mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pStb);
@ -2563,7 +2602,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) {
} }
} else { } else {
mInfo("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); mInfo("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName);
if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp, NULL) != 0) { if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) {
goto _OVER; goto _OVER;
} }
} }
@ -2674,14 +2713,15 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
for (int32_t i = 0; i < numOfStbs; ++i) { for (int32_t i = 0; i < numOfStbs; ++i) {
SSTableVersion *pStbVersion = &pStbVersions[i]; SSTableVersion *pStbVersion = &pStbVersions[i];
pStbVersion->suid = be64toh(pStbVersion->suid); pStbVersion->suid = be64toh(pStbVersion->suid);
pStbVersion->sversion = ntohs(pStbVersion->sversion); pStbVersion->sversion = ntohl(pStbVersion->sversion);
pStbVersion->tversion = ntohs(pStbVersion->tversion); pStbVersion->tversion = ntohl(pStbVersion->tversion);
pStbVersion->smaVer = ntohl(pStbVersion->smaVer); pStbVersion->smaVer = ntohl(pStbVersion->smaVer);
STableMetaRsp metaRsp = {0}; bool schema = false;
int32_t smaVer = 0; bool sma = false;
mInfo("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); int32_t code = mndValidateStbVersion(pMnode, pStbVersion, &schema, &sma);
if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { if (TSDB_CODE_SUCCESS != code) {
STableMetaRsp metaRsp = {0};
metaRsp.numOfColumns = -1; metaRsp.numOfColumns = -1;
metaRsp.suid = pStbVersion->suid; metaRsp.suid = pStbVersion->suid;
tstrncpy(metaRsp.dbFName, pStbVersion->dbFName, sizeof(metaRsp.dbFName)); tstrncpy(metaRsp.dbFName, pStbVersion->dbFName, sizeof(metaRsp.dbFName));
@ -2691,13 +2731,23 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t
continue; continue;
} }
if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { if (schema) {
STableMetaRsp metaRsp = {0};
mInfo("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName);
if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) {
metaRsp.numOfColumns = -1;
metaRsp.suid = pStbVersion->suid;
tstrncpy(metaRsp.dbFName, pStbVersion->dbFName, sizeof(metaRsp.dbFName));
tstrncpy(metaRsp.tbName, pStbVersion->stbName, sizeof(metaRsp.tbName));
tstrncpy(metaRsp.stbName, pStbVersion->stbName, sizeof(metaRsp.stbName));
taosArrayPush(hbRsp.pMetaRsp, &metaRsp);
continue;
}
taosArrayPush(hbRsp.pMetaRsp, &metaRsp); taosArrayPush(hbRsp.pMetaRsp, &metaRsp);
} else {
tFreeSTableMetaRsp(&metaRsp);
} }
if (pStbVersion->smaVer && pStbVersion->smaVer != smaVer) { if (sma) {
bool exist = false; bool exist = false;
char tbFName[TSDB_TABLE_FNAME_LEN]; char tbFName[TSDB_TABLE_FNAME_LEN];
STableIndexRsp indexRsp = {0}; STableIndexRsp indexRsp = {0};

492
source/dnode/mnode/impl/src/mndStream.c
View File

@ -42,22 +42,22 @@ typedef struct SNodeEntry {
int64_t hbTimestamp; // second int64_t hbTimestamp; // second
} SNodeEntry; } SNodeEntry;
typedef struct SStreamExecNodeInfo { typedef struct SStreamExecInfo {
SArray *pNodeEntryList; SArray *pNodeEntryList;
int64_t ts; // snapshot ts int64_t ts; // snapshot ts
int64_t activeCheckpoint; // active check point id int64_t activeCheckpoint; // active check point id
SHashObj *pTaskMap; SHashObj *pTaskMap;
SArray *pTaskList; SArray *pTaskList;
TdThreadMutex lock; TdThreadMutex lock;
} SStreamExecNodeInfo; } SStreamExecInfo;
typedef struct SVgroupChangeInfo { typedef struct SVgroupChangeInfo {
SHashObj *pDBMap; SHashObj *pDBMap;
SArray *pUpdateNodeList; // SArray<SNodeUpdateInfo> SArray *pUpdateNodeList; // SArray<SNodeUpdateInfo>
} SVgroupChangeInfo; } SVgroupChangeInfo;
static int32_t mndNodeCheckSentinel = 0; static int32_t mndNodeCheckSentinel = 0;
static SStreamExecNodeInfo execNodeList; static SStreamExecInfo execInfo;
static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
@ -77,19 +77,21 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
int64_t streamId, int32_t taskId); int64_t streamId, int32_t taskId);
static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheck(SRpcMsg *pReq);
static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg);
static SArray *extractNodeListFromStream(SMnode *pMnode);
static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady);
static SArray *extractNodeListFromStream(SMnode *pMnode);
static SArray *mndTakeVgroupSnapshot(SMnode *pMnode);
static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList);
static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name); static STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name);
static int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans); static int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans);
static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset); static void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo); int32_t retryCode);
static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans);
static void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode); static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode); static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode);
static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot);
static int32_t doKillActiveCheckpointTrans(SMnode *pMnode);
static int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList);
int32_t mndInitStream(SMnode *pMnode) { int32_t mndInitStream(SMnode *pMnode) {
SSdbTable table = { SSdbTable table = {
@ -129,18 +131,18 @@ int32_t mndInitStream(SMnode *pMnode) {
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndRetrieveStreamTask);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAM_TASKS, mndCancelGetNextStreamTask);
taosThreadMutexInit(&execNodeList.lock, NULL); taosThreadMutexInit(&execInfo.lock, NULL);
execNodeList.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK); execInfo.pTaskMap = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK);
execNodeList.pTaskList = taosArrayInit(4, sizeof(STaskId)); execInfo.pTaskList = taosArrayInit(4, sizeof(STaskId));
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
} }
void mndCleanupStream(SMnode *pMnode) { void mndCleanupStream(SMnode *pMnode) {
taosArrayDestroy(execNodeList.pTaskList); taosArrayDestroy(execInfo.pTaskList);
taosHashCleanup(execNodeList.pTaskMap); taosHashCleanup(execInfo.pTaskMap);
taosThreadMutexDestroy(&execNodeList.lock); taosThreadMutexDestroy(&execInfo.lock);
mDebug("mnd stream cleanup"); mDebug("mnd stream exec info cleanup");
} }
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {
@ -286,7 +288,7 @@ void mndReleaseStream(SMnode *pMnode, SStreamObj *pStream) {
static void mndShowStreamStatus(char *dst, SStreamObj *pStream) { static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
int8_t status = atomic_load_8(&pStream->status); int8_t status = atomic_load_8(&pStream->status);
if (status == STREAM_STATUS__NORMAL) { if (status == STREAM_STATUS__NORMAL) {
strcpy(dst, "normal"); strcpy(dst, "ready");
} else if (status == STREAM_STATUS__STOP) { } else if (status == STREAM_STATUS__STOP) {
strcpy(dst, "stop"); strcpy(dst, "stop");
} else if (status == STREAM_STATUS__FAILED) { } else if (status == STREAM_STATUS__FAILED) {
@ -294,7 +296,7 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
} else if (status == STREAM_STATUS__RECOVER) { } else if (status == STREAM_STATUS__RECOVER) {
strcpy(dst, "recover"); strcpy(dst, "recover");
} else if (status == STREAM_STATUS__PAUSE) { } else if (status == STREAM_STATUS__PAUSE) {
strcpy(dst, "pause"); strcpy(dst, "paused");
} }
} }
@ -516,7 +518,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) {
STransAction action = {0}; STransAction action = {0};
action.mTraceId = pTrans->mTraceId; action.mTraceId = pTrans->mTraceId;
initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet); initTransAction(&action, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(buf); taosMemoryFree(buf);
return -1; return -1;
@ -688,7 +690,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) {
pReq->streamId = pTask->id.streamId; pReq->streamId = pTask->id.streamId;
STransAction action = {0}; STransAction action = {0};
initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet); initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet, 0);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -847,10 +849,10 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
mndTransDrop(pTrans); mndTransDrop(pTrans);
taosThreadMutexLock(&execNodeList.lock); taosThreadMutexLock(&execInfo.lock);
mDebug("register to stream task node list"); mDebug("register to stream task node list");
keepStreamTasksInBuf(&streamObj, &execNodeList); keepStreamTasksInBuf(&streamObj, &execInfo);
taosThreadMutexUnlock(&execNodeList.lock); taosThreadMutexUnlock(&execInfo.lock);
code = TSDB_CODE_ACTION_IN_PROGRESS; code = TSDB_CODE_ACTION_IN_PROGRESS;
@ -882,9 +884,8 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
return 0; return 0;
} }
int64_t checkpointId = taosGetTimestampMs();
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
pMsg->checkpointId = checkpointId; pMsg->checkpointId = taosGetTimestampMs();
int32_t size = sizeof(SMStreamDoCheckpointMsg); int32_t size = sizeof(SMStreamDoCheckpointMsg);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size}; SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@ -936,7 +937,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in
} }
// static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) { // static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStream, int64_t checkpointId) {
// int64_t timestampMs = taosGetTimestampMs(); // int64_t timestampMs = taosGetTimestampMs();
// if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointTickInterval * 1000) { // if (timestampMs - pStream->checkpointFreq < tsStreamCheckpointInterval * 1000) {
// return -1; // return -1;
// } // }
@ -1069,7 +1070,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
STransAction action = {0}; STransAction action = {0};
SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj);
initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset); initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY);
mndReleaseVgroup(pMnode, pVgObj); mndReleaseVgroup(pMnode, pVgObj);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
@ -1084,6 +1085,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream
pStream->checkpointId = chkptId; pStream->checkpointId = chkptId;
pStream->checkpointFreq = taosGetTimestampMs(); pStream->checkpointFreq = taosGetTimestampMs();
pStream->currentTick = 0; pStream->currentTick = 0;
// 3. commit log: stream checkpoint info // 3. commit log: stream checkpoint info
pStream->version = pStream->version + 1; pStream->version = pStream->version + 1;
@ -1133,31 +1135,37 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
{ // check if the node update happens or not { // check if the node update happens or not
int64_t ts = taosGetTimestampSec(); int64_t ts = taosGetTimestampSec();
if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) { if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
if (execNodeList.pNodeEntryList != NULL) { if (execInfo.pNodeEntryList != NULL) {
execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList); execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
} }
execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode); execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
} }
if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) { if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
mDebug("stream task node change checking done, no vgroups exist, do nothing"); mDebug("stream task node change checking done, no vgroups exist, do nothing");
execNodeList.ts = ts; execInfo.ts = ts;
return 0; return 0;
} }
for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, i); SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
if (pNodeEntry->stageUpdated) { if (pNodeEntry->stageUpdated) {
mDebug("stream task not ready due to node update detected, checkpoint not issued"); mDebug("stream task not ready due to node update detected, checkpoint not issued");
return 0; return 0;
} }
} }
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode); bool allReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady);
if (!allReady) {
mWarn("not all vnodes are ready, ignore the checkpoint")
taosArrayDestroy(pNodeSnapshot);
return 0;
}
SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot); SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
bool nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0); bool nodeUpdated = (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0);
taosArrayDestroy(changeInfo.pUpdateNodeList); taosArrayDestroy(changeInfo.pUpdateNodeList);
taosHashCleanup(changeInfo.pDBMap); taosHashCleanup(changeInfo.pDBMap);
@ -1169,26 +1177,25 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
} }
} }
{ // check if all tasks are in TASK_STATUS__NORMAL status { // check if all tasks are in TASK_STATUS__READY status
bool ready = true; bool ready = true;
taosThreadMutexLock(&execNodeList.lock); taosThreadMutexLock(&execInfo.lock);
for (int32_t i = 0; i < taosArrayGetSize(execNodeList.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execNodeList.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, p, sizeof(*p)); STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
if (pEntry == NULL) { if (pEntry == NULL) {
continue; continue;
} }
if (pEntry->status != TASK_STATUS__NORMAL) { if (pEntry->status != TASK_STATUS__READY) {
mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued",
pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamGetTaskStatusStr(pEntry->status)); pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status));
ready = false; ready = false;
break; break;
} }
} }
taosThreadMutexUnlock(&execNodeList.lock); taosThreadMutexUnlock(&execInfo.lock);
if (!ready) { if (!ready) {
return 0; return 0;
} }
@ -1202,7 +1209,8 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to trigger checkpoint, reason: %s", tstrerror(TSDB_CODE_OUT_OF_MEMORY));
return -1; return -1;
} }
mDebug("start to trigger checkpoint, checkpointId: %" PRId64 "", checkpointId);
mDebug("start to trigger checkpoint, checkpointId: %" PRId64, checkpointId);
const char *pDb = mndGetStreamDB(pMnode); const char *pDb = mndGetStreamDB(pMnode);
mndTransSetDbName(pTrans, pDb, "checkpoint"); mndTransSetDbName(pTrans, pDb, "checkpoint");
@ -1228,11 +1236,16 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) {
if (code == 0) { if (code == 0) {
if (mndTransPrepare(pMnode, pTrans) != 0) { if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("failed to prepre trans rebalance since %s", terrstr()); mError("failed to prepare trans rebalance since %s", terrstr());
} }
} }
mndTransDrop(pTrans); mndTransDrop(pTrans);
// only one trans here
taosThreadMutexLock(&execInfo.lock);
execInfo.activeCheckpoint = checkpointId;
taosThreadMutexUnlock(&execInfo.lock);
return code; return code;
} }
@ -1310,7 +1323,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
return -1; return -1;
} }
removeStreamTasksInBuf(pStream, &execNodeList); removeStreamTasksInBuf(pStream, &execInfo);
SName name = {0}; SName name = {0};
tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB); tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB);
@ -1561,12 +1574,12 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
char status[20 + VARSTR_HEADER_SIZE] = {0}; char status[20 + VARSTR_HEADER_SIZE] = {0};
STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId};
STaskStatusEntry* pe = taosHashGet(execNodeList.pTaskMap, &id, sizeof(id)); STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id));
if (pe == NULL) { if (pe == NULL) {
continue; continue;
} }
const char* pStatus = streamGetTaskStatusStr(pe->status); const char* pStatus = streamTaskGetStatusStr(pe->status);
STR_TO_VARSTR(status, pStatus); STR_TO_VARSTR(status, pStatus);
// status // status
@ -1641,7 +1654,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) {
pReq->streamId = pTask->id.streamId; pReq->streamId = pTask->id.streamId;
STransAction action = {0}; STransAction action = {0};
initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet); initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet, 0);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -1774,7 +1787,7 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t ig
pReq->igUntreated = igUntreated; pReq->igUntreated = igUntreated;
STransAction action = {0}; STransAction action = {0};
initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet); initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet, 0);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
return -1; return -1;
@ -1878,18 +1891,19 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return TSDB_CODE_ACTION_IN_PROGRESS; return TSDB_CODE_ACTION_IN_PROGRESS;
} }
static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, int64_t streamId, static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChangeInfo *pInfo, SStreamTaskId *pId,
int32_t taskId) { int32_t transId) {
pMsg->streamId = streamId; pMsg->streamId = pId->streamId;
pMsg->taskId = taskId; pMsg->taskId = pId->taskId;
pMsg->transId = transId;
pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo)); pMsg->pNodeList = taosArrayInit(taosArrayGetSize(pInfo->pUpdateNodeList), sizeof(SNodeUpdateInfo));
taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList); taosArrayAddAll(pMsg->pNodeList, pInfo->pUpdateNodeList);
} }
static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId, static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId,
int64_t streamId, int32_t taskId) { SStreamTaskId* pId, int32_t transId) {
SStreamTaskNodeUpdateMsg req = {0}; SStreamTaskNodeUpdateMsg req = {0};
initNodeUpdateMsg(&req, pInfo, streamId, taskId); initNodeUpdateMsg(&req, pInfo, pId, transId);
int32_t code = 0; int32_t code = 0;
int32_t blen; int32_t blen;
@ -1953,20 +1967,19 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans) {
return 0; return 0;
} }
void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset) { void initTransAction(STransAction *pAction, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset,
int32_t retryCode) {
pAction->epSet = *pEpset; pAction->epSet = *pEpset;
pAction->contLen = contLen; pAction->contLen = contLen;
pAction->pCont = pCont; pAction->pCont = pCont;
pAction->msgType = msgType; pAction->msgType = msgType;
pAction->retryCode = retryCode;
} }
// todo extract method: traverse stream tasks // todo extract method: traverse stream tasks
// build trans to update the epset // build trans to update the epset
static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgroupChangeInfo *pInfo) { static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *pInfo, STrans *pTrans) {
STrans* pTrans = doCreateTrans(pMnode, pStream, "stream-task-update"); mDebug("start to build stream:0x%" PRIx64 " tasks epset update", pStream->uid);
if (pTrans == NULL) {
return terrno;
}
taosWLockLatch(&pStream->lock); taosWLockLatch(&pStream->lock);
int32_t numOfLevels = taosArrayGetSize(pStream->tasks); int32_t numOfLevels = taosArrayGetSize(pStream->tasks);
@ -1981,38 +1994,20 @@ static int32_t createStreamUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SVgr
void *pBuf = NULL; void *pBuf = NULL;
int32_t len = 0; int32_t len = 0;
streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList);
doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, pTask->id.streamId, pTask->id.taskId); doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id);
STransAction action = {0}; STransAction action = {0};
initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet); initTransAction(&action, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &pTask->info.epSet, 0);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pBuf); taosMemoryFree(pBuf);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
mndTransDrop(pTrans);
return -1; return -1;
} }
} }
} }
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
return 0;
int32_t code = mndPersistTransLog(pStream, pTrans);
if (code != TSDB_CODE_SUCCESS) {
sdbRelease(pMnode->pSdb, pStream);
return -1;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return TSDB_CODE_ACTION_IN_PROGRESS;
} }
static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) { static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) {
@ -2050,8 +2045,8 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
char buf[256] = {0}; char buf[256] = {0};
EPSET_TO_STR(&pCurrent->epset, buf); EPSET_TO_STR(&pCurrent->epset, buf);
mDebug("nodeId:%d epset changed detected, old:%s:%d -> new:%s", pCurrent->nodeId, pPrevEp->fqdn, mDebug("nodeId:%d restart/epset changed detected, old:%s:%d -> new:%s, stageUpdate:%d", pCurrent->nodeId,
pPrevEp->port, buf); pPrevEp->fqdn, pPrevEp->port, buf, pPrevEntry->stageUpdated);
SNodeUpdateInfo updateInfo = {.nodeId = pPrevEntry->nodeId}; SNodeUpdateInfo updateInfo = {.nodeId = pPrevEntry->nodeId};
epsetAssign(&updateInfo.prevEp, &pPrevEntry->epset); epsetAssign(&updateInfo.prevEp, &pPrevEntry->epset);
@ -2071,11 +2066,12 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
return info; return info;
} }
static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) { static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
SVgObj *pVgroup = NULL; SVgObj *pVgroup = NULL;
*allReady = true;
SArray *pVgroupListSnapshot = taosArrayInit(4, sizeof(SNodeEntry)); SArray *pVgroupListSnapshot = taosArrayInit(4, sizeof(SNodeEntry));
while (1) { while (1) {
@ -2087,8 +2083,26 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode) {
SNodeEntry entry = {0}; SNodeEntry entry = {0};
entry.epset = mndGetVgroupEpset(pMnode, pVgroup); entry.epset = mndGetVgroupEpset(pMnode, pVgroup);
entry.nodeId = pVgroup->vgId; entry.nodeId = pVgroup->vgId;
entry.hbTimestamp = -1; entry.hbTimestamp = pVgroup->updateTime;
if (*allReady) {
for (int32_t i = 0; i < pVgroup->replica; ++i) {
if (!pVgroup->vnodeGid[i].syncRestore) {
*allReady = false;
break;
}
ESyncState state = pVgroup->vnodeGid[i].syncState;
if (state == TAOS_SYNC_STATE_OFFLINE || state == TAOS_SYNC_STATE_ERROR) {
*allReady = false;
break;
}
}
}
char buf[256] = {0};
EPSET_TO_STR(&entry.epset, buf);
mDebug("take node snapshot, nodeId:%d %s", entry.nodeId, buf);
taosArrayPush(pVgroupListSnapshot, &entry); taosArrayPush(pVgroupListSnapshot, &entry);
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
} }
@ -2102,27 +2116,63 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange
// check all streams that involved this vnode should update the epset info // check all streams that involved this vnode should update the epset info
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
void *pIter = NULL; void *pIter = NULL;
STrans *pTrans = NULL;
while (1) { while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) { if (pIter == NULL) {
break; break;
} }
// here create only one trans
if (pTrans == NULL) {
pTrans = doCreateTrans(pMnode, pStream, "stream-task-update");
if (pTrans == NULL) {
sdbRelease(pSdb, pStream);
sdbCancelFetch(pSdb, pIter);
return terrno;
}
}
void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb)); void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb));
void *p1 = taosHashGet(pChangeInfo->pDBMap, pStream->sourceDb, strlen(pStream->sourceDb)); void *p1 = taosHashGet(pChangeInfo->pDBMap, pStream->sourceDb, strlen(pStream->sourceDb));
if (p == NULL && p1 == NULL) { if (p == NULL && p1 == NULL) {
mndReleaseStream(pMnode, pStream); mDebug("stream:0x%" PRIx64 " %s not involved nodeUpdate, ignore", pStream->uid, pStream->name);
sdbRelease(pSdb, pStream);
continue; continue;
} }
mDebug("stream:0x%" PRIx64 " involved node changed, create update trans", pStream->uid); mDebug("stream:0x%" PRIx64 " %s involved node changed, create update trans, transId:%d", pStream->uid,
int32_t code = createStreamUpdateTrans(pMnode, pStream, pChangeInfo); pStream->name, pTrans->id);
int32_t code = createStreamUpdateTrans(pStream, pChangeInfo, pTrans);
// todo: not continue, drop all and retry again
if (code != TSDB_CODE_SUCCESS) {
mError("stream:0x%" PRIx64 " build nodeUpdate trans failed, ignore and continue, code:%s", pStream->uid,
tstrerror(code));
sdbRelease(pSdb, pStream);
continue;
}
code = mndPersistTransLog(pStream, pTrans);
sdbRelease(pSdb, pStream);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return code; return -1;
} }
} }
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare update stream trans since %s", pTrans->id, terrstr());
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return -1;
}
sdbRelease(pMnode->pSdb, pStream);
mndTransDrop(pTrans);
return 0; return 0;
} }
@ -2165,6 +2215,10 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) {
while ((pIter = taosHashIterate(pHash, pIter)) != NULL) { while ((pIter = taosHashIterate(pHash, pIter)) != NULL) {
SNodeEntry *pEntry = (SNodeEntry *)pIter; SNodeEntry *pEntry = (SNodeEntry *)pIter;
taosArrayPush(plist, pEntry); taosArrayPush(plist, pEntry);
char buf[256] = {0};
EPSET_TO_STR(&pEntry->epset, buf);
mDebug("extract nodeInfo from stream obj, nodeId:%d, %s", pEntry->nodeId, buf);
} }
taosHashCleanup(pHash); taosHashCleanup(pHash);
@ -2182,28 +2236,31 @@ static void doExtractTasksFromStream(SMnode *pMnode) {
break; break;
} }
keepStreamTasksInBuf(pStream, &execNodeList); keepStreamTasksInBuf(pStream, &execInfo);
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
} }
} }
static int32_t doRemoveFromTask(SStreamExecNodeInfo* pExecNode, STaskId* pRemovedId) { static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) {
void *p = taosHashGet(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId)); void *p = taosHashGet(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
if (p == NULL) {
return TSDB_CODE_SUCCESS;
}
if (p != NULL) { taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
taosHashRemove(pExecNode->pTaskMap, pRemovedId, sizeof(*pRemovedId));
for(int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) {
STaskId* pId = taosArrayGet(pExecNode->pTaskList, k); STaskId *pId = taosArrayGet(pExecNode->pTaskList, k);
if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) { if (pId->taskId == pRemovedId->taskId && pId->streamId == pRemovedId->streamId) {
taosArrayRemove(pExecNode->pTaskList, k); taosArrayRemove(pExecNode->pTaskList, k);
mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t) pRemovedId->taskId,
(int32_t)taosArrayGetSize(pExecNode->pTaskList)); int32_t num = taosArrayGetSize(pExecNode->pTaskList);
break; mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)pRemovedId->taskId, num);
} break;
} }
} }
return 0;
return TSDB_CODE_SUCCESS;
} }
static bool taskNodeExists(SArray* pList, int32_t nodeId) { static bool taskNodeExists(SArray* pList, int32_t nodeId) {
@ -2220,31 +2277,31 @@ static bool taskNodeExists(SArray* pList, int32_t nodeId) {
} }
int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
SArray* pRemoveTaskList = taosArrayInit(4, sizeof(STaskId)); SArray* pRemovedTasks = taosArrayInit(4, sizeof(STaskId));
int32_t numOfTask = taosArrayGetSize(execNodeList.pTaskList); int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList);
for(int32_t i = 0; i < numOfTask; ++i) { for(int32_t i = 0; i < numOfTask; ++i) {
STaskId* pId = taosArrayGet(execNodeList.pTaskList, i); STaskId* pId = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry* pEntry = taosHashGet(execNodeList.pTaskMap, pId, sizeof(*pId)); STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId));
bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId);
if (!existed) { if (!existed) {
taosArrayPush(pRemoveTaskList, pId); taosArrayPush(pRemovedTasks, pId);
} }
} }
for(int32_t i = 0; i < taosArrayGetSize(pRemoveTaskList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) {
STaskId* pId = taosArrayGet(pRemoveTaskList, i); STaskId* pId = taosArrayGet(pRemovedTasks, i);
doRemoveFromTask(&execNodeList, pId); doRemoveTasks(&execInfo, pId);
} }
mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemoveTaskList), mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks),
(int32_t) taosArrayGetSize(execNodeList.pTaskList)); (int32_t) taosArrayGetSize(execInfo.pTaskList));
int32_t size = taosArrayGetSize(pNodeSnapshot); int32_t size = taosArrayGetSize(pNodeSnapshot);
SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry));
for(int32_t i = 0; i < taosArrayGetSize(execNodeList.pNodeEntryList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) {
SNodeEntry* p = taosArrayGet(execNodeList.pNodeEntryList, i); SNodeEntry* p = taosArrayGet(execInfo.pNodeEntryList, i);
for(int32_t j = 0; j < size; ++j) { for(int32_t j = 0; j < size; ++j) {
SNodeEntry* pEntry = taosArrayGet(pNodeSnapshot, j); SNodeEntry* pEntry = taosArrayGet(pNodeSnapshot, j);
@ -2255,10 +2312,11 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) {
} }
} }
execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList); execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
execNodeList.pNodeEntryList = pValidNodeEntryList; execInfo.pNodeEntryList = pValidNodeEntryList;
taosArrayDestroy(pRemoveTaskList); mDebug("remain %d valid node entries", (int32_t) taosArrayGetSize(pValidNodeEntryList));
taosArrayDestroy(pRemovedTasks);
return 0; return 0;
} }
@ -2275,43 +2333,55 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) {
int64_t ts = taosGetTimestampSec(); int64_t ts = taosGetTimestampSec();
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
if (execNodeList.pNodeEntryList == NULL || (taosArrayGetSize(execNodeList.pNodeEntryList) == 0)) { if (execInfo.pNodeEntryList == NULL || (taosArrayGetSize(execInfo.pNodeEntryList) == 0)) {
if (execNodeList.pNodeEntryList != NULL) { if (execInfo.pNodeEntryList != NULL) {
execNodeList.pNodeEntryList = taosArrayDestroy(execNodeList.pNodeEntryList); execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList);
} }
execInfo.pNodeEntryList = extractNodeListFromStream(pMnode);
execNodeList.pNodeEntryList = extractNodeListFromStream(pMnode);
} }
if (taosArrayGetSize(execNodeList.pNodeEntryList) == 0) { if (taosArrayGetSize(execInfo.pNodeEntryList) == 0) {
mDebug("end to do stream task node change checking, no vgroup exists, do nothing"); mDebug("end to do stream task node change checking, no vgroup exists, do nothing");
execNodeList.ts = ts; execInfo.ts = ts;
atomic_store_32(&mndNodeCheckSentinel, 0); atomic_store_32(&mndNodeCheckSentinel, 0);
return 0; return 0;
} }
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode); bool allVnodeReady = true;
SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVnodeReady);
if (!allVnodeReady) {
taosArrayDestroy(pNodeSnapshot);
atomic_store_32(&mndNodeCheckSentinel, 0);
mWarn("not all vnodes are ready, ignore the exec nodeUpdate check");
return 0;
}
taosThreadMutexLock(&execNodeList.lock); taosThreadMutexLock(&execInfo.lock);
removeExpirednodeEntryAndTask(pNodeSnapshot); removeExpirednodeEntryAndTask(pNodeSnapshot);
SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execNodeList.pNodeEntryList, pNodeSnapshot); SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot);
if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) { if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) {
// kill current active checkpoint transaction, since the transaction is vnode wide.
doKillActiveCheckpointTrans(pMnode);
code = mndProcessVgroupChange(pMnode, &changeInfo); code = mndProcessVgroupChange(pMnode, &changeInfo);
// keep the new vnode snapshot // keep the new vnode snapshot
if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) {
mDebug("create trans successfully, update cached node list"); mDebug("create trans successfully, update cached node list");
taosArrayDestroy(execNodeList.pNodeEntryList); taosArrayDestroy(execInfo.pNodeEntryList);
execNodeList.pNodeEntryList = pNodeSnapshot; execInfo.pNodeEntryList = pNodeSnapshot;
execNodeList.ts = ts; execInfo.ts = ts;
} else {
mDebug("unexpect code during create nodeUpdate trans, code:%s", tstrerror(code));
taosArrayDestroy(pNodeSnapshot);
} }
} else { } else {
mDebug("no update found in nodeList"); mDebug("no update found in nodeList");
taosArrayDestroy(pNodeSnapshot); taosArrayDestroy(pNodeSnapshot);
} }
taosThreadMutexUnlock(&execNodeList.lock); taosThreadMutexUnlock(&execInfo.lock);
taosArrayDestroy(changeInfo.pUpdateNodeList); taosArrayDestroy(changeInfo.pUpdateNodeList);
taosHashCleanup(changeInfo.pDBMap); taosHashCleanup(changeInfo.pDBMap);
@ -2324,10 +2394,6 @@ typedef struct SMStreamNodeCheckMsg {
int8_t placeHolder; // // to fix windows compile error, define place holder int8_t placeHolder; // // to fix windows compile error, define place holder
} SMStreamNodeCheckMsg; } SMStreamNodeCheckMsg;
typedef struct SMStreamTaskResetMsg {
int8_t placeHolder;
} SMStreamTaskResetMsg;
static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -2343,7 +2409,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) {
return 0; return 0;
} }
void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) { void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) {
int32_t level = taosArrayGetSize(pStream->tasks); int32_t level = taosArrayGetSize(pStream->tasks);
for (int32_t i = 0; i < level; i++) { for (int32_t i = 0; i < level; i++) {
@ -2368,7 +2434,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecNodeInfo *pExecNode) {
} }
} }
void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode) { void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) {
int32_t level = taosArrayGetSize(pStream->tasks); int32_t level = taosArrayGetSize(pStream->tasks);
for (int32_t i = 0; i < level; i++) { for (int32_t i = 0; i < level; i++) {
SArray *pLevel = taosArrayGetP(pStream->tasks, i); SArray *pLevel = taosArrayGetP(pStream->tasks, i);
@ -2399,7 +2465,7 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecNodeInfo * pExecNode
ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList)); ASSERT(taosHashGetSize(pExecNode->pTaskMap) == taosArrayGetSize(pExecNode->pTaskList));
} }
static STrans* doCreateTrans(SMnode* pMnode, SStreamObj* pStream, const char* name) { STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, name); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, NULL, name);
if (pTrans == NULL) { if (pTrans == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY));
@ -2451,7 +2517,7 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
pReq->streamId = pTask->id.streamId; pReq->streamId = pTask->id.streamId;
STransAction action = {0}; STransAction action = {0};
initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet); initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet, 0);
if (mndTransAppendRedoAction(pTrans, &action) != 0) { if (mndTransAppendRedoAction(pTrans, &action) != 0) {
taosMemoryFree(pReq); taosMemoryFree(pReq);
taosWUnLockLatch(&pStream->lock); taosWUnLockLatch(&pStream->lock);
@ -2482,29 +2548,26 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) {
return TSDB_CODE_ACTION_IN_PROGRESS; return TSDB_CODE_ACTION_IN_PROGRESS;
} }
int32_t mndResetFromCheckpoint(SMnode* pMnode) { int32_t doKillActiveCheckpointTrans(SMnode *pMnode) {
// find the checkpoint trans id
int32_t transId = 0; int32_t transId = 0;
SSdb *pSdb = pMnode->pSdb;
STrans *pTrans = NULL;
void *pIter = NULL;
{ while (1) {
SSdb *pSdb = pMnode->pSdb; pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans);
STrans *pTrans = NULL; if (pIter == NULL) {
void* pIter = NULL; break;
while (1) {
pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans);
if (pIter == NULL) {
break;
}
if (strncmp(pTrans->opername, MND_STREAM_CHECKPOINT_NAME, tListLen(pTrans->opername) - 1) == 0) {
transId = pTrans->id;
sdbRelease(pSdb, pTrans);
sdbCancelFetch(pSdb, pIter);
break;
}
sdbRelease(pSdb, pTrans);
} }
if (strncmp(pTrans->opername, MND_STREAM_CHECKPOINT_NAME, tListLen(pTrans->opername) - 1) == 0) {
transId = pTrans->id;
sdbRelease(pSdb, pTrans);
sdbCancelFetch(pSdb, pIter);
break;
}
sdbRelease(pSdb, pTrans);
} }
if (transId == 0) { if (transId == 0) {
@ -2512,8 +2575,16 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
STrans* pTrans = mndAcquireTrans(pMnode, transId); pTrans = mndAcquireTrans(pMnode, transId);
mInfo("kill checkpoint trans:%d", transId);
mndKillTrans(pMnode, pTrans); mndKillTrans(pMnode, pTrans);
mndReleaseTrans(pMnode, pTrans);
return TSDB_CODE_SUCCESS;
}
int32_t mndResetFromCheckpoint(SMnode* pMnode) {
doKillActiveCheckpointTrans(pMnode);
// set all tasks status to be normal, refactor later to be stream level, instead of vnode level. // set all tasks status to be normal, refactor later to be stream level, instead of vnode level.
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -2525,6 +2596,7 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
break; break;
} }
// todo this transaction should exist be only one
mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid); mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid);
int32_t code = createStreamResetStatusTrans(pMnode, pStream); int32_t code = createStreamResetStatusTrans(pMnode, pStream);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -2536,6 +2608,43 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) {
return 0; return 0;
} }
int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) {
int32_t num = taosArrayGetSize(pNodeList);
for (int k = 0; k < num; ++k) {
int32_t* pVgId = taosArrayGet(pNodeList, k);
int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
for (int i = 0; i < numOfNodes; ++i) {
SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i);
if (pNodeEntry->nodeId == *pVgId) {
mInfo("vgId:%d expired in stream task, needs update nodeEp", *pVgId);
pNodeEntry->stageUpdated = true;
break;
}
}
}
return TSDB_CODE_SUCCESS;
}
static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) {
int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList);
for(int32_t j = 0; j < numOfNodes; ++j) {
SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j);
if (pNodeEntry->nodeId == pTaskEntry->nodeId) {
mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId,
pTaskEntry->stage, stage, pTaskEntry->id.taskId);
pNodeEntry->stageUpdated = true;
pTaskEntry->stage = stage;
break;
}
}
}
int32_t mndProcessStreamHb(SRpcMsg *pReq) { int32_t mndProcessStreamHb(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SStreamHbMsg req = {0}; SStreamHbMsg req = {0};
@ -2555,35 +2664,26 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks); mTrace("receive stream-meta hb from vgId:%d, active numOfTasks:%d", req.vgId, req.numOfTasks);
taosThreadMutexLock(&execNodeList.lock); taosThreadMutexLock(&execInfo.lock);
int32_t numOfExisted = taosHashGetSize(execNodeList.pTaskMap); int32_t numOfExisted = taosHashGetSize(execInfo.pTaskMap);
if (numOfExisted == 0) { if (numOfExisted == 0) {
doExtractTasksFromStream(pMnode); doExtractTasksFromStream(pMnode);
} }
setNodeEpsetExpiredFlag(req.pUpdateNodes);
for (int32_t i = 0; i < req.numOfTasks; ++i) { for (int32_t i = 0; i < req.numOfTasks; ++i) {
STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i); STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i);
STaskStatusEntry *pEntry = taosHashGet(execNodeList.pTaskMap, &p->id, sizeof(p->id)); STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id));
if (pEntry == NULL) { if (pTaskEntry == NULL) {
mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId); mError("s-task:0x%" PRIx64 " not found in mnode task list", p->id.taskId);
continue; continue;
} }
if (p->stage != pEntry->stage && pEntry->stage != -1) { if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) {
int32_t numOfNodes = taosArrayGetSize(execNodeList.pNodeEntryList); updateStageInfo(pTaskEntry, p->stage);
for(int32_t j = 0; j < numOfNodes; ++j) {
SNodeEntry* pNodeEntry = taosArrayGet(execNodeList.pNodeEntryList, j);
if (pNodeEntry->nodeId == pEntry->nodeId) {
mInfo("vgId:%d stage updated, from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64,
pEntry->nodeId, pEntry->stage, p->stage, pEntry->id.taskId);
pNodeEntry->stageUpdated = true;
pEntry->stage = p->stage;
break;
}
}
} else { } else {
streamTaskStatusCopy(pEntry, p); streamTaskStatusCopy(pTaskEntry, p);
if (p->activeCheckpointId != 0) { if (p->activeCheckpointId != 0) {
if (activeCheckpointId != 0) { if (activeCheckpointId != 0) {
ASSERT(activeCheckpointId == p->activeCheckpointId); ASSERT(activeCheckpointId == p->activeCheckpointId);
@ -2597,26 +2697,32 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) {
} }
} }
pEntry->status = p->status; pTaskEntry->status = p->status;
if (p->status != TASK_STATUS__NORMAL) { if (p->status != TASK_STATUS__READY) {
mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamGetTaskStatusStr(p->status)); mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status));
} }
} }
// current checkpoint is failed, rollback from the checkpoint trans // current checkpoint is failed, rollback from the checkpoint trans
// kill the checkpoint trans and then set all tasks status to be normal // kill the checkpoint trans and then set all tasks status to be normal
if (checkpointFailed && activeCheckpointId != 0) { if (checkpointFailed && activeCheckpointId != 0) {
if (execNodeList.activeCheckpoint != activeCheckpointId) { bool allReady = true;
mInfo("checkpointId:%"PRId64" failed, issue task-reset trans to reset all tasks status", activeCheckpointId); SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady);
execNodeList.activeCheckpoint = activeCheckpointId; taosArrayDestroy(p);
if (allReady) {
// if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal
mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status",
execInfo.activeCheckpoint);
mndResetFromCheckpoint(pMnode); mndResetFromCheckpoint(pMnode);
} else { } else {
mDebug("checkpoint:%"PRId64" reset has issued already, ignore it", activeCheckpointId); mInfo("not all vgroups are ready, wait for next HB from stream tasks");
} }
} }
taosThreadMutexUnlock(&execNodeList.lock); taosThreadMutexUnlock(&execInfo.lock);
taosArrayDestroy(req.pTaskStatus); taosArrayDestroy(req.pTaskStatus);
taosArrayDestroy(req.pUpdateNodes);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

2
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -720,7 +720,7 @@ int32_t mndSetRpcInfoForDbTrans(SMnode *pMnode, SRpcMsg *pMsg, EOperType oper, c
void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname) { void mndTransSetDbName(STrans *pTrans, const char *dbname, const char *stbname) {
if (dbname != NULL) { if (dbname != NULL) {
tstrncpy(pTrans->dbname, dbname, TSDB_TABLE_FNAME_LEN); tstrncpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN);
} }
if (stbname != NULL) { if (stbname != NULL) {
tstrncpy(pTrans->stbname, stbname, TSDB_TABLE_FNAME_LEN); tstrncpy(pTrans->stbname, stbname, TSDB_TABLE_FNAME_LEN);

738
source/dnode/mnode/impl/src/mndUser.c
View File

@ -28,9 +28,53 @@
// clang-format on // clang-format on
#define USER_VER_NUMBER 5 #define USER_VER_NUMBER 6
#define USER_RESERVE_SIZE 64 #define USER_RESERVE_SIZE 64
#define BIT_FLAG_MASK(n) (1 << n)
#define BIT_FLAG_SET_MASK(val, mask) ((val) |= (mask))
#define BIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0)
#define PRIVILEGE_TYPE_ALL BIT_FLAG_MASK(0)
#define PRIVILEGE_TYPE_READ BIT_FLAG_MASK(1)
#define PRIVILEGE_TYPE_WRITE BIT_FLAG_MASK(2)
#define PRIVILEGE_TYPE_SUBSCRIBE BIT_FLAG_MASK(3)
#define PRIVILEGE_TYPE_ALTER BIT_FLAG_MASK(4)
#define ALTER_USER_ADD_PRIVS(_type) ((_type) == TSDB_ALTER_USER_ADD_PRIVILEGES)
#define ALTER_USER_DEL_PRIVS(_type) ((_type) == TSDB_ALTER_USER_DEL_PRIVILEGES)
#define ALTER_USER_ALL_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_READ_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_READ) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_WRITE_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_WRITE) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_ALTER_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALTER) || BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_ALL))
#define ALTER_USER_SUBSCRIBE_PRIV(_priv) (BIT_FLAG_TEST_MASK((_priv), PRIVILEGE_TYPE_SUBSCRIBE))
#define ALTER_USER_TARGET_DB(_tbname) (0 == (_tbname)[0])
#define ALTER_USER_TARGET_TB(_tbname) (0 != (_tbname)[0])
#define ALTER_USER_ADD_READ_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_READ_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_WRITE_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_WRITE_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_ALTER_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_ALTER_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_ALL_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_DEL_ALL_DB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_DB(_tbname))
#define ALTER_USER_ADD_READ_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_READ_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_READ_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_WRITE_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_WRITE_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_WRITE_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_ALTER_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_ALTER_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALTER_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_ALL_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_DEL_ALL_TB_PRIV(_type, _priv, _tbname) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_ALL_PRIV(_priv) && ALTER_USER_TARGET_TB(_tbname))
#define ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(_type, _priv) (ALTER_USER_ADD_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
#define ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(_type, _priv) (ALTER_USER_DEL_PRIVS(_type) && ALTER_USER_SUBSCRIBE_PRIV(_priv))
static SIpWhiteList *createDefaultIpWhiteList(); static SIpWhiteList *createDefaultIpWhiteList();
SIpWhiteList *createIpWhiteList(void *buf, int32_t len); SIpWhiteList *createIpWhiteList(void *buf, int32_t len);
static bool updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew); static bool updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew);
@ -656,8 +700,12 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
pUser->pIpWhiteList ? (sizeof(SIpV4Range) * pUser->pIpWhiteList->num + sizeof(SIpWhiteList) + 4) : 16; pUser->pIpWhiteList ? (sizeof(SIpV4Range) * pUser->pIpWhiteList->num + sizeof(SIpWhiteList) + 4) : 16;
int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs);
int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs);
int32_t numOfReadStbs = taosHashGetSize(pUser->readTbs); int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs);
int32_t numOfWriteStbs = taosHashGetSize(pUser->writeTbs); int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs);
int32_t numOfAlterTbs = taosHashGetSize(pUser->alterTbs);
int32_t numOfReadViews = taosHashGetSize(pUser->readViews);
int32_t numOfWriteViews = taosHashGetSize(pUser->writeViews);
int32_t numOfAlterViews = taosHashGetSize(pUser->alterViews);
int32_t numOfTopics = taosHashGetSize(pUser->topics); int32_t numOfTopics = taosHashGetSize(pUser->topics);
int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs); int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs);
int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE +
@ -692,6 +740,62 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
stb = taosHashIterate(pUser->writeTbs, stb); stb = taosHashIterate(pUser->writeTbs, stb);
} }
stb = taosHashIterate(pUser->alterTbs, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
size += sizeof(int32_t);
size += keyLen;
size_t valueLen = 0;
valueLen = strlen(stb);
size += sizeof(int32_t);
size += valueLen;
stb = taosHashIterate(pUser->alterTbs, stb);
}
stb = taosHashIterate(pUser->readViews, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
size += sizeof(int32_t);
size += keyLen;
size_t valueLen = 0;
valueLen = strlen(stb);
size += sizeof(int32_t);
size += valueLen;
stb = taosHashIterate(pUser->readViews, stb);
}
stb = taosHashIterate(pUser->writeViews, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
size += sizeof(int32_t);
size += keyLen;
size_t valueLen = 0;
valueLen = strlen(stb);
size += sizeof(int32_t);
size += valueLen;
stb = taosHashIterate(pUser->writeViews, stb);
}
stb = taosHashIterate(pUser->alterViews, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
size += sizeof(int32_t);
size += keyLen;
size_t valueLen = 0;
valueLen = strlen(stb);
size += sizeof(int32_t);
size += valueLen;
stb = taosHashIterate(pUser->alterViews, stb);
}
SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size);
if (pRaw == NULL) goto _OVER; if (pRaw == NULL) goto _OVER;
@ -729,8 +833,12 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
topic = taosHashIterate(pUser->topics, topic); topic = taosHashIterate(pUser->topics, topic);
} }
SDB_SET_INT32(pRaw, dataPos, numOfReadStbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadTbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteStbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteTbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfAlterTbs, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfReadViews, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfWriteViews, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfAlterViews, _OVER)
SDB_SET_INT32(pRaw, dataPos, numOfUseDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfUseDbs, _OVER)
stb = taosHashIterate(pUser->readTbs, NULL); stb = taosHashIterate(pUser->readTbs, NULL);
@ -761,6 +869,62 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) {
stb = taosHashIterate(pUser->writeTbs, stb); stb = taosHashIterate(pUser->writeTbs, stb);
} }
stb = taosHashIterate(pUser->alterTbs, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
size_t valueLen = 0;
valueLen = strlen(stb) + 1;
SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
stb = taosHashIterate(pUser->alterTbs, stb);
}
stb = taosHashIterate(pUser->readViews, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
size_t valueLen = 0;
valueLen = strlen(stb) + 1;
SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
stb = taosHashIterate(pUser->readViews, stb);
}
stb = taosHashIterate(pUser->writeViews, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
size_t valueLen = 0;
valueLen = strlen(stb) + 1;
SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
stb = taosHashIterate(pUser->writeViews, stb);
}
stb = taosHashIterate(pUser->alterViews, NULL);
while (stb != NULL) {
size_t keyLen = 0;
void *key = taosHashGetKey(stb, &keyLen);
SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
size_t valueLen = 0;
valueLen = strlen(stb) + 1;
SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER)
SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER);
stb = taosHashIterate(pUser->alterViews, stb);
}
int32_t *useDb = taosHashIterate(pUser->useDbs, NULL); int32_t *useDb = taosHashIterate(pUser->useDbs, NULL);
while (useDb != NULL) { while (useDb != NULL) {
size_t keyLen = 0; size_t keyLen = 0;
@ -873,20 +1037,40 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
} }
if (sver >= 3) { if (sver >= 3) {
int32_t numOfReadStbs = 0; int32_t numOfReadTbs = 0;
int32_t numOfWriteStbs = 0; int32_t numOfWriteTbs = 0;
int32_t numOfAlterTbs = 0;
int32_t numOfReadViews = 0;
int32_t numOfWriteViews = 0;
int32_t numOfAlterViews = 0;
int32_t numOfUseDbs = 0; int32_t numOfUseDbs = 0;
SDB_GET_INT32(pRaw, dataPos, &numOfReadStbs, _OVER) SDB_GET_INT32(pRaw, dataPos, &numOfReadTbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteStbs, _OVER) SDB_GET_INT32(pRaw, dataPos, &numOfWriteTbs, _OVER)
if (sver >= 6) {
SDB_GET_INT32(pRaw, dataPos, &numOfAlterTbs, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfReadViews, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfWriteViews, _OVER)
SDB_GET_INT32(pRaw, dataPos, &numOfAlterViews, _OVER)
}
SDB_GET_INT32(pRaw, dataPos, &numOfUseDbs, _OVER) SDB_GET_INT32(pRaw, dataPos, &numOfUseDbs, _OVER)
pUser->readTbs = pUser->readTbs =
taosHashInit(numOfReadStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taosHashInit(numOfReadTbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->writeTbs = pUser->writeTbs =
taosHashInit(numOfWriteStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); taosHashInit(numOfWriteTbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->alterTbs =
taosHashInit(numOfAlterTbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->readViews =
taosHashInit(numOfReadViews, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->writeViews =
taosHashInit(numOfWriteViews, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->alterViews =
taosHashInit(numOfAlterViews, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pUser->useDbs = taosHashInit(numOfUseDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pUser->useDbs = taosHashInit(numOfUseDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
for (int32_t i = 0; i < numOfReadStbs; ++i) { for (int32_t i = 0; i < numOfReadTbs; ++i) {
int32_t keyLen = 0; int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
@ -906,7 +1090,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
taosMemoryFree(value); taosMemoryFree(value);
} }
for (int32_t i = 0; i < numOfWriteStbs; ++i) { for (int32_t i = 0; i < numOfWriteTbs; ++i) {
int32_t keyLen = 0; int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
@ -926,6 +1110,88 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) {
taosMemoryFree(value); taosMemoryFree(value);
} }
if (sver >= 6) {
for (int32_t i = 0; i < numOfAlterTbs; ++i) {
int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
char *key = taosMemoryCalloc(keyLen, sizeof(char));
memset(key, 0, keyLen);
SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
int32_t valuelen = 0;
SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
char *value = taosMemoryCalloc(valuelen, sizeof(char));
memset(value, 0, valuelen);
SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
taosHashPut(pUser->alterTbs, key, keyLen, value, valuelen);
taosMemoryFree(key);
taosMemoryFree(value);
}
for (int32_t i = 0; i < numOfReadViews; ++i) {
int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
char *key = taosMemoryCalloc(keyLen, sizeof(char));
memset(key, 0, keyLen);
SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
int32_t valuelen = 0;
SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
char *value = taosMemoryCalloc(valuelen, sizeof(char));
memset(value, 0, valuelen);
SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
taosHashPut(pUser->readViews, key, keyLen, value, valuelen);
taosMemoryFree(key);
taosMemoryFree(value);
}
for (int32_t i = 0; i < numOfWriteViews; ++i) {
int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
char *key = taosMemoryCalloc(keyLen, sizeof(char));
memset(key, 0, keyLen);
SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
int32_t valuelen = 0;
SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
char *value = taosMemoryCalloc(valuelen, sizeof(char));
memset(value, 0, valuelen);
SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
taosHashPut(pUser->writeViews, key, keyLen, value, valuelen);
taosMemoryFree(key);
taosMemoryFree(value);
}
for (int32_t i = 0; i < numOfAlterViews; ++i) {
int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
char *key = taosMemoryCalloc(keyLen, sizeof(char));
memset(key, 0, keyLen);
SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER);
int32_t valuelen = 0;
SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER);
char *value = taosMemoryCalloc(valuelen, sizeof(char));
memset(value, 0, valuelen);
SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER)
taosHashPut(pUser->alterViews, key, keyLen, value, valuelen);
taosMemoryFree(key);
taosMemoryFree(value);
}
}
for (int32_t i = 0; i < numOfUseDbs; ++i) { for (int32_t i = 0; i < numOfUseDbs; ++i) {
int32_t keyLen = 0; int32_t keyLen = 0;
SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER);
@ -975,6 +1241,10 @@ _OVER:
taosHashCleanup(pUser->topics); taosHashCleanup(pUser->topics);
taosHashCleanup(pUser->readTbs); taosHashCleanup(pUser->readTbs);
taosHashCleanup(pUser->writeTbs); taosHashCleanup(pUser->writeTbs);
taosHashCleanup(pUser->alterTbs);
taosHashCleanup(pUser->readViews);
taosHashCleanup(pUser->writeViews);
taosHashCleanup(pUser->alterViews);
taosHashCleanup(pUser->useDbs); taosHashCleanup(pUser->useDbs);
taosMemoryFreeClear(pUser->pIpWhiteList); taosMemoryFreeClear(pUser->pIpWhiteList);
} }
@ -1062,6 +1332,10 @@ int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) {
pNew->writeDbs = mndDupDbHash(pUser->writeDbs); pNew->writeDbs = mndDupDbHash(pUser->writeDbs);
pNew->readTbs = mndDupTableHash(pUser->readTbs); pNew->readTbs = mndDupTableHash(pUser->readTbs);
pNew->writeTbs = mndDupTableHash(pUser->writeTbs); pNew->writeTbs = mndDupTableHash(pUser->writeTbs);
pNew->alterTbs = mndDupTableHash(pUser->alterTbs);
pNew->readViews = mndDupTableHash(pUser->readViews);
pNew->writeViews = mndDupTableHash(pUser->writeViews);
pNew->alterViews = mndDupTableHash(pUser->alterViews);
pNew->topics = mndDupTopicHash(pUser->topics); pNew->topics = mndDupTopicHash(pUser->topics);
pNew->useDbs = mndDupUseDbHash(pUser->useDbs); pNew->useDbs = mndDupUseDbHash(pUser->useDbs);
pNew->pIpWhiteList = cloneIpWhiteList(pUser->pIpWhiteList); pNew->pIpWhiteList = cloneIpWhiteList(pUser->pIpWhiteList);
@ -1080,6 +1354,10 @@ void mndUserFreeObj(SUserObj *pUser) {
taosHashCleanup(pUser->topics); taosHashCleanup(pUser->topics);
taosHashCleanup(pUser->readTbs); taosHashCleanup(pUser->readTbs);
taosHashCleanup(pUser->writeTbs); taosHashCleanup(pUser->writeTbs);
taosHashCleanup(pUser->alterTbs);
taosHashCleanup(pUser->readViews);
taosHashCleanup(pUser->writeViews);
taosHashCleanup(pUser->alterViews);
taosHashCleanup(pUser->useDbs); taosHashCleanup(pUser->useDbs);
taosMemoryFreeClear(pUser->pIpWhiteList); taosMemoryFreeClear(pUser->pIpWhiteList);
pUser->readDbs = NULL; pUser->readDbs = NULL;
@ -1087,6 +1365,10 @@ void mndUserFreeObj(SUserObj *pUser) {
pUser->topics = NULL; pUser->topics = NULL;
pUser->readTbs = NULL; pUser->readTbs = NULL;
pUser->writeTbs = NULL; pUser->writeTbs = NULL;
pUser->alterTbs = NULL;
pUser->readViews = NULL;
pUser->writeViews = NULL;
pUser->alterViews = NULL;
pUser->useDbs = NULL; pUser->useDbs = NULL;
} }
@ -1110,6 +1392,10 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) {
TSWAP(pOld->topics, pNew->topics); TSWAP(pOld->topics, pNew->topics);
TSWAP(pOld->readTbs, pNew->readTbs); TSWAP(pOld->readTbs, pNew->readTbs);
TSWAP(pOld->writeTbs, pNew->writeTbs); TSWAP(pOld->writeTbs, pNew->writeTbs);
TSWAP(pOld->alterTbs, pNew->alterTbs);
TSWAP(pOld->readViews, pNew->readViews);
TSWAP(pOld->writeViews, pNew->writeViews);
TSWAP(pOld->alterViews, pNew->alterViews);
TSWAP(pOld->useDbs, pNew->useDbs); TSWAP(pOld->useDbs, pNew->useDbs);
int32_t sz = sizeof(SIpWhiteList) + pNew->pIpWhiteList->num * sizeof(SIpV4Range); int32_t sz = sizeof(SIpWhiteList) + pNew->pIpWhiteList->num * sizeof(SIpV4Range);
@ -1429,7 +1715,7 @@ static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useD
return -1; return -1;
} }
} else { } else {
if (taosHashPut(hash, tbFName, len, "t", 2) != 0) { if (taosHashPut(hash, tbFName, len, alterReq->isView ? "v" : "t", 2) != 0) {
return -1; return -1;
} }
} }
@ -1481,60 +1767,162 @@ static char *mndUserAuditTypeStr(int32_t type) {
if (type == TSDB_ALTER_USER_SUPERUSER) { if (type == TSDB_ALTER_USER_SUPERUSER) {
return "changeSuperUser"; return "changeSuperUser";
} }
if (type == TSDB_ALTER_USER_ADD_READ_DB) {
return "addReadToDB";
}
if (type == TSDB_ALTER_USER_ADD_READ_DB) {
return "addReadToDB";
}
if (type == TSDB_ALTER_USER_REMOVE_READ_DB) {
return "removeReadFromDB";
}
if (type == TSDB_ALTER_USER_ADD_WRITE_DB) {
return "addWriteToDB";
}
if (type == TSDB_ALTER_USER_REMOVE_WRITE_DB) {
return "removeWriteFromDB";
}
if (type == TSDB_ALTER_USER_ADD_ALL_DB) {
return "addToAllDB";
}
if (type == TSDB_ALTER_USER_REMOVE_ALL_DB) {
return "removeFromAllDB";
}
if (type == TSDB_ALTER_USER_ENABLE) { if (type == TSDB_ALTER_USER_ENABLE) {
return "enableUser"; return "enableUser";
} }
if (type == TSDB_ALTER_USER_SYSINFO) { if (type == TSDB_ALTER_USER_SYSINFO) {
return "userSysInfo"; return "userSysInfo";
} }
if (type == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
return "addSubscribeTopic";
}
if (type == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) {
return "removeSubscribeTopic";
}
if (type == TSDB_ALTER_USER_ADD_READ_TABLE) {
return "addReadToTable";
}
if (type == TSDB_ALTER_USER_REMOVE_READ_TABLE) {
return "removeReadFromTable";
}
if (type == TSDB_ALTER_USER_ADD_WRITE_TABLE) {
return "addWriteToTable";
}
if (type == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) {
return "removeWriteFromTable";
}
if (type == TSDB_ALTER_USER_ADD_ALL_TABLE) {
return "addToAllTable";
}
if (type == TSDB_ALTER_USER_REMOVE_ALL_TABLE) {
return "removeFromAllTable";
}
return "error"; return "error";
} }
static int32_t mndProcessAlterUserPrivilegesReq(SAlterUserReq *pAlterReq, SMnode *pMnode, SUserObj* pNewUser) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
if (ALTER_USER_ADD_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
return -1;
}
if (taosHashPut(pNewUser->readDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN) != 0) {
mndReleaseDb(pMnode, pDb);
return -1;
}
mndReleaseDb(pMnode, pDb);
} else {
while (1) {
SDbObj *pDb = NULL;
pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
if (pIter == NULL) break;
int32_t len = strlen(pDb->name) + 1;
taosHashPut(pNewUser->readDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
sdbRelease(pSdb, pDb);
}
}
}
if (ALTER_USER_ADD_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
return -1;
}
if (taosHashPut(pNewUser->writeDbs, pAlterReq->objname, len, pAlterReq->objname, TSDB_DB_FNAME_LEN) != 0) {
mndReleaseDb(pMnode, pDb);
return -1;
}
mndReleaseDb(pMnode, pDb);
} else {
while (1) {
SDbObj *pDb = NULL;
pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
if (pIter == NULL) break;
int32_t len = strlen(pDb->name) + 1;
taosHashPut(pNewUser->writeDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
sdbRelease(pSdb, pDb);
}
}
}
if (ALTER_USER_DEL_READ_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
return -1;
}
taosHashRemove(pNewUser->readDbs, pAlterReq->objname, len);
mndReleaseDb(pMnode, pDb);
} else {
taosHashClear(pNewUser->readDbs);
}
}
if (ALTER_USER_DEL_WRITE_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_DEL_ALL_DB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (strcmp(pAlterReq->objname, "1.*") != 0) {
int32_t len = strlen(pAlterReq->objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, pAlterReq->objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
return -1;
}
taosHashRemove(pNewUser->writeDbs, pAlterReq->objname, len);
mndReleaseDb(pMnode, pDb);
} else {
taosHashClear(pNewUser->writeDbs);
}
}
SHashObj* pReadTbs = pNewUser->readTbs;
SHashObj* pWriteTbs = pNewUser->writeTbs;
SHashObj* pAlterTbs = pNewUser->alterTbs;
#ifdef TD_ENTERPRISE
if (pAlterReq->isView) {
pReadTbs = pNewUser->readViews;
pWriteTbs = pNewUser->writeViews;
pAlterTbs = pNewUser->alterViews;
}
#endif
if (ALTER_USER_ADD_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_ADD_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_ADD_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) || ALTER_USER_ADD_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_DEL_READ_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndRemoveTablePriviledge(pMnode, pReadTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_DEL_WRITE_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndRemoveTablePriviledge(pMnode, pWriteTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_DEL_ALTER_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName) ||
ALTER_USER_DEL_ALL_TB_PRIV(pAlterReq->alterType, pAlterReq->privileges, pAlterReq->tabName)) {
if (mndRemoveTablePriviledge(pMnode, pAlterTbs, pNewUser->useDbs, pAlterReq, pSdb) != 0) return -1;
}
if (ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
int32_t len = strlen(pAlterReq->objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname);
if (pTopic == NULL) {
mndReleaseTopic(pMnode, pTopic);
return -1;
}
taosHashPut(pNewUser->topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
}
if (ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(pAlterReq->alterType, pAlterReq->privileges)) {
int32_t len = strlen(pAlterReq->objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pAlterReq->objname);
if (pTopic == NULL) {
mndReleaseTopic(pMnode, pTopic);
return -1;
}
taosHashRemove(pNewUser->topics, pAlterReq->objname, len);
}
return TSDB_CODE_SUCCESS;
}
static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
SMnode *pMnode = pReq->info.node; SMnode *pMnode = pReq->info.node;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
@ -1602,122 +1990,8 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
newUser.sysInfo = alterReq.sysInfo; newUser.sysInfo = alterReq.sysInfo;
} }
if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) { if (ALTER_USER_ADD_PRIVS(alterReq.alterType) || ALTER_USER_DEL_PRIVS(alterReq.alterType)) {
if (strcmp(alterReq.objname, "1.*") != 0) { if (0 != mndProcessAlterUserPrivilegesReq(&alterReq, pMnode, &newUser)) goto _OVER;
int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
goto _OVER;
}
if (taosHashPut(newUser.readDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
mndReleaseDb(pMnode, pDb);
goto _OVER;
}
mndReleaseDb(pMnode, pDb);
} else {
while (1) {
SDbObj *pDb = NULL;
pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
if (pIter == NULL) break;
int32_t len = strlen(pDb->name) + 1;
taosHashPut(newUser.readDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
sdbRelease(pSdb, pDb);
}
}
}
if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB) {
if (strcmp(alterReq.objname, "1.*") != 0) {
int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
goto _OVER;
}
if (taosHashPut(newUser.writeDbs, alterReq.objname, len, alterReq.objname, TSDB_DB_FNAME_LEN) != 0) {
mndReleaseDb(pMnode, pDb);
goto _OVER;
}
mndReleaseDb(pMnode, pDb);
} else {
while (1) {
SDbObj *pDb = NULL;
pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb);
if (pIter == NULL) break;
int32_t len = strlen(pDb->name) + 1;
taosHashPut(newUser.writeDbs, pDb->name, len, pDb->name, TSDB_DB_FNAME_LEN);
sdbRelease(pSdb, pDb);
}
}
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
if (strcmp(alterReq.objname, "1.*") != 0) {
int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
goto _OVER;
}
taosHashRemove(newUser.readDbs, alterReq.objname, len);
mndReleaseDb(pMnode, pDb);
} else {
taosHashClear(newUser.readDbs);
}
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_DB) {
if (strcmp(alterReq.objname, "1.*") != 0) {
int32_t len = strlen(alterReq.objname) + 1;
SDbObj *pDb = mndAcquireDb(pMnode, alterReq.objname);
if (pDb == NULL) {
mndReleaseDb(pMnode, pDb);
goto _OVER;
}
taosHashRemove(newUser.writeDbs, alterReq.objname, len);
mndReleaseDb(pMnode, pDb);
} else {
taosHashClear(newUser.writeDbs);
}
}
if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) {
if (mndTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE || alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) {
if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE ||
alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) {
if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE ||
alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) {
if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER;
}
if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) {
int32_t len = strlen(alterReq.objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
if (pTopic == NULL) {
mndReleaseTopic(pMnode, pTopic);
goto _OVER;
}
taosHashPut(newUser.topics, pTopic->name, len, pTopic->name, TSDB_TOPIC_FNAME_LEN);
}
if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) {
int32_t len = strlen(alterReq.objname) + 1;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname);
if (pTopic == NULL) {
mndReleaseTopic(pMnode, pTopic);
goto _OVER;
}
taosHashRemove(newUser.topics, alterReq.objname, len);
} }
if (alterReq.alterType == TSDB_ALTER_USER_ADD_WHITE_LIST) { if (alterReq.alterType == TSDB_ALTER_USER_ADD_WHITE_LIST) {
@ -1827,12 +2101,12 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
alterReq.alterType == TSDB_ALTER_USER_SYSINFO){ alterReq.alterType == TSDB_ALTER_USER_SYSINFO){
auditRecord(pReq, pMnode->clusterId, "alterUser", alterReq.user, "", alterReq.sql, alterReq.sqlLen); auditRecord(pReq, pMnode->clusterId, "alterUser", alterReq.user, "", alterReq.sql, alterReq.sqlLen);
} }
else if(alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB|| else if(ALTER_USER_ADD_READ_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB|| ALTER_USER_ADD_WRITE_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB|| ALTER_USER_ADD_ALL_DB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE|| ALTER_USER_ADD_READ_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE|| ALTER_USER_ADD_WRITE_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)||
alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE){ ALTER_USER_ADD_ALL_TB_PRIV(alterReq.alterType, alterReq.privileges, alterReq.tabName)){
if (strcmp(alterReq.objname, "1.*") != 0){ if (strcmp(alterReq.objname, "1.*") != 0){
SName name = {0}; SName name = {0};
tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB); tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB);
@ -1843,11 +2117,11 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) {
alterReq.sql, alterReq.sqlLen); alterReq.sql, alterReq.sqlLen);
} }
} }
else if(alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC){ else if(ALTER_USER_ADD_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)){
auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, alterReq.objname, auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, alterReq.objname,
alterReq.sql, alterReq.sqlLen); alterReq.sql, alterReq.sqlLen);
} }
else if(alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC){ else if(ALTER_USER_DEL_SUBSCRIBE_TOPIC_PRIV(alterReq.alterType, alterReq.privileges)){
auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, alterReq.objname, auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, alterReq.objname,
alterReq.sql, alterReq.sqlLen); alterReq.sql, alterReq.sqlLen);
} }
@ -2092,7 +2366,7 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)tableNameContent, false); colDataSetVal(pColInfo, *numOfRows, (const char *)tableNameContent, false);
if (strcmp("t", value) != 0) { if (strcmp("t", value) != 0 && strcmp("v", value) != 0) {
SNode *pAst = NULL; SNode *pAst = NULL;
int32_t sqlLen = 0; int32_t sqlLen = 0;
size_t bufSz = strlen(value) + 1; size_t bufSz = strlen(value) + 1;
@ -2113,12 +2387,22 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3
colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false); colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false);
taosMemoryFree(obj); taosMemoryFree(obj);
taosMemoryFree(sql); taosMemoryFree(sql);
char notes[2] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)notes, false);
} else { } else {
char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE);
STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes);
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false); colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false);
taosMemoryFree(condition); taosMemoryFree(condition);
char notes[64 + VARSTR_HEADER_SIZE] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(notes, value[0] == 'v' ? "view" : "", sizeof(notes));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, *numOfRows, (const char *)notes, false);
} }
(*numOfRows)++; (*numOfRows)++;
@ -2155,11 +2439,15 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
int32_t numOfTopics = taosHashGetSize(pUser->topics); int32_t numOfTopics = taosHashGetSize(pUser->topics);
int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs); int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs);
int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs); int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs);
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) { int32_t numOfAlterTbs = taosHashGetSize(pUser->alterTbs);
int32_t numOfReadViews = taosHashGetSize(pUser->readViews);
int32_t numOfWriteViews = taosHashGetSize(pUser->writeViews);
int32_t numOfAlterViews = taosHashGetSize(pUser->alterViews);
if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs + numOfAlterTbs + numOfReadViews + numOfWriteViews + numOfAlterViews >= rows) {
mInfo( mInfo(
"will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables " "will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables "
"%d", "%d, alter tables %d, read views %d, write views %d, alter views %d",
numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs); numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs, numOfAlterTbs, numOfReadViews, numOfWriteViews, numOfAlterViews);
pShow->restore = true; pShow->restore = true;
sdbRelease(pSdb, pUser); sdbRelease(pSdb, pUser);
break; break;
@ -2193,6 +2481,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
taosMemoryFree(condition); taosMemoryFree(condition);
char notes[2] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
numOfRows++; numOfRows++;
} }
@ -2228,6 +2521,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
taosMemoryFree(condition); taosMemoryFree(condition);
char notes[2] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
numOfRows++; numOfRows++;
db = taosHashIterate(pUser->readDbs, db); db = taosHashIterate(pUser->readDbs, db);
} }
@ -2264,6 +2562,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
taosMemoryFree(condition); taosMemoryFree(condition);
char notes[2] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
numOfRows++; numOfRows++;
db = taosHashIterate(pUser->writeDbs, db); db = taosHashIterate(pUser->writeDbs, db);
} }
@ -2272,6 +2575,14 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pUser->user, pShow); mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pUser->user, pShow);
mndLoopHash(pUser->alterTbs, "alter", pBlock, &numOfRows, pUser->user, pShow);
mndLoopHash(pUser->readViews, "read", pBlock, &numOfRows, pUser->user, pShow);
mndLoopHash(pUser->writeViews, "write", pBlock, &numOfRows, pUser->user, pShow);
mndLoopHash(pUser->alterViews, "alter", pBlock, &numOfRows, pUser->user, pShow);
char *topic = taosHashIterate(pUser->topics, NULL); char *topic = taosHashIterate(pUser->topics, NULL);
while (topic != NULL) { while (topic != NULL) {
cols = 0; cols = 0;
@ -2302,6 +2613,11 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock
colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false);
taosMemoryFree(condition); taosMemoryFree(condition);
char notes[2] = {0};
STR_WITH_MAXSIZE_TO_VARSTR(notes, "", sizeof(notes));
pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataSetVal(pColInfo, numOfRows, (const char *)notes, false);
numOfRows++; numOfRows++;
topic = taosHashIterate(pUser->topics, topic); topic = taosHashIterate(pUser->topics, topic);
} }
@ -2450,9 +2766,11 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) {
bool inRead = (taosHashGet(newUser.readTbs, stb, len) != NULL); bool inRead = (taosHashGet(newUser.readTbs, stb, len) != NULL);
bool inWrite = (taosHashGet(newUser.writeTbs, stb, len) != NULL); bool inWrite = (taosHashGet(newUser.writeTbs, stb, len) != NULL);
if (inRead || inWrite) { bool inAlter = (taosHashGet(newUser.alterTbs, stb, len) != NULL);
if (inRead || inWrite || inAlter) {
(void)taosHashRemove(newUser.readTbs, stb, len); (void)taosHashRemove(newUser.readTbs, stb, len);
(void)taosHashRemove(newUser.writeTbs, stb, len); (void)taosHashRemove(newUser.writeTbs, stb, len);
(void)taosHashRemove(newUser.alterTbs, stb, len);
SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser); SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
@ -2472,6 +2790,50 @@ int32_t mndUserRemoveStb(SMnode *pMnode, STrans *pTrans, char *stb) {
return code; return code;
} }
int32_t mndUserRemoveView(SMnode *pMnode, STrans *pTrans, char *view) {
int32_t code = 0;
SSdb *pSdb = pMnode->pSdb;
int32_t len = strlen(view) + 1;
void *pIter = NULL;
SUserObj *pUser = NULL;
SUserObj newUser = {0};
while (1) {
pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser);
if (pIter == NULL) break;
code = -1;
if (mndUserDupObj(pUser, &newUser) != 0) {
break;
}
bool inRead = (taosHashGet(newUser.readViews, view, len) != NULL);
bool inWrite = (taosHashGet(newUser.writeViews, view, len) != NULL);
bool inAlter = (taosHashGet(newUser.alterViews, view, len) != NULL);
if (inRead || inWrite || inAlter) {
(void)taosHashRemove(newUser.readViews, view, len);
(void)taosHashRemove(newUser.writeViews, view, len);
(void)taosHashRemove(newUser.alterViews, view, len);
SSdbRaw *pCommitRaw = mndUserActionEncode(&newUser);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
break;
}
(void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
}
mndUserFreeObj(&newUser);
sdbRelease(pSdb, pUser);
code = 0;
}
if (pUser != NULL) sdbRelease(pSdb, pUser);
if (pIter != NULL) sdbCancelFetch(pSdb, pIter);
mndUserFreeObj(&newUser);
return code;
}
int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) { int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic) {
int32_t code = 0; int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;

18
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -754,10 +754,14 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SDbObj *pDb, SVgObj *pVgroup
for (int32_t v = 0; v < pVgroup->replica; ++v) { for (int32_t v = 0; v < pVgroup->replica; ++v) {
SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SVnodeGid *pVgid = &pVgroup->vnodeGid[v];
SDnodeObj *pDnode = taosArrayGet(pArray, v); SDnodeObj *pDnode = taosArrayGet(pArray, v);
if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1; return -1;
} }
if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
return -1;
}
int64_t vgMem = mndGetVgroupMemory(pMnode, pDb, pVgroup); int64_t vgMem = mndGetVgroupMemory(pMnode, pDb, pVgroup);
if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) { if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@ -1180,10 +1184,14 @@ static int32_t mndAddVnodeToVgroup(SMnode *pMnode, STrans *pTrans, SVgObj *pVgro
} }
if (used) continue; if (used) continue;
if (pDnode == NULL || pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) { if (pDnode == NULL) {
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES;
return -1; return -1;
} }
if (pDnode->numOfVnodes >= pDnode->numOfSupportVnodes) {
terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
return -1;
}
int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup); int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) { if (pDnode->memAvail - vgMem - pDnode->memUsed <= 0) {
@ -1912,7 +1920,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
if (numOfVnodes >= pNew1->numOfSupportVnodes) { if (numOfVnodes >= pNew1->numOfSupportVnodes) {
mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes, mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew1->id, numOfVnodes,
pNew1->numOfSupportVnodes); pNew1->numOfSupportVnodes);
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
goto _OVER; goto _OVER;
} }
@ -1935,7 +1943,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
if (numOfVnodes >= pNew2->numOfSupportVnodes) { if (numOfVnodes >= pNew2->numOfSupportVnodes) {
mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes, mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew2->id, numOfVnodes,
pNew2->numOfSupportVnodes); pNew2->numOfSupportVnodes);
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
goto _OVER; goto _OVER;
} }
int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup); int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);
@ -1956,7 +1964,7 @@ static int32_t mndRedistributeVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb,
if (numOfVnodes >= pNew3->numOfSupportVnodes) { if (numOfVnodes >= pNew3->numOfSupportVnodes) {
mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes, mError("vgId:%d, no enough vnodes in dnode:%d, numOfVnodes:%d support:%d", newVg.vgId, pNew3->id, numOfVnodes,
pNew3->numOfSupportVnodes); pNew3->numOfSupportVnodes);
terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; terrno = TSDB_CODE_MND_NO_ENOUGH_VNODES;
goto _OVER; goto _OVER;
} }
int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup); int64_t vgMem = mndGetVgroupMemory(pMnode, NULL, pVgroup);

112
source/dnode/mnode/impl/src/mndView.c Executable file
View File

@ -0,0 +1,112 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mndView.h"
#include "mndShow.h"
int32_t mndInitView(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_MND_CREATE_VIEW, mndProcessCreateViewReq);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_VIEW, mndProcessDropViewReq);
mndSetMsgHandle(pMnode, TDMT_MND_VIEW_META, mndProcessGetViewMetaReq);
mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_VIEWS, mndRetrieveView);
mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_VIEWS, mndCancelGetNextView);
#ifdef TD_ENTERPRISE
initDynViewVersion();
SSdbTable table = {
.sdbType = SDB_VIEW,
.keyType = SDB_KEY_BINARY,
.encodeFp = (SdbEncodeFp)mndViewActionEncode,
.decodeFp = (SdbDecodeFp)mndViewActionDecode,
.insertFp = (SdbInsertFp)mndViewActionInsert,
.updateFp = (SdbUpdateFp)mndViewActionUpdate,
.deleteFp = (SdbDeleteFp)mndViewActionDelete,
};
return sdbSetTable(pMnode->pSdb, table);
#else
return TSDB_CODE_SUCCESS;
#endif
}
void mndCleanupView(SMnode *pMnode) {
mDebug("mnd view cleanup");
}
int32_t mndProcessCreateViewReq(SRpcMsg *pReq) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#else
SCMCreateViewReq createViewReq = {0};
if (tDeserializeSCMCreateViewReq(pReq->pCont, pReq->contLen, &createViewReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("start to create view:%s, sql:%s", createViewReq.fullname, createViewReq.sql);
return mndProcessCreateViewReqImpl(&createViewReq, pReq);
#endif
}
int32_t mndProcessDropViewReq(SRpcMsg *pReq) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#else
SCMDropViewReq dropViewReq = {0};
if (tDeserializeSCMDropViewReq(pReq->pCont, pReq->contLen, &dropViewReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
mInfo("start to drop view:%s, sql:%s", dropViewReq.name, dropViewReq.sql);
return mndProcessDropViewReqImpl(&dropViewReq, pReq);
#endif
}
int32_t mndProcessGetViewMetaReq(SRpcMsg *pReq) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#else
SViewMetaReq req = {0};
if (tDeserializeSViewMetaReq(pReq->pCont, pReq->contLen, &req) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
return mndProcessViewMetaReqImpl(&req, pReq);
#endif
}
int32_t mndRetrieveView(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
#ifndef TD_ENTERPRISE
return 0;
#else
return mndRetrieveViewImpl(pReq, pShow, pBlock, rows);
#endif
}
void mndCancelGetNextView(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}

18
source/dnode/mnode/impl/test/user/user.cpp
View File

@ -250,7 +250,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
{ {
SAlterUserReq alterReq = {0}; SAlterUserReq alterReq = {0};
alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB; alterReq.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
alterReq.privileges = PRIVILEGE_TYPE_ALL；
strcpy(alterReq.user, "u3"); strcpy(alterReq.user, "u3");
strcpy(alterReq.pass, "1"); strcpy(alterReq.pass, "1");
strcpy(alterReq.dbname, "1.*"); strcpy(alterReq.dbname, "1.*");
@ -266,7 +267,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
{ {
SAlterUserReq alterReq = {0}; SAlterUserReq alterReq = {0};
alterReq.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB; alterReq.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
alterReq.privileges = PRIVILEGE_TYPE_ALL；
strcpy(alterReq.user, "u3"); strcpy(alterReq.user, "u3");
strcpy(alterReq.pass, "1"); strcpy(alterReq.pass, "1");
strcpy(alterReq.dbname, "1.*"); strcpy(alterReq.dbname, "1.*");
@ -282,7 +284,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
{ {
SAlterUserReq alterReq = {0}; SAlterUserReq alterReq = {0};
alterReq.alterType = TSDB_ALTER_USER_ADD_READ_DB; alterReq.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
alterReq.privileges = PRIVILEGE_TYPE_READ;
strcpy(alterReq.user, "u3"); strcpy(alterReq.user, "u3");
strcpy(alterReq.pass, "1"); strcpy(alterReq.pass, "1");
strcpy(alterReq.dbname, "d1"); strcpy(alterReq.dbname, "d1");
@ -329,7 +332,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
{ {
SAlterUserReq alterReq = {0}; SAlterUserReq alterReq = {0};
alterReq.alterType = TSDB_ALTER_USER_ADD_READ_DB; alterReq.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
alterReq.privileges = PRIVILEGE_TYPE_READ;
strcpy(alterReq.user, "u3"); strcpy(alterReq.user, "u3");
strcpy(alterReq.pass, "1"); strcpy(alterReq.pass, "1");
strcpy(alterReq.dbname, "1.d2"); strcpy(alterReq.dbname, "1.d2");
@ -345,7 +349,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
{ {
SAlterUserReq alterReq = {0}; SAlterUserReq alterReq = {0};
alterReq.alterType = TSDB_ALTER_USER_ADD_READ_DB; alterReq.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
alterReq.privileges = PRIVILEGE_TYPE_READ;
strcpy(alterReq.user, "u3"); strcpy(alterReq.user, "u3");
strcpy(alterReq.pass, "1"); strcpy(alterReq.pass, "1");
strcpy(alterReq.dbname, "1.d2"); strcpy(alterReq.dbname, "1.d2");
@ -388,7 +393,8 @@ TEST_F(MndTestUser, 03_Alter_User) {
{ {
SAlterUserReq alterReq = {0}; SAlterUserReq alterReq = {0};
alterReq.alterType = TSDB_ALTER_USER_REMOVE_READ_DB; alterReq.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
alterReq.alterType = PRIVILEGE_TYPE_READ;
strcpy(alterReq.user, "u3"); strcpy(alterReq.user, "u3");
strcpy(alterReq.pass, "1"); strcpy(alterReq.pass, "1");
strcpy(alterReq.dbname, "1.d2"); strcpy(alterReq.dbname, "1.d2");

3
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -148,7 +148,8 @@ typedef enum {
SDB_DB = 19, SDB_DB = 19,
SDB_FUNC = 20, SDB_FUNC = 20,
SDB_IDX = 21, SDB_IDX = 21,
SDB_MAX = 22 SDB_VIEW = 22,
SDB_MAX = 23
} ESdbType; } ESdbType;
typedef struct SSdbRaw { typedef struct SSdbRaw {

2
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -62,6 +62,8 @@ const char *sdbTableName(ESdbType type) {
return "func"; return "func";
case SDB_IDX: case SDB_IDX:
return "idx"; return "idx";
case SDB_VIEW:
return "view";
default: default:
return "undefine"; return "undefine";
} }

26
source/dnode/snode/src/snode.c
View File

@ -92,11 +92,13 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer
} }
} }
qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 char* p = NULL;
" child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", streamTaskGetStatus(pTask, &p);
qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms",
SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.triggerParam);
pTask->info.fillHistory, pTask->info.triggerParam);
return 0; return 0;
} }
@ -174,9 +176,15 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta);
taosWUnLockLatch(&pSnode->pMeta->lock); taosWUnLockLatch(&pSnode->pMeta->lock);
qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr,
streamGetTaskStatusStr(pTask->status.taskStatus), numOfTasks);
char* p = NULL;
streamTaskGetStatus(pTask, &p);
qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE,
pTask->id.idStr, p, numOfTasks);
EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
streamTaskHandleEvent(pTask->status.pSM, event);
streamTaskCheckDownstream(pTask); streamTaskCheckDownstream(pTask);
return 0; return 0;
} }
@ -352,10 +360,10 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) {
if (pTask != NULL) { if (pTask != NULL) {
rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage); rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
streamMetaReleaseTask(pSnode->pMeta, pTask); streamMetaReleaseTask(pSnode->pMeta, pTask);
char* p = NULL;
const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); streamTaskGetStatus(pTask, &p);
qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d",
pTask->id.idStr, pStatus, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); pTask->id.idStr, p, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
} else { } else {
rsp.status = TASK_DOWNSTREAM_NOT_READY; rsp.status = TASK_DOWNSTREAM_NOT_READY;
qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d",

12
source/dnode/vnode/src/inc/tq.h
View File

@ -126,7 +126,6 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR
int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision);
int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp,
int32_t type, int32_t vgId); int32_t type, int32_t vgId);
//int32_t tqPushDataRsp(STqHandle* pHandle, int32_t vgId);
int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId);
// tqMeta // tqMeta
@ -134,7 +133,6 @@ int32_t tqMetaOpen(STQ* pTq);
int32_t tqMetaClose(STQ* pTq); int32_t tqMetaClose(STQ* pTq);
int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle); int32_t tqMetaSaveHandle(STQ* pTq, const char* key, const STqHandle* pHandle);
int32_t tqMetaDeleteHandle(STQ* pTq, const char* key); int32_t tqMetaDeleteHandle(STQ* pTq, const char* key);
//int32_t tqMetaRestoreHandle(STQ* pTq);
int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen); int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_t vLen);
int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key);
int32_t tqMetaRestoreCheckInfo(STQ* pTq); int32_t tqMetaRestoreCheckInfo(STQ* pTq);
@ -160,8 +158,8 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname);
// tqStream // tqStream
int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver);
int32_t tqScanWal(STQ* pTq); int32_t tqScanWal(STQ* pTq);
int32_t tqCheckAndRunStreamTask(STQ* pTq); int32_t tqStartStreamTask(STQ* pTq);
int32_t tqStartStreamTasks(STQ* pTq); int32_t tqResetStreamTaskStatus(STQ* pTq);
int32_t tqStopStreamTasks(STQ* pTq); int32_t tqStopStreamTasks(STQ* pTq);
// tq util // tq util
@ -171,6 +169,12 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp*
int32_t type, int64_t sver, int64_t ever); int32_t type, int64_t sver, int64_t ever);
int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset);
void tqUpdateNodeStage(STQ* pTq, bool isLeader); void tqUpdateNodeStage(STQ* pTq, bool isLeader);
int32_t setDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id);
int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
SSDataBlock* pDataBlock, SArray* pTagArray);
#ifdef __cplusplus #ifdef __cplusplus
} }

2
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -231,7 +231,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq); int32_t tqLaunchStreamTaskAsync(STQ* pTq);
int tqCommit(STQ*); int tqCommit(STQ*);
int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd);

45
source/dnode/vnode/src/sma/smaOpen.c
View File

@ -31,21 +31,21 @@ static int32_t rsmaRestore(SSma *pSma);
} while (0) } while (0)
#define SMA_OPEN_RSMA_IMPL(v, l, force) \ #define SMA_OPEN_RSMA_IMPL(v, l, force) \
do { \ do { \
SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \ SRetention *r = (SRetention *)VND_RETENTIONS(v) + l; \
if (!RETENTION_VALID(r)) { \ if (!RETENTION_VALID(l, r)) { \
if (l == 0) { \ if (l == 0) { \
code = TSDB_CODE_INVALID_PARA; \ code = TSDB_CODE_INVALID_PARA; \
TSDB_CHECK_CODE(code, lino, _exit); \ TSDB_CHECK_CODE(code, lino, _exit); \
} \ } \
break; \ break; \
} \ } \
code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \ code = smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l); \
TSDB_CHECK_CODE(code, lino, _exit); \ TSDB_CHECK_CODE(code, lino, _exit); \
if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force) < 0) { \ if (tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force) < 0) { \
code = terrno; \ code = terrno; \
TSDB_CHECK_CODE(code, lino, _exit); \ TSDB_CHECK_CODE(code, lino, _exit); \
} \ } \
} while (0) } while (0)
/** /**
@ -79,20 +79,18 @@ static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t p
freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE); freqDuration = convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE);
keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE); keepDuration = convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE);
int32_t nFreqTimes = (r + level)->freq / (r + TSDB_RETENTION_L0)->freq; int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level
days *= (nFreqTimes > 1 ? nFreqTimes : 1); days *= (nFreqTimes > 1 ? nFreqTimes : 1);
if (days > keepDuration) {
days = keepDuration;
}
if (days > TSDB_MAX_DURATION_PER_FILE) {
days = TSDB_MAX_DURATION_PER_FILE;
}
if (days < freqDuration) { if (days < freqDuration) {
days = freqDuration; days = freqDuration;
} }
int32_t maxKeepDuration = TMIN(keepDuration, TSDB_MAX_DURATION_PER_FILE);
if (days > maxKeepDuration) {
days = maxKeepDuration;
}
_exit: _exit:
smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration); smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, days, duration);
return days; return days;
@ -157,6 +155,7 @@ int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) {
_exit: _exit:
if (code) { if (code) {
smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
terrno = code;
} }
return code; return code;
} }

173
source/dnode/vnode/src/sma/smaTimeRange.c
View File

@ -35,8 +35,8 @@ int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) {
return code; return code;
} }
int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) { int32_t tdProcessTSmaCreate(SSma *pSma, int64_t ver, const char *msg) {
int32_t code = tdProcessTSmaCreateImpl(pSma, version, msg); int32_t code = tdProcessTSmaCreateImpl(pSma, ver, msg);
return code; return code;
} }
@ -109,7 +109,7 @@ _exit:
* @param pMsg * @param pMsg
* @return int32_t * @return int32_t
*/ */
static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t ver, const char *pMsg) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
SSmaCfg *pCfg = (SSmaCfg *)pMsg; SSmaCfg *pCfg = (SSmaCfg *)pMsg;
@ -118,7 +118,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *
if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { if (TD_VID(pSma->pVnode) == pCfg->dstVgId) {
// create tsma meta in dstVgId // create tsma meta in dstVgId
if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { if (metaCreateTSma(SMA_META(pSma), ver, pCfg) < 0) {
code = terrno; code = terrno;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -130,7 +130,7 @@ static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *
pReq.schemaRow = pCfg->schemaRow; pReq.schemaRow = pCfg->schemaRow;
pReq.schemaTag = pCfg->schemaTag; pReq.schemaTag = pCfg->schemaTag;
if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { if (metaCreateSTable(SMA_META(pSma), ver, &pReq) < 0) {
code = terrno; code = terrno;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -154,94 +154,36 @@ _exit:
return code; return code;
} }
int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *pTSchema, int64_t suid,
SSchemaWrapper *pTagSchemaWrapper, bool createTb, int64_t suid, const char *stbFullName, const char *stbFullName, SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
SBatchDeleteReq *pDeleteReq, void **ppData, int32_t *pLen) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
void *pBuf = NULL; void *pBuf = NULL;
int32_t len = 0; int32_t len = 0;
SSubmitReq2 *pReq = NULL; SSubmitReq2 *pReq = NULL;
SArray *tagArray = NULL; SArray *tagArray = NULL;
SArray *createTbArray = NULL;
SArray *pVals = NULL;
int32_t sz = taosArrayGetSize(pBlocks); int32_t numOfBlocks = taosArrayGetSize(pBlocks);
tagArray = taosArrayInit(1, sizeof(STagVal)); tagArray = taosArrayInit(1, sizeof(STagVal));
createTbArray = taosArrayInit(sz, POINTER_BYTES);
pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)); pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2));
pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));
if(!tagArray || !createTbArray || !pReq || !pReq->aSubmitTbData) { if (!tagArray || !pReq) {
code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno; code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
// create table req pReq->aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData));
if (createTb) { if (pReq->aSubmitTbData == NULL) {
for (int32_t i = 0; i < sz; ++i) { code = terrno == TSDB_CODE_SUCCESS ? TSDB_CODE_OUT_OF_MEMORY : terrno;
SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); TSDB_CHECK_CODE(code, lino, _exit);
SVCreateTbReq *pCreateTbReq = NULL;
if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
taosArrayPush(createTbArray, &pCreateTbReq);
continue;
}
if (!(pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq)))) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
};
// don't move to the end of loop as to destroy in the end of func when error occur
taosArrayPush(createTbArray, &pCreateTbReq);
// set const
pCreateTbReq->flags = 0;
pCreateTbReq->type = TSDB_CHILD_TABLE;
pCreateTbReq->ctb.suid = suid;
// set super table name
SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTbReq->ctb.stbName = taosStrdup((char *)tNameGetTableName(&name)); // taosStrdup(stbFullName);
// set tag content
taosArrayClear(tagArray);
STagVal tagVal = {
.cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.id.groupId,
};
taosArrayPush(tagArray, &tagVal);
pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
STag *pTag = NULL;
tTagNew(tagArray, 1, false, &pTag);
if (pTag == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
TSDB_CHECK_CODE(code, lino, _exit);
}
pCreateTbReq->ctb.pTag = (uint8_t *)pTag;
// set tag name
SArray *tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = {0};
strcpy(tagNameStr, "group_id");
taosArrayPush(tagName, tagNameStr);
pCreateTbReq->ctb.tagName = tagName;
// set table name
if (pDataBlock->info.parTbName[0]) {
pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
} else {
pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId);
}
}
} }
SHashObj *pTableIndexMap =
taosHashInit(numOfBlocks, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
// SSubmitTbData req // SSubmitTbData req
for (int32_t i = 0; i < sz; ++i) { for (int32_t i = 0; i < numOfBlocks; ++i) {
SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i); SSDataBlock *pDataBlock = taosArrayGet(pBlocks, i);
if (pDataBlock->info.type == STREAM_DELETE_RESULT) { if (pDataBlock->info.type == STREAM_DELETE_RESULT) {
pDeleteReq->suid = suid; pDeleteReq->suid = suid;
@ -250,62 +192,42 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
continue; continue;
} }
int32_t rows = pDataBlock->info.rows; SSubmitTbData tbData = {.suid = suid, .uid = 0, .sver = pTSchema->version, .flags = SUBMIT_REQ_AUTO_CREATE_TABLE,};
SSubmitTbData tbData = {0}; int32_t cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1;
tbData.pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, cid, pDataBlock, tagArray);
if (!(tbData.aRowP = taosArrayInit(rows, sizeof(SRow *)))) { {
code = terrno; uint64_t groupId = pDataBlock->info.id.groupId;
TSDB_CHECK_CODE(code, lino, _exit);
}
tbData.suid = suid;
tbData.uid = 0; // uid is assigned by vnode
tbData.sver = pTSchema->version;
if (createTb) { int32_t *index = taosHashGet(pTableIndexMap, &groupId, sizeof(groupId));
tbData.pCreateTbReq = taosArrayGetP(createTbArray, i); if (index == NULL) { // no data yet, append it
if (tbData.pCreateTbReq) tbData.flags = SUBMIT_REQ_AUTO_CREATE_TABLE; code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
} if (code != TSDB_CODE_SUCCESS) {
continue;
}
if (!pVals && !(pVals = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)))) { taosArrayPush(pReq->aSubmitTbData, &tbData);
taosArrayDestroy(tbData.aRowP);
code = terrno;
TSDB_CHECK_CODE(code, lino, _exit);
}
for (int32_t j = 0; j < rows; ++j) { int32_t size = (int32_t)taosArrayGetSize(pReq->aSubmitTbData) - 1;
taosArrayClear(pVals); taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
for (int32_t k = 0; k < pTSchema->numOfCols; k++) { } else {
const STColumn *pCol = &pTSchema->columns[k]; code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, "");
SColumnInfoData *pColData = taosArrayGet(pDataBlock->pDataBlock, k); if (code != TSDB_CODE_SUCCESS) {
if (colDataIsNull_s(pColData, j)) { continue;
SColVal cv = COL_VAL_NULL(pCol->colId, pCol->type); }
taosArrayPush(pVals, &cv);
} else { SSubmitTbData *pExisted = taosArrayGet(pReq->aSubmitTbData, *index);
void *data = colDataGetData(pColData, j); code = doMergeExistedRows(pExisted, &tbData, "id");
if (IS_STR_DATA_TYPE(pCol->type)) { if (code != TSDB_CODE_SUCCESS) {
SValue sv = (SValue){.nData = varDataLen(data), .pData = varDataVal(data)}; // address copy, no value continue;
SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
taosArrayPush(pVals, &cv);
} else {
SValue sv;
memcpy(&sv.val, data, tDataTypes[pCol->type].bytes);
SColVal cv = COL_VAL_VALUE(pCol->colId, pCol->type, sv);
taosArrayPush(pVals, &cv);
}
} }
} }
SRow *pRow = NULL;
if ((code = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) {
tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosArrayPush(tbData.aRowP, &pRow);
} }
taosArrayPush(pReq->aSubmitTbData, &tbData);
} }
taosHashCleanup(pTableIndexMap);
// encode // encode
tEncodeSize(tEncodeSubmitReq, pReq, len, code); tEncodeSize(tEncodeSubmitReq, pReq, len, code);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -327,11 +249,10 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema *
} }
tEncoderClear(&encoder); tEncoderClear(&encoder);
} }
_exit: _exit:
taosArrayDestroy(createTbArray);
taosArrayDestroy(tagArray); taosArrayDestroy(tagArray);
taosArrayDestroy(pVals); if (pReq != NULL) {
if (pReq) {
tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE);
taosMemoryFree(pReq); taosMemoryFree(pReq);
} }
@ -442,8 +363,8 @@ static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char
void *pSubmitReq = NULL; void *pSubmitReq = NULL;
int32_t contLen = 0; int32_t contLen = 0;
code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, &pTsmaStat->pTSma->schemaTag, true, code = smaBlockToSubmit(pSma->pVnode, (const SArray *)msg, pTsmaStat->pTSchema, pTsmaStat->pTSma->dstTbUid,
pTsmaStat->pTSma->dstTbUid, pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen); pTsmaStat->pTSma->dstTbName, &deleteReq, &pSubmitReq, &contLen);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) { if ((terrno = tsmaProcessDelReq(pSma, indexUid, &deleteReq)) != 0) {

244
source/dnode/vnode/src/tq/tq.c
View File

@ -20,6 +20,12 @@ typedef struct {
int8_t inited; int8_t inited;
} STqMgmt; } STqMgmt;
typedef struct STaskUpdateEntry {
int64_t streamId;
int32_t taskId;
int32_t transId;
} STaskUpdateEntry;
static STqMgmt tqMgmt = {0}; static STqMgmt tqMgmt = {0};
// 0: not init // 0: not init
@ -820,28 +826,29 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
} }
// sink // sink
if (pTask->outputInfo.type == TASK_OUTPUT__SMA) { STaskOutputInfo* pOutputInfo = &pTask->outputInfo;
pTask->outputInfo.smaSink.vnode = pTq->pVnode; if (pOutputInfo->type == TASK_OUTPUT__SMA) {
pTask->outputInfo.smaSink.smaSink = smaHandleRes; pOutputInfo->smaSink.vnode = pTq->pVnode;
} else if (pTask->outputInfo.type == TASK_OUTPUT__TABLE) { pOutputInfo->smaSink.smaSink = smaHandleRes;
pTask->outputInfo.tbSink.vnode = pTq->pVnode; } else if (pOutputInfo->type == TASK_OUTPUT__TABLE) {
pTask->outputInfo.tbSink.tbSinkFunc = tqSinkDataIntoDstTable; pOutputInfo->tbSink.vnode = pTq->pVnode;
pOutputInfo->tbSink.tbSinkFunc = tqSinkDataIntoDstTable;
int32_t ver1 = 1; int32_t ver1 = 1;
SMetaInfo info = {0}; SMetaInfo info = {0};
code = metaGetInfo(pTq->pVnode->pMeta, pTask->outputInfo.tbSink.stbUid, &info, NULL); code = metaGetInfo(pTq->pVnode->pMeta, pOutputInfo->tbSink.stbUid, &info, NULL);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
ver1 = info.skmVer; ver1 = info.skmVer;
} }
SSchemaWrapper* pschemaWrapper = pTask->outputInfo.tbSink.pSchemaWrapper; SSchemaWrapper* pschemaWrapper = pOutputInfo->tbSink.pSchemaWrapper;
pTask->outputInfo.tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); pOutputInfo->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1);
if (pTask->outputInfo.tbSink.pTSchema == NULL) { if (pOutputInfo->tbSink.pTSchema == NULL) {
return -1; return -1;
} }
pTask->outputInfo.tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); pOutputInfo->tbSink.pTblInfo = tSimpleHashInit(10240, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
tSimpleHashSetFreeFp(pTask->outputInfo.tbSink.pTblInfo, freePtr); tSimpleHashSetFreeFp(pOutputInfo->tbSink.pTblInfo, freePtr);
} }
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
@ -849,11 +856,11 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId); pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, &cond, pTask->id.taskId);
} }
// reset the task status from unfinished transaction // // reset the task status from unfinished transaction
if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { // if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr); // tqWarn("s-task:%s reset task status to be normal, status kept in taskMeta: Paused", pTask->id.idStr);
pTask->status.taskStatus = TASK_STATUS__NORMAL; // pTask->status.taskStatus = TASK_STATUS__READY;
} // }
streamTaskResetUpstreamStageInfo(pTask); streamTaskResetUpstreamStageInfo(pTask);
streamSetupScheduleTrigger(pTask); streamSetupScheduleTrigger(pTask);
@ -866,20 +873,23 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) {
pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer);
} }
char* p = NULL;
streamTaskGetStatus(pTask, &p);
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms", " child id:%d, level:%d, status:%s fill-history:%d, related stream task:0x%x trigger:%" PRId64 " ms",
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
pTask->info.fillHistory, (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam); (int32_t)pTask->streamTaskId.taskId, pTask->info.triggerParam);
} else { } else {
tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 tqInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64
" nextProcessVer:%" PRId64 " nextProcessVer:%" PRId64
" child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms", " child id:%d, level:%d, status:%s fill-history:%d, related fill-task:0x%x trigger:%" PRId64 " ms",
vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, vgId, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer,
pTask->info.selfChildId, pTask->info.taskLevel, streamGetTaskStatusStr(pTask->status.taskStatus), pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory,
pTask->info.fillHistory, (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam); (int32_t)pTask->hTaskInfo.id.taskId, pTask->info.triggerParam);
} }
return 0; return 0;
@ -921,9 +931,10 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage); rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); char* p = NULL;
streamTaskGetStatus(pTask, &p);
tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d",
pTask->id.idStr, pStatus, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status);
} else { } else {
rsp.status = TASK_DOWNSTREAM_NOT_READY; rsp.status = TASK_DOWNSTREAM_NOT_READY;
tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64 tqDebug("tq recv task check(taskId:0x%" PRIx64 "-0x%x not built yet) req(reqId:0x%" PRIx64
@ -957,6 +968,12 @@ int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) {
tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", tqDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d",
rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status);
if (!vnodeIsRoleLeader(pTq->pVnode)) {
tqError("vgId:%d not leader, task:0x%x not handle the check rsp, downstream:0x%x (vgId:%d)", vgId,
rsp.upstreamTaskId, rsp.downstreamTaskId, rsp.downstreamNodeId);
return code;
}
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.streamId, rsp.upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, rsp.streamId, rsp.upstreamTaskId);
if (pTask == NULL) { if (pTask == NULL) {
tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped", tqError("tq failed to locate the stream task:0x%" PRIx64 "-0x%x (vgId:%d), it may have been destroyed or stopped",
@ -1026,11 +1043,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
SStreamTask* p = streamMetaAcquireTask(pStreamMeta, streamId, taskId); SStreamTask* p = streamMetaAcquireTask(pStreamMeta, streamId, taskId);
bool restored = pTq->pVnode->restored; bool restored = pTq->pVnode->restored;
if (p != NULL && restored) { if (p != NULL && restored && p->info.fillHistory == 0) {
p->execInfo.init = taosGetTimestampMs(); EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(p)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
tqDebug("s-task:%s set the init ts:%"PRId64, p->id.idStr, p->execInfo.init); streamTaskHandleEvent(p->status.pSM, event);
streamTaskCheckDownstream(p);
} else if (!restored) { } else if (!restored) {
tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId); tqWarn("s-task:%s not launched since vnode(vgId:%d) not ready", p->id.idStr, vgId);
} }
@ -1064,7 +1079,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
// do recovery step1 // do recovery step1
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); char* pStatus = NULL;
streamTaskGetStatus(pTask, &pStatus);
// avoid multi-thread exec // avoid multi-thread exec
while(1) { while(1) {
@ -1117,8 +1133,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
streamScanHistoryData(pTask); streamScanHistoryData(pTask);
double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0; double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0;
if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE) {
int8_t status = streamTaskSetSchedStatusInActive(pTask); int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status); tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status);
atomic_store_32(&pTask->status.inScanHistorySentinel, 0); atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
@ -1127,12 +1143,11 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
} }
// the following procedure should be executed, no matter status is stop/pause or not // the following procedure should be executed, no matter status is stop/pause or not
tqDebug("s-task:%s scan-history stage(step 1) ended, elapsed time:%.2fs", id, el); tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, el);
if (pTask->info.fillHistory) { if (pTask->info.fillHistory) {
SVersionRange* pRange = NULL; SVersionRange* pRange = NULL;
SStreamTask* pStreamTask = NULL; SStreamTask* pStreamTask = NULL;
bool done = false;
// 1. get the related stream task // 1. get the related stream task
pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId); pStreamTask = streamMetaAcquireTask(pMeta, pTask->streamTaskId.streamId, pTask->streamTaskId.taskId);
@ -1151,10 +1166,10 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
} }
ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE); ASSERT(pStreamTask->info.taskLevel == TASK_LEVEL__SOURCE);
#if 0
// 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the // 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the
// stream task get ready for scan history data // stream task get ready for scan history data
while (pStreamTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) { while (streamTaskGetStatus(pStreamTask, NULL) == TASK_STATUS__SCAN_HISTORY) {
tqDebug( tqDebug(
"s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms", "s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms",
id, pTask->info.taskLevel, pStreamTask->id.idStr, streamGetTaskStatusStr(pStreamTask->status.taskStatus)); id, pTask->info.taskLevel, pStreamTask->id.idStr, streamGetTaskStatusStr(pStreamTask->status.taskStatus));
@ -1199,6 +1214,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
pStreamTask->status.keepTaskStatus = status; pStreamTask->status.keepTaskStatus = status;
pStreamTask->status.taskStatus = TASK_STATUS__HALT; pStreamTask->status.taskStatus = TASK_STATUS__HALT;
// wal scan not start yet, reset it to be the start position
nextProcessedVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader); nextProcessedVer = walReaderGetCurrentVer(pStreamTask->exec.pWalReader);
if (nextProcessedVer == -1) { if (nextProcessedVer == -1) {
nextProcessedVer = pStreamTask->dataRange.range.maxVer + 1; nextProcessedVer = pStreamTask->dataRange.range.maxVer + 1;
@ -1211,21 +1227,26 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
taosThreadMutexUnlock(&pStreamTask->lock); taosThreadMutexUnlock(&pStreamTask->lock);
break; break;
} }
#endif
streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT);
int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer;
// if it's an source task, extract the last version in wal. // if it's an source task, extract the last version in wal.
pRange = &pTask->dataRange.range; pRange = &pTask->dataRange.range;
done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer); bool done = streamHistoryTaskSetVerRangeStep2(pTask, nextProcessedVer);
pTask->execInfo.step2Start = taosGetTimestampMs(); pTask->execInfo.step2Start = taosGetTimestampMs();
if (done) { if (done) {
qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, 0.0); qDebug("s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs", id, 0.0);
streamTaskPutTranstateIntoInputQ(pTask); streamTaskPutTranstateIntoInputQ(pTask);
// streamTaskRestoreStatus(pTask);
if (pTask->status.taskStatus == TASK_STATUS__PAUSE) { // if (pTask->status.taskStatus == TASK_STATUS__PAUSE) {
pTask->status.keepTaskStatus = TASK_STATUS__NORMAL; // pTask->status.keepTaskStatus = TASK_STATUS__READY;
qDebug("s-task:%s prev status is %s, update the kept status to be:%s when after step 2", id, // qDebug("s-task:%s prev status is %s, update the kept status to be:%s when after step 2", id,
streamGetTaskStatusStr(TASK_STATUS__PAUSE), streamGetTaskStatusStr(pTask->status.keepTaskStatus)); // streamGetTaskStatusStr(TASK_STATUS__PAUSE), streamGetTaskStatusStr(pTask->status.keepTaskStatus));
} // }
streamExecTask(pTask); // exec directly streamExecTask(pTask); // exec directly
} else { } else {
@ -1245,35 +1266,24 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer, tqDebug("s-task:%s wal reader start scan WAL verRange:%" PRId64 "-%" PRId64 ", set sched-status:%d", id, dstVer,
pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE); pTask->dataRange.range.maxVer, TASK_SCHED_STATUS__INACTIVE);
/*int8_t status = */streamTaskSetSchedStatusInActive(pTask); /*int8_t status = */streamTaskSetSchedStatusInactive(pTask);
// the fill-history task starts to process data in wal, let's set it status to be normal now
if (pTask->info.fillHistory == 1 && !streamTaskShouldStop(&pTask->status)) {
streamSetStatusNormal(pTask);
}
// now the fill-history task starts to scan data from wal files.
streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE);
tqScanWalAsync(pTq, false); tqScanWalAsync(pTq, false);
} }
streamMetaReleaseTask(pMeta, pStreamTask); streamMetaReleaseTask(pMeta, pStreamTask);
} else { } else {
STimeWindow* pWindow = &pTask->dataRange.window; STimeWindow* pWindow = &pTask->dataRange.window;
ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask));
if (pTask->hTaskInfo.id.taskId == 0) { // Not update the fill-history time window until the state transfer is completed if the related fill-history task
*pWindow = (STimeWindow){INT64_MIN, INT64_MAX}; // exists.
tqDebug( tqDebug(
"s-task:%s scan-history in stream time window completed, no related fill-history task, reset the time " "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, startVer:%" PRId64
"window:%" PRId64 " - %" PRId64, ", window:%" PRId64 " - %" PRId64,
id, pWindow->skey, pWindow->ekey); id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey);
qStreamInfoResetTimewindowFilter(pTask->exec.pExecutor);
} else {
// when related fill-history task exists, update the fill-history time window only when the
// state transfer is completed.
tqDebug(
"s-task:%s scan-history in stream time window completed, now start to handle data from WAL, start "
"ver:%" PRId64 ", window:%" PRId64 " - %" PRId64,
id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey);
}
code = streamTaskScanHistoryDataComplete(pTask); code = streamTaskScanHistoryDataComplete(pTask);
} }
@ -1352,7 +1362,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
if (taskId == STREAM_EXEC_TASK_STATUS_CHECK_ID) { if (taskId == STREAM_EXEC_TASK_STATUS_CHECK_ID) {
tqCheckAndRunStreamTask(pTq); tqStartStreamTask(pTq);
return 0; return 0;
} }
@ -1362,17 +1372,16 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
} }
SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId);
if (pTask != NULL) { if (pTask != NULL) { // even in halt status, the data in inputQ must be processed
// even in halt status, the data in inputQ must be processed char* p = NULL;
int8_t st = pTask->status.taskStatus; if (streamTaskReadyToRun(pTask, &p)) {
if (st == TASK_STATUS__NORMAL || st == TASK_STATUS__SCAN_HISTORY || st == TASK_STATUS__CK) {
tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr,
pTask->chkInfo.nextProcessVer); pTask->chkInfo.nextProcessVer);
streamExecTask(pTask); streamExecTask(pTask);
} else { } else {
int8_t status = streamTaskSetSchedStatusInActive(pTask); int8_t status = streamTaskSetSchedStatusInactive(pTask);
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId, tqDebug("vgId:%d s-task:%s ignore run req since not in ready state, status:%s, sched-status:%d", vgId,
pTask->id.idStr, streamGetTaskStatusStr(st), status); pTask->id.idStr, p, status);
} }
streamMetaReleaseTask(pTq->pStreamMeta, pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@ -1446,7 +1455,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) {
SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId);
if (pTask != NULL) { if (pTask != NULL) {
// drop the related fill-history task firstly // drop the related fill-history task firstly
if (pTask->hTaskInfo.id.taskId != 0) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
STaskId* pHTaskId = &pTask->hTaskInfo.id; STaskId* pHTaskId = &pTask->hTaskInfo.id;
streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId); streamMetaUnregisterTask(pMeta, pHTaskId->streamId, pHTaskId->taskId);
tqDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId); tqDebug("vgId:%d drop fill-history task:0x%x dropped firstly", vgId, (int32_t)pHTaskId->taskId);
@ -1486,7 +1495,7 @@ int32_t tqProcessTaskPauseReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
streamTaskPause(pTask, pMeta); streamTaskPause(pTask, pMeta);
SStreamTask* pHistoryTask = NULL; SStreamTask* pHistoryTask = NULL;
if (pTask->hTaskInfo.id.taskId != 0) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); pHistoryTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId);
if (pHistoryTask == NULL) { if (pHistoryTask == NULL) {
tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64 tqError("vgId:%d process pause req, failed to acquire fill-history task:0x%" PRIx64
@ -1514,17 +1523,19 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
return -1; return -1;
} }
// todo: handle the case: resume from halt to pause/ from halt to normal/ from pause to normal streamTaskResume(pTask);
streamTaskResume(pTask, pTq->pStreamMeta); ETaskStatus status = streamTaskGetStatus(pTask, NULL);
int32_t level = pTask->info.taskLevel; int32_t level = pTask->info.taskLevel;
if (level == TASK_LEVEL__SINK) { if (level == TASK_LEVEL__SINK) {
if (status == TASK_STATUS__UNINIT) {
}
streamMetaReleaseTask(pTq->pStreamMeta, pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask);
return 0; return 0;
} }
int8_t status = pTask->status.taskStatus; if (status == TASK_STATUS__READY || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
if (status == TASK_STATUS__NORMAL || status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__CK) {
// no lock needs to secure the access of the version // no lock needs to secure the access of the version
if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) { if (igUntreated && level == TASK_LEVEL__SOURCE && !pTask->info.fillHistory) {
// discard all the data when the stream task is suspended. // discard all the data when the stream task is suspended.
@ -1537,14 +1548,18 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus); vgId, pTask->id.idStr, pTask->chkInfo.nextProcessVer, sversion, pTask->status.schedStatus);
} }
if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && if (level == TASK_LEVEL__SOURCE && pTask->info.fillHistory && status == TASK_STATUS__SCAN_HISTORY) {
pTask->status.taskStatus == TASK_STATUS__SCAN_HISTORY) {
streamStartScanHistoryAsync(pTask, igUntreated); streamStartScanHistoryAsync(pTask, igUntreated);
} else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputInfo.queue) == 0)) { } else if (level == TASK_LEVEL__SOURCE && (streamQueueGetNumOfItems(pTask->inputq.queue) == 0)) {
tqScanWalAsync(pTq, false); tqScanWalAsync(pTq, false);
} else { } else {
streamSchedExec(pTask); streamSchedExec(pTask);
} }
} else if (status == TASK_STATUS__UNINIT) {
if (pTask->info.fillHistory == 0) {
EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
streamTaskHandleEvent(pTask->status.pSM, event);
}
} }
streamMetaReleaseTask(pTq->pStreamMeta, pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask);
@ -1683,7 +1698,6 @@ FAIL:
return -1; return -1;
} }
// todo error code cannot be return, since this is invoked by an mnode-launched transaction.
int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) { int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
@ -1694,7 +1708,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
// disable auto rsp to mnode // disable auto rsp to mnode
pRsp->info.handle = NULL; pRsp->info.handle = NULL;
// todo: add counter to make sure other tasks would not be trapped in checkpoint state
SStreamCheckpointSourceReq req = {0}; SStreamCheckpointSourceReq req = {0};
if (!vnodeIsRoleLeader(pTq->pVnode)) { if (!vnodeIsRoleLeader(pTq->pVnode)) {
tqDebug("vgId:%d not leader, ignore checkpoint-source msg", vgId); tqDebug("vgId:%d not leader, ignore checkpoint-source msg", vgId);
@ -1725,6 +1738,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
} }
tDecoderClear(&decoder); tDecoderClear(&decoder);
// todo handle failure to reset from checkpoint procedure
SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId); SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.taskId);
if (pTask == NULL) { if (pTask == NULL) {
tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. it may have been destroyed already", vgId, tqError("vgId:%d failed to find s-task:0x%x, ignore checkpoint msg. it may have been destroyed already", vgId,
@ -1735,6 +1749,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// todo handle failure to reset from checkpoint procedure
// downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req. // downstream not ready, current the stream tasks are not all ready. Ignore this checkpoint req.
if (pTask->status.downstreamReady != 1) { if (pTask->status.downstreamReady != 1) {
pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id
@ -1750,8 +1765,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp)
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// todo save the checkpoint failed info
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
if (pTask->status.taskStatus == TASK_STATUS__HALT) { ETaskStatus status = streamTaskGetStatus(pTask, NULL);
if (status == TASK_STATUS__HALT || status == TASK_STATUS__PAUSE) {
qError("s-task:%s not ready for checkpoint, since it is halt, ignore this checkpoint:%" PRId64 ", set it failure", qError("s-task:%s not ready for checkpoint, since it is halt, ignore this checkpoint:%" PRId64 ", set it failure",
pTask->id.idStr, req.checkpointId); pTask->id.idStr, req.checkpointId);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
@ -1832,7 +1850,6 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead));
int32_t len = pMsg->contLen - sizeof(SMsgHead); int32_t len = pMsg->contLen - sizeof(SMsgHead);
SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS};
bool allStopped = false;
SStreamTaskNodeUpdateMsg req = {0}; SStreamTaskNodeUpdateMsg req = {0};
@ -1863,17 +1880,43 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
} }
SStreamTask* pTask = *ppTask; SStreamTask* pTask = *ppTask;
tqDebug("s-task:%s receive nodeEp update msg from mnode", pTask->id.idStr);
if (pMeta->updateInfo.transId != req.transId) {
pMeta->updateInfo.transId = req.transId;
tqDebug("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId);
// info needs to be kept till the new trans to update the nodeEp arrived.
taosHashClear(pMeta->updateInfo.pTasks);
} else {
tqDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId);
}
STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId};
void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry));
if (exist != NULL) {
tqDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId,
req.transId);
rsp.code = TSDB_CODE_SUCCESS;
taosWUnLockLatch(&pMeta->lock);
taosArrayDestroy(req.pNodeList);
return rsp.code;
}
taosWUnLockLatch(&pMeta->lock);
// the following two functions should not be executed within the scope of meta lock to avoid deadlock
streamTaskUpdateEpsetInfo(pTask, req.pNodeList); streamTaskUpdateEpsetInfo(pTask, req.pNodeList);
streamSetStatusNormal(pTask); streamTaskResetStatus(pTask);
// continue after lock the meta again
taosWLockLatch(&pMeta->lock);
SStreamTask** ppHTask = NULL; SStreamTask** ppHTask = NULL;
if (pTask->hTaskInfo.id.taskId != 0) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id));
if (ppHTask == NULL || *ppHTask == NULL) { if (ppHTask == NULL || *ppHTask == NULL) {
tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", tqError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already",
pMeta->vgId, req.taskId); pMeta->vgId, req.taskId);
CLEAR_RELATED_FILLHISTORY_TASK(pTask);
} else { } else {
tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); tqDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr);
streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList);
@ -1892,12 +1935,14 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
} }
streamTaskStop(pTask); streamTaskStop(pTask);
taosHashPut(pMeta->pUpdateTaskSet, &pTask->id, sizeof(pTask->id), NULL, 0);
// keep the already handled info
taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0);
if (ppHTask != NULL) { if (ppHTask != NULL) {
streamTaskStop(*ppHTask); streamTaskStop(*ppHTask);
tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); tqDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr);
taosHashPut(pMeta->pUpdateTaskSet, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0);
} else { } else {
tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); tqDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr);
} }
@ -1906,25 +1951,30 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
// possibly only handle the stream task. // possibly only handle the stream task.
int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta);
int32_t updateTasks = taosHashGetSize(pMeta->pUpdateTaskSet); int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks);
pMeta->startInfo.startedAfterNodeUpdate = 1; pMeta->startInfo.startAllTasksFlag = 1;
if (updateTasks < numOfTasks) { if (updateTasks < numOfTasks) {
tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId,
updateTasks, (numOfTasks - updateTasks)); updateTasks, (numOfTasks - updateTasks));
taosWUnLockLatch(&pMeta->lock); taosWUnLockLatch(&pMeta->lock);
} else { } else {
taosHashClear(pMeta->pUpdateTaskSet);
if (!pTq->pVnode->restored) { if (!pTq->pVnode->restored) {
tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId);
pMeta->startInfo.startedAfterNodeUpdate = 0; pMeta->startInfo.startAllTasksFlag = 0;
taosWUnLockLatch(&pMeta->lock); taosWUnLockLatch(&pMeta->lock);
} else { } else {
tqDebug("vgId:%d tasks are all updated and stopped, restart them", vgId); tqDebug("vgId:%d tasks are all updated and stopped, restart them", vgId);
terrno = 0; terrno = 0;
taosWUnLockLatch(&pMeta->lock);
while (streamMetaTaskInTimer(pMeta)) {
qDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId);
taosMsleep(100);
}
taosWLockLatch(&pMeta->lock);
int32_t code = streamMetaReopen(pMeta); int32_t code = streamMetaReopen(pMeta);
if (code != 0) { if (code != 0) {
tqError("vgId:%d failed to reopen stream meta", vgId); tqError("vgId:%d failed to reopen stream meta", vgId);
@ -1942,8 +1992,8 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) {
if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) {
vInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); vInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId);
tqStartStreamTasks(pTq); tqResetStreamTaskStatus(pTq);
tqCheckAndRunStreamTaskAsync(pTq); tqLaunchStreamTaskAsync(pTq);
} else { } else {
vInfo("vgId:%d, follower node not start stream tasks", vgId); vInfo("vgId:%d, follower node not start stream tasks", vgId);
} }
@ -1970,10 +2020,10 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) {
tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr);
// clear flag set during do checkpoint, and open inputQ for all upstream tasks // clear flag set during do checkpoint, and open inputQ for all upstream tasks
if (pTask->status.taskStatus == TASK_STATUS__CK) { if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) {
streamTaskClearCheckInfo(pTask); streamTaskClearCheckInfo(pTask);
taosArrayClear(pTask->pReadyMsgList); taosArrayClear(pTask->pReadyMsgList);
streamSetStatusNormal(pTask); streamTaskSetStatusReady(pTask);
} }
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -39,7 +39,7 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) {
int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta);
taosRUnLockLatch(&pTq->pStreamMeta->lock); taosRUnLockLatch(&pTq->pStreamMeta->lock);
tqDebug("handle submit, restore:%d, numOfTasks:%d", pTq->pVnode->restored, numOfTasks); // tqTrace("vgId:%d handle submit, restore:%d, numOfTasks:%d", TD_VID(pTq->pVnode), pTq->pVnode->restored, numOfTasks);
// push data for stream processing: // push data for stream processing:
// 1. the vnode has already been restored. // 1. the vnode has already been restored.

181
source/dnode/vnode/src/tq/tqSink.c
View File

@ -13,6 +13,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include <common/tmsg.h>
#include "tcommon.h" #include "tcommon.h"
#include "tmsg.h" #include "tmsg.h"
#include "tq.h" #include "tq.h"
@ -28,19 +29,19 @@ static bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks);
static int32_t tsAscendingSortFn(const void* p1, const void* p2); static int32_t tsAscendingSortFn(const void* p1, const void* p2);
static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName,
SSubmitTbData* pTableData); SSubmitTbData* pTableData);
static int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData);
static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, static int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
int64_t suid); int64_t suid);
static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks); static int32_t doBuildAndSendSubmitMsg(SVnode* pVnode, SStreamTask* pTask, SSubmitReq2* pReq, int32_t numOfBlocks);
static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen); static int32_t buildSubmitMsgImpl(SSubmitReq2* pSubmitReq, int32_t vgId, void** pMsg, int32_t* msgLen);
static int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id); static int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id);
static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo, static int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkInfo* pTableSinkInfo,
const char* dstTableName, int64_t* uid); const char* dstTableName, int64_t* uid);
static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id); static int32_t doPutIntoCache(SSHashObj* pSinkTableMap, STableSinkInfo* pTableSinkInfo, uint64_t groupId, const char* id);
static int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, const char* id);
static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid); static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
static SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock); static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags);
static SArray* createDefaultTagColName();
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid);
int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq, int32_t tqBuildDeleteReq(const char* stbFullName, const SSDataBlock* pDataBlock, SBatchDeleteReq* deleteReq,
const char* pIdStr) { const char* pIdStr) {
@ -138,61 +139,68 @@ static int32_t tqPutReqToQueue(SVnode* pVnode, SVCreateTbBatchReq* pReqs) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName, int32_t numOfTags) {
pCreateTableReq->flags = 0;
pCreateTableReq->type = TSDB_CHILD_TABLE;
pCreateTableReq->ctb.suid = suid;
// set super table name
SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
pCreateTableReq->ctb.tagNum = numOfTags;
return TSDB_CODE_SUCCESS;
}
SArray* createDefaultTagColName() {
SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
taosArrayPush(pTagColNameList, tagNameStr);
return pTagColNameList;
}
void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid) {
if (pDataBlock->info.parTbName[0]) {
pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
} else {
pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid);
}
}
static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
int64_t suid) { int64_t suid) {
tqDebug("s-task:%s build create table msg", pTask->id.idStr); tqDebug("s-task:%s build create table msg", pTask->id.idStr);
STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema; STSchema* pTSchema = pTask->outputInfo.tbSink.pTSchema;
int32_t rows = pDataBlock->info.rows; int32_t rows = pDataBlock->info.rows;
SArray* tagArray = NULL; SArray* tagArray = taosArrayInit(4, sizeof(STagVal));;
int32_t code = 0; int32_t code = 0;
SVCreateTbBatchReq reqs = {0}; SVCreateTbBatchReq reqs = {0};
SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
if (NULL == reqs.pArray) { if (NULL == reqs.pArray) {
tqError("s-task:%s failed to init create table msg, code:%s", pTask->id.idStr, tstrerror(terrno));
goto _end; goto _end;
} }
for (int32_t rowId = 0; rowId < rows; rowId++) { for (int32_t rowId = 0; rowId < rows; rowId++) {
SVCreateTbReq* pCreateTbReq = &((SVCreateTbReq){0}); SVCreateTbReq* pCreateTbReq = &((SVCreateTbReq){0});
// set const
pCreateTbReq->flags = 0;
pCreateTbReq->type = TSDB_CHILD_TABLE;
pCreateTbReq->ctb.suid = suid;
// set super table name
SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); // taosStrdup(stbFullName);
// set tag content
int32_t size = taosArrayGetSize(pDataBlock->pDataBlock); int32_t size = taosArrayGetSize(pDataBlock->pDataBlock);
if (size == 2) { int32_t numOfTags = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
tagArray = taosArrayInit(1, sizeof(STagVal));
if (!tagArray) {
tdDestroySVCreateTbReq(pCreateTbReq);
goto _end;
}
initCreateTableMsg(pCreateTbReq, suid, stbFullName, numOfTags);
taosArrayClear(tagArray);
if (size == 2) {
STagVal tagVal = { STagVal tagVal = {
.cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; .cid = pTSchema->numOfCols + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(tagArray, &tagVal); taosArrayPush(tagArray, &tagVal);
pCreateTbReq->ctb.tagName = createDefaultTagColName();
// set tag name
SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
taosArrayPush(tagName, tagNameStr);
pCreateTbReq->ctb.tagName = tagName;
} else { } else {
tagArray = taosArrayInit(size - 1, sizeof(STagVal));
if (!tagArray) {
tdDestroySVCreateTbReq(pCreateTbReq);
goto _end;
}
for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) {
SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId);
@ -209,29 +217,26 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
taosArrayPush(tagArray, &tagVal); taosArrayPush(tagArray, &tagVal);
} }
} }
pCreateTbReq->ctb.tagNum = TMAX(size - UD_TAG_COLUMN_INDEX, 1);
STag* pTag = NULL; tTagNew(tagArray, 1, false, (STag**)&pCreateTbReq->ctb.pTag);
tTagNew(tagArray, 1, false, &pTag);
tagArray = taosArrayDestroy(tagArray); tagArray = taosArrayDestroy(tagArray);
if (pTag == NULL) { if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq); tdDestroySVCreateTbReq(pCreateTbReq);
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _end; goto _end;
} }
pCreateTbReq->ctb.pTag = (uint8_t*)pTag; uint64_t gid = pDataBlock->info.id.groupId;
if (taosArrayGetSize(pDataBlock->pDataBlock) > UD_GROUPID_COLUMN_INDEX) {
// set table name
if (!pDataBlock->info.parTbName[0]) {
SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX); SColumnInfoData* pGpIdColInfo = taosArrayGet(pDataBlock->pDataBlock, UD_GROUPID_COLUMN_INDEX);
// todo remove this
void* pGpIdData = colDataGetData(pGpIdColInfo, rowId); void* pGpIdData = colDataGetData(pGpIdColInfo, rowId);
pCreateTbReq->name = buildCtbNameByGroupId(stbFullName, *(uint64_t*)pGpIdData); ASSERT(gid == *(int64_t*)pGpIdData);
} else {
pCreateTbReq->name = taosStrdup(pDataBlock->info.parTbName);
} }
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid);
taosArrayPush(reqs.pArray, pCreateTbReq); taosArrayPush(reqs.pArray, pCreateTbReq);
tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name); tqDebug("s-task:%s build create table:%s msg complete", pTask->id.idStr, pCreateTbReq->name);
} }
@ -330,6 +335,9 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
tqTrace("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id, tqTrace("s-task:%s rows merged, final rows:%d, uid:%" PRId64 ", existed auto-create table:%d, new-block:%d", id,
(int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (pNew->pCreateTbReq != NULL)); (int32_t)taosArrayGetSize(pFinal), pExisted->uid, (pExisted->pCreateTbReq != NULL), (pNew->pCreateTbReq != NULL));
tdDestroySVCreateTbReq(pNew->pCreateTbReq);
taosMemoryFree(pNew->pCreateTbReq);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -390,60 +398,33 @@ bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbNam
return true; return true;
} }
SVCreateTbReq* buildAutoCreateTableReq(char* stbFullName, int64_t suid, int32_t numOfCols, SSDataBlock* pDataBlock) { SVCreateTbReq* buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t numOfCols,
char* ctbName = pDataBlock->info.parTbName; SSDataBlock* pDataBlock, SArray* pTagArray) {
SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
SVCreateTbReq* pCreateTbReq = taosMemoryCalloc(1, sizeof(SVCreateStbReq));
if (pCreateTbReq == NULL) { if (pCreateTbReq == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
// set tag content taosArrayClear(pTagArray);
SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); initCreateTableMsg(pCreateTbReq, suid, stbFullName, 1);
if (tagArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq);
return NULL;
}
// set const
pCreateTbReq->flags = 0;
pCreateTbReq->type = TSDB_CHILD_TABLE;
pCreateTbReq->ctb.suid = suid;
// set super table name
SName name = {0};
tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
pCreateTbReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
STagVal tagVal = { .cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId}; STagVal tagVal = { .cid = numOfCols, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = pDataBlock->info.id.groupId};
taosArrayPush(tagArray, &tagVal); taosArrayPush(pTagArray, &tagVal);
pCreateTbReq->ctb.tagNum = taosArrayGetSize(tagArray);
STag* pTag = NULL; tTagNew(pTagArray, 1, false, (STag**) &pCreateTbReq->ctb.pTag);
tTagNew(tagArray, 1, false, &pTag);
taosArrayDestroy(tagArray);
if (pTag == NULL) { if (pCreateTbReq->ctb.pTag == NULL) {
tdDestroySVCreateTbReq(pCreateTbReq); tdDestroySVCreateTbReq(pCreateTbReq);
taosMemoryFreeClear(pCreateTbReq); taosMemoryFreeClear(pCreateTbReq);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
pCreateTbReq->ctb.pTag = (uint8_t*)pTag; pCreateTbReq->ctb.tagName = createDefaultTagColName();
// set tag name
SArray* tagName = taosArrayInit(1, TSDB_COL_NAME_LEN);
char k[TSDB_COL_NAME_LEN] = "group_id";
taosArrayPush(tagName, k);
pCreateTbReq->ctb.tagName = tagName;
// set table name // set table name
pCreateTbReq->name = taosStrdup(ctbName); setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId);
return pCreateTbReq; return pCreateTbReq;
} }
@ -514,7 +495,7 @@ int32_t tsAscendingSortFn(const void* p1, const void* p2) {
} }
} }
int32_t doConvertRows(SSubmitTbData* pTableData, STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) { int32_t doConvertRows(SSubmitTbData* pTableData, const STSchema* pTSchema, SSDataBlock* pDataBlock, const char* id) {
int32_t numOfRows = pDataBlock->info.rows; int32_t numOfRows = pDataBlock->info.rows;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -592,7 +573,7 @@ int32_t doWaitForDstTableCreated(SVnode* pVnode, SStreamTask* pTask, STableSinkI
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
while (pTableSinkInfo->uid == 0) { while (pTableSinkInfo->uid == 0) {
if (streamTaskShouldStop(&pTask->status)) { if (streamTaskShouldStop(pTask)) {
tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName); tqDebug("s-task:%s task will stop, quit from waiting for table:%s create", id, dstTableName);
return TSDB_CODE_STREAM_EXEC_CANCELLED; return TSDB_CODE_STREAM_EXEC_CANCELLED;
} }
@ -693,8 +674,13 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName);
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
pTableData->pCreateTbReq = buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock); pTableData->pCreateTbReq =
buildAutoCreateTableReq(stbFullName, suid, pTSchema->numOfCols + 1, pDataBlock, pTagArray);
taosArrayDestroy(pTagArray);
if (pTableData->pCreateTbReq == NULL) { if (pTableData->pCreateTbReq == NULL) {
tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno)); tqError("s-task:%s failed to build auto create table req, code:%s", id, tstrerror(terrno));
taosMemoryFree(pTableSinkInfo); taosMemoryFree(pTableSinkInfo);
@ -724,17 +710,16 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t setDstTableDataPayload(SStreamTask* pTask, int32_t blockIndex, SSDataBlock* pDataBlock, int32_t setDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData) { SSubmitTbData* pTableData, const char* id) {
int32_t numOfRows = pDataBlock->info.rows; int32_t numOfRows = pDataBlock->info.rows;
const char* id = pTask->id.idStr;
tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64,
id, blockIndex + 1, numOfRows, pTask->outputInfo.tbSink.stbUid); id, blockIndex + 1, numOfRows, suid);
char* dstTableName = pDataBlock->info.parTbName; char* dstTableName = pDataBlock->info.parTbName;
// convert all rows // convert all rows
int32_t code = doConvertRows(pTableData, pTask->outputInfo.tbSink.pTSchema, pDataBlock, id); int32_t code = doConvertRows(pTableData, pTSchema, pDataBlock, id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno)); tqError("s-task:%s failed to convert rows from result block, code:%s", id, tstrerror(terrno));
return code; return code;
@ -773,7 +758,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
numOfBlocks); numOfBlocks);
for(int32_t i = 0; i < numOfBlocks; ++i) { for(int32_t i = 0; i < numOfBlocks; ++i) {
if (streamTaskShouldStop(&pTask->status)) { if (streamTaskShouldStop(pTask)) {
return; return;
} }
@ -800,7 +785,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
continue; continue;
} }
code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
continue; continue;
} }
@ -823,7 +808,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
bool hasSubmit = false; bool hasSubmit = false;
for (int32_t i = 0; i < numOfBlocks; i++) { for (int32_t i = 0; i < numOfBlocks; i++) {
if (streamTaskShouldStop(&pTask->status)) { if (streamTaskShouldStop(pTask)) {
taosHashCleanup(pTableIndexMap); taosHashCleanup(pTableIndexMap);
tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE);
return; return;
@ -847,7 +832,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
continue; continue;
} }
code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
continue; continue;
} }
@ -857,7 +842,7 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data) {
int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1; int32_t size = (int32_t)taosArrayGetSize(submitReq.aSubmitTbData) - 1;
taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size)); taosHashPut(pTableIndexMap, &groupId, sizeof(groupId), &size, sizeof(size));
} else { } else {
code = setDstTableDataPayload(pTask, i, pDataBlock, &tbData); code = setDstTableDataPayload(suid, pTSchema, i, pDataBlock, &tbData, id);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
continue; continue;
} }

55
source/dnode/vnode/src/tq/tqStreamTask.c
View File

@ -60,7 +60,7 @@ int32_t tqScanWal(STQ* pTq) {
return 0; return 0;
} }
int32_t tqCheckAndRunStreamTask(STQ* pTq) { int32_t tqStartStreamTask(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
@ -92,19 +92,19 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
} }
if (pTask->status.downstreamReady == 1) { if (pTask->status.downstreamReady == 1) {
tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", if (HAS_RELATED_FILLHISTORY_TASK(pTask)) {
pTask->id.idStr); tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task",
streamLaunchFillHistoryTask(pTask); pTask->id.idStr);
streamLaunchFillHistoryTask(pTask);
}
streamMetaUpdateTaskReadyInfo(pTask);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
continue; continue;
} }
pTask->execInfo.init = taosGetTimestampMs(); EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT;
tqDebug("s-task:%s start check downstream tasks, set the init ts:%"PRId64, pTask->id.idStr, pTask->execInfo.init); streamTaskHandleEvent(pTask->status.pSM, event);
streamSetStatusNormal(pTask);
streamTaskCheckDownstream(pTask);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
} }
@ -112,9 +112,9 @@ int32_t tqCheckAndRunStreamTask(STQ* pTq) {
return 0; return 0;
} }
int32_t tqCheckAndRunStreamTaskAsync(STQ* pTq) { int32_t tqLaunchStreamTaskAsync(STQ* pTq) {
int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t vgId = pMeta->vgId;
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
if (numOfTasks == 0) { if (numOfTasks == 0) {
@ -228,7 +228,7 @@ int32_t tqStopStreamTasks(STQ* pTq) {
return 0; return 0;
} }
int32_t tqStartStreamTasks(STQ* pTq) { int32_t tqResetStreamTaskStatus(STQ* pTq) {
SStreamMeta* pMeta = pTq->pStreamMeta; SStreamMeta* pMeta = pTq->pStreamMeta;
int32_t vgId = TD_VID(pTq->pVnode); int32_t vgId = TD_VID(pTq->pVnode);
int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList);
@ -243,11 +243,7 @@ int32_t tqStartStreamTasks(STQ* pTq) {
STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId};
SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
streamTaskResetStatus(*pTask);
int8_t status = (*pTask)->status.taskStatus;
if (status == TASK_STATUS__STOP && (*pTask)->info.fillHistory != 1) {
streamSetStatusNormal(*pTask);
}
} }
return 0; return 0;
@ -328,15 +324,17 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
} }
// not in ready state, do not handle the data from wal // not in ready state, do not handle the data from wal
int32_t status = pTask->status.taskStatus; // int32_t status = pTask->status.taskStatus;
if (status != TASK_STATUS__NORMAL) { char* p = NULL;
tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, streamGetTaskStatusStr(status)); int32_t status = streamTaskGetStatus(pTask, &p);
if (streamTaskGetStatus(pTask, &p) != TASK_STATUS__READY) {
tqTrace("s-task:%s not ready for submit block in wal, status:%s", pTask->id.idStr, p);
return false; return false;
} }
// fill-history task has entered into the last phase, no need to anything // fill-history task has entered into the last phase, no need to anything
if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) { if ((pTask->info.fillHistory == 1) && pTask->status.appendTranstateBlock) {
ASSERT(status == TASK_STATUS__NORMAL); ASSERT(status == TASK_STATUS__READY);
// the maximum version of data in the WAL has reached already, the step2 is done // the maximum version of data in the WAL has reached already, the step2 is done
tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr, tqDebug("s-task:%s fill-history reach the maximum ver:%" PRId64 ", not scan wal anymore", pTask->id.idStr,
pTask->dataRange.range.maxVer); pTask->dataRange.range.maxVer);
@ -344,13 +342,13 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) {
} }
// check if input queue is full or not // check if input queue is full or not
if (streamQueueIsFull(pTask->inputInfo.queue)) { if (streamQueueIsFull(pTask->inputq.queue)) {
tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr); tqTrace("s-task:%s input queue is full, do nothing", pTask->id.idStr);
return false; return false;
} }
// the input queue of downstream task is full, so the output is blocked, stopped for a while // the input queue of downstream task is full, so the output is blocked, stopped for a while
if (pTask->inputInfo.status == TASK_INPUT_STATUS__BLOCKED) { if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) {
tqDebug("s-task:%s inputQ is blocked, do nothing", pTask->id.idStr); tqDebug("s-task:%s inputQ is blocked, do nothing", pTask->id.idStr);
return false; return false;
} }
@ -444,14 +442,15 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) {
continue; continue;
} }
int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputInfo.queue); int32_t numOfItems = streamQueueGetNumOfItems(pTask->inputq.queue);
int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX; int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX;
taosThreadMutexLock(&pTask->lock); taosThreadMutexLock(&pTask->lock);
const char* pStatus = streamGetTaskStatusStr(pTask->status.taskStatus); char* p = NULL;
if (pTask->status.taskStatus != TASK_STATUS__NORMAL) { ETaskStatus status = streamTaskGetStatus(pTask, &p);
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, pStatus); if (status != TASK_STATUS__READY) {
tqDebug("s-task:%s not ready for submit block from wal, status:%s", pTask->id.idStr, p);
taosThreadMutexUnlock(&pTask->lock); taosThreadMutexUnlock(&pTask->lock);
streamMetaReleaseTask(pStreamMeta, pTask); streamMetaReleaseTask(pStreamMeta, pTask);
continue; continue;

55
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -1200,11 +1200,28 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p
} }
} }
static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SFileDataBlockInfo* pBlock) { static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, SLastBlockReader* pLastBlockReader, int32_t order) {
bool ascScan = ASCENDING_TRAVERSE(order); bool ascScan = ASCENDING_TRAVERSE(order);
bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);
return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->record.firstKey)) || int64_t key = 0;
(!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->record.lastKey)); if (bHasDataInLastBlock) {
int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt);
} else {
key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey;
}
return key;
}
static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock,
SLastBlockReader* pLastBlockReader, int32_t order) {
bool ascScan = ASCENDING_TRAVERSE(order);
int64_t key = getBoarderKeyInFiles(pBlock, pLastBlockReader, order);
return (ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts < key)) ||
(!ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts > key));
} }
static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) { static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) {
@ -2667,6 +2684,15 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) {
} }
} }
static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, SLastBlockReader* pLastBlockReader, bool asc) {
if(!hasDataInLastBlock(pLastBlockReader)) {
return true;
} else {
int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt);
}
}
static int32_t doBuildDataBlock(STsdbReader* pReader) { static int32_t doBuildDataBlock(STsdbReader* pReader) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
@ -2702,17 +2728,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
// build composed data block // build composed data block
code = buildComposedDataBlock(pReader); code = buildComposedDataBlock(pReader);
} else if (bufferDataInFileBlockGap(pReader->info.order, keyInBuf, pBlockInfo)) { } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pLastBlockReader, pReader->info.order)) {
// data in memory that are earlier than current file block // data in memory that are earlier than current file block and stt blocks
// rows in buffer should be less than the file block in asc, greater than file block in desc // rows in buffer should be less than the file block in asc, greater than file block in desc
int64_t endKey = int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pLastBlockReader, pReader->info.order);
(ASCENDING_TRAVERSE(pReader->info.order)) ? pBlockInfo->record.firstKey : pBlockInfo->record.lastKey;
code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
} else { } else {
bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader); if (notOverlapWithSttFiles(pBlockInfo, pLastBlockReader, asc)) {
int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN;
if (!bHasDataInLastBlock ||
((asc && pBlockInfo->record.lastKey < tsLast) || (!asc && pBlockInfo->record.firstKey > tsLast))) {
// whole block is required, return it directly // whole block is required, return it directly
SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
pInfo->rows = pBlockInfo->record.numRow; pInfo->rows = pBlockInfo->record.numRow;
@ -2723,7 +2745,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
// update the last key for the corresponding table // update the last key for the corresponding table
pScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? pInfo->window.ekey : pInfo->window.skey; pScanInfo->lastKey = asc ? pInfo->window.ekey : pInfo->window.skey;
tsdbDebug("%p uid:%" PRIu64 tsdbDebug("%p uid:%" PRIu64
" clean file block retrieved from file, global index:%d, " " clean file block retrieved from file, global index:%d, "
"table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
@ -2750,10 +2772,11 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
} }
// data in stt now overlaps with current active file data block, need to composed with file data block. // data in stt now overlaps with current active file data block, need to composed with file data block.
int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader); int64_t lastKeyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
if ((keyInStt >= pBlockInfo->record.firstKey && asc) || (keyInStt <= pBlockInfo->record.lastKey && (!asc))) { if ((lastKeyInStt >= pBlockInfo->record.firstKey && asc) ||
tsdbDebug("%p keyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader, (lastKeyInStt <= pBlockInfo->record.lastKey && (!asc))) {
keyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr); tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
lastKeyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
break; break;
} }
} }

2
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -106,7 +106,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep1", pCfg->tsdbCfg.keep1) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keep2", pCfg->tsdbCfg.keep2) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "keepTimeOffset", pCfg->tsdbCfg.keepTimeOffset) < 0) return -1;
if (pCfg->tsdbCfg.retentions[0].freq > 0) { if (pCfg->tsdbCfg.retentions[0].keep > 0) {
int32_t nRetention = 1; int32_t nRetention = 1;
if (pCfg->tsdbCfg.retentions[1].freq > 0) { if (pCfg->tsdbCfg.retentions[1].freq > 0) {
++nRetention; ++nRetention;

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -585,7 +585,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
} }
} break; } break;
case TDMT_VND_STREAM_TASK_RESET: { case TDMT_VND_STREAM_TASK_RESET: {
if (pVnode->restored/* && vnodeIsLeader(pVnode)*/) { if (pVnode->restored && vnodeIsLeader(pVnode)) {
tqProcessTaskResetReq(pVnode->pTq, pMsg); tqProcessTaskResetReq(pVnode->pTq, pMsg);
} }
} break; } break;

6
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -555,7 +555,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
pVnode->restored = true; pVnode->restored = true;
taosWLockLatch(&pVnode->pTq->pStreamMeta->lock); taosWLockLatch(&pVnode->pTq->pStreamMeta->lock);
if (pVnode->pTq->pStreamMeta->startInfo.startedAfterNodeUpdate) { if (pVnode->pTq->pStreamMeta->startInfo.startAllTasksFlag) {
vInfo("vgId:%d, sync restore finished, stream tasks will be launched by other thread", vgId); vInfo("vgId:%d, sync restore finished, stream tasks will be launched by other thread", vgId);
taosWUnLockLatch(&pVnode->pTq->pStreamMeta->lock); taosWUnLockLatch(&pVnode->pTq->pStreamMeta->lock);
return; return;
@ -567,8 +567,8 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx)
vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId);
} else { } else {
vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId);
tqStartStreamTasks(pVnode->pTq); tqResetStreamTaskStatus(pVnode->pTq);
tqCheckAndRunStreamTaskAsync(pVnode->pTq); tqLaunchStreamTaskAsync(pVnode->pTq);
} }
} else { } else {
vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId);

90
source/libs/catalog/inc/catalogInt.h
View File

@ -30,6 +30,7 @@ extern "C" {
#define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100
#define CTG_DEFAULT_CACHE_DB_NUMBER 20 #define CTG_DEFAULT_CACHE_DB_NUMBER 20
#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000 #define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000
#define CTG_DEFAULT_CACHE_VIEW_NUMBER 256
#define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SECOND 10
#define CTG_DEFAULT_RENT_SLOT_SIZE 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10
#define CTG_DEFAULT_MAX_RETRY_TIMES 3 #define CTG_DEFAULT_MAX_RETRY_TIMES 3
@ -67,6 +68,7 @@ typedef enum {
CTG_CI_USER, CTG_CI_USER,
CTG_CI_UDF, CTG_CI_UDF,
CTG_CI_SVR_VER, CTG_CI_SVR_VER,
CTG_CI_VIEW,
CTG_CI_MAX_VALUE, CTG_CI_MAX_VALUE,
} CTG_CACHE_ITEM; } CTG_CACHE_ITEM;
@ -82,6 +84,7 @@ enum {
enum { enum {
CTG_RENT_DB = 1, CTG_RENT_DB = 1,
CTG_RENT_STABLE, CTG_RENT_STABLE,
CTG_RENT_VIEW,
}; };
enum { enum {
@ -96,6 +99,8 @@ enum {
CTG_OP_UPDATE_VG_EPSET, CTG_OP_UPDATE_VG_EPSET,
CTG_OP_UPDATE_TB_INDEX, CTG_OP_UPDATE_TB_INDEX,
CTG_OP_DROP_TB_INDEX, CTG_OP_DROP_TB_INDEX,
CTG_OP_UPDATE_VIEW_META,
CTG_OP_DROP_VIEW_META,
CTG_OP_CLEAR_CACHE, CTG_OP_CLEAR_CACHE,
CTG_OP_MAX CTG_OP_MAX
}; };
@ -117,6 +122,7 @@ typedef enum {
CTG_TASK_GET_TB_META_BATCH, CTG_TASK_GET_TB_META_BATCH,
CTG_TASK_GET_TB_HASH_BATCH, CTG_TASK_GET_TB_HASH_BATCH,
CTG_TASK_GET_TB_TAG, CTG_TASK_GET_TB_TAG,
CTG_TASK_GET_VIEW,
} CTG_TASK_TYPE; } CTG_TASK_TYPE;
typedef enum { typedef enum {
@ -146,6 +152,7 @@ typedef struct SCtgAuthReq {
SGetUserAuthRsp authInfo; SGetUserAuthRsp authInfo;
AUTH_TYPE singleType; AUTH_TYPE singleType;
bool onlyCache; bool onlyCache;
bool tbNotExists;
} SCtgAuthReq; } SCtgAuthReq;
typedef struct SCtgAuthRsp { typedef struct SCtgAuthRsp {
@ -238,8 +245,17 @@ typedef struct SCtgUdfCtx {
typedef struct SCtgUserCtx { typedef struct SCtgUserCtx {
SUserAuthInfo user; SUserAuthInfo user;
int32_t subTaskCode;
} SCtgUserCtx; } SCtgUserCtx;
typedef struct SCtgViewsCtx {
int32_t fetchNum;
SArray* pNames;
SArray* pResList;
SArray* pFetchs;
} SCtgViewsCtx;
typedef STableIndexRsp STableIndex; typedef STableIndexRsp STableIndex;
typedef struct SCtgTbCache { typedef struct SCtgTbCache {
@ -259,12 +275,19 @@ typedef struct SCtgCfgCache {
SDbCfgInfo* cfgInfo; SDbCfgInfo* cfgInfo;
} SCtgCfgCache; } SCtgCfgCache;
typedef struct SCtgViewCache {
SRWLatch viewLock;
SViewMeta* pMeta;
} SCtgViewCache;
typedef struct SCtgDBCache { typedef struct SCtgDBCache {
SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads
uint64_t dbId; uint64_t dbId;
int8_t deleted; int8_t deleted;
SCtgVgCache vgCache; SCtgVgCache vgCache;
SCtgCfgCache cfgCache; SCtgCfgCache cfgCache;
SHashObj* viewCache; // key:viewname, value:SCtgViewCache
SHashObj* tbCache; // key:tbname, value:SCtgTbCache SHashObj* tbCache; // key:tbname, value:SCtgTbCache
SHashObj* stbCache; // key:suid, value:char* SHashObj* stbCache; // key:suid, value:char*
uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; uint64_t dbCacheNum[CTG_CI_MAX_VALUE];
@ -294,13 +317,15 @@ typedef struct SCtgUserAuth {
} SCtgUserAuth; } SCtgUserAuth;
typedef struct SCatalog { typedef struct SCatalog {
uint64_t clusterId; uint64_t clusterId;
bool stopUpdate; bool stopUpdate;
SHashObj* userCache; // key:user, value:SCtgUserAuth SDynViewVersion dynViewVer;
SHashObj* dbCache; // key:dbname, value:SCtgDBCache SHashObj* userCache; // key:user, value:SCtgUserAuth
SCtgRentMgmt dbRent; SHashObj* dbCache; // key:dbname, value:SCtgDBCache
SCtgRentMgmt stbRent; SCtgRentMgmt dbRent;
SCtgCacheStat cacheStat; SCtgRentMgmt stbRent;
SCtgRentMgmt viewRent;
SCtgCacheStat cacheStat;
} SCatalog; } SCatalog;
typedef struct SCtgBatch { typedef struct SCtgBatch {
@ -344,6 +369,7 @@ typedef struct SCtgJob {
int32_t tbIndexNum; int32_t tbIndexNum;
int32_t tbCfgNum; int32_t tbCfgNum;
int32_t svrVerNum; int32_t svrVerNum;
int32_t viewNum;
} SCtgJob; } SCtgJob;
typedef struct SCtgMsgCtx { typedef struct SCtgMsgCtx {
@ -509,6 +535,20 @@ typedef struct SCtgUpdateEpsetMsg {
SEpSet epSet; SEpSet epSet;
} SCtgUpdateEpsetMsg; } SCtgUpdateEpsetMsg;
typedef struct SCtgUpdateViewMetaMsg {
SCatalog* pCtg;
SViewMetaRsp* pRsp;
} SCtgUpdateViewMetaMsg;
typedef struct SCtgDropViewMetaMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char viewName[TSDB_VIEW_NAME_LEN];
uint64_t dbId;
uint64_t viewId;
} SCtgDropViewMetaMsg;
typedef struct SCtgCacheOperation { typedef struct SCtgCacheOperation {
int32_t opId; int32_t opId;
void* data; void* data;
@ -686,10 +726,10 @@ typedef struct SCtgCacheItemInfo {
(CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || \ (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || \
(CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE))
#define CTG_IS_BATCH_TASK(_taskType) ((CTG_TASK_GET_TB_META_BATCH == (_taskType)) || (CTG_TASK_GET_TB_HASH_BATCH == (_taskType)) || (CTG_TASK_GET_VIEW == (_taskType)))
#define CTG_GET_TASK_MSGCTX(_task, _id) \ #define CTG_GET_TASK_MSGCTX(_task, _id) \
(((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) \ (CTG_IS_BATCH_TASK((_task)->type) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx)
? taosArrayGet((_task)->msgCtxs, (_id)) \
: &(_task)->msgCtx)
#define CTG_META_SIZE(pMeta) \ #define CTG_META_SIZE(pMeta) \
(sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema))
@ -870,6 +910,7 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation* action);
int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action);
int32_t ctgOpDropStbMeta(SCtgCacheOperation* action); int32_t ctgOpDropStbMeta(SCtgCacheOperation* action);
int32_t ctgOpDropTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropTbMeta(SCtgCacheOperation* action);
int32_t ctgOpDropViewMeta(SCtgCacheOperation* action);
int32_t ctgOpUpdateUser(SCtgCacheOperation* action); int32_t ctgOpUpdateUser(SCtgCacheOperation* action);
int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation); int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation);
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache);
@ -879,7 +920,7 @@ int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char* dbFName, char* tbName, int32
int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType, int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType,
uint64_t* suid, char* stbName); uint64_t* suid, char* stbName);
int32_t ctgChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pReq, bool* inCache, SCtgAuthRsp* pRes); int32_t ctgChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pReq, bool tbNotExists, bool* inCache, SCtgAuthRsp* pRes);
int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId);
int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char* dbFName, bool syncReq); int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char* dbFName, bool syncReq);
int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* stbName, uint64_t suid, int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* stbName, uint64_t suid,
@ -891,11 +932,22 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput* output, bool sy
int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp* pAuth, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp* pAuth, bool syncReq);
int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char* dbFName, int32_t vgId, SEpSet* pEpSet); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char* dbFName, int32_t vgId, SEpSet* pEpSet);
int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex** pIndex, bool syncOp); int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex** pIndex, bool syncOp);
int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp);
int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool clearMeta, bool freeCtg, bool stopQueue, bool syncOp); int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool clearMeta, bool freeCtg, bool stopQueue, bool syncOp);
int32_t ctgMetaRentInit(SCtgRentMgmt* mgmt, uint32_t rentSec, int8_t type, int32_t size); int32_t ctgMetaRentInit(SCtgRentMgmt* mgmt, uint32_t rentSec, int8_t type, int32_t size);
int32_t ctgMetaRentAdd(SCtgRentMgmt* mgmt, void* meta, int64_t id, int32_t size); int32_t ctgMetaRentAdd(SCtgRentMgmt* mgmt, void* meta, int64_t id, int32_t size);
int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare,
__compar_fn_t searchCompare);
int32_t ctgMetaRentGet(SCtgRentMgmt* mgmt, void** res, uint32_t* num, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt* mgmt, void** res, uint32_t* num, int32_t size);
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare);
void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache);
void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache);
int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid,
SCtgTbCache *pCache);
int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId,
SCtgViewCache *pCache);
int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq);
int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq);
int32_t ctgStartUpdateThread(); int32_t ctgStartUpdateThread();
int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask);
void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache);
@ -904,9 +956,11 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp);
int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation);
int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation);
int32_t ctgOpClearCache(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation);
int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation);
int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType);
int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup);
int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx,
int32_t *fetchIdx, int32_t baseResIdx, SArray *pList);
int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target);
int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildUseDBInput* input, SUseDbOutput* out, int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildUseDBInput* input, SUseDbOutput* out,
SCtgTaskReq* tReq); SCtgTaskReq* tReq);
@ -932,6 +986,8 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S
int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** out, int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** out,
SCtgTask* pTask); SCtgTask* pTask);
int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** out, SCtgTask* pTask); int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** out, SCtgTask* pTask);
int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pName, SViewMetaOutput* out,
SCtgTaskReq* tReq);
int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs); int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs);
int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp,
@ -942,6 +998,7 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f
int32_t ctgGetTbCfgCb(SCtgTask* pTask); int32_t ctgGetTbCfgCb(SCtgTask* pTask);
void ctgFreeHandle(SCatalog* pCatalog); void ctgFreeHandle(SCatalog* pCatalog);
void ctgFreeSViewMeta(SViewMeta* pMeta);
void ctgFreeMsgSendParam(void* param); void ctgFreeMsgSendParam(void* param);
void ctgFreeBatch(SCtgBatch* pBatch); void ctgFreeBatch(SCtgBatch* pBatch);
void ctgFreeBatchs(SHashObj* pBatchs); void ctgFreeBatchs(SHashObj* pBatchs);
@ -958,9 +1015,11 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam
void ctgResetTbMetaTask(SCtgTask* pTask); void ctgResetTbMetaTask(SCtgTask* pTask);
void ctgFreeDbCache(SCtgDBCache* dbCache); void ctgFreeDbCache(SCtgDBCache* dbCache);
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2);
int32_t ctgViewVersionSortCompare(const void* key1, const void* key2);
int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2);
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2);
int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2);
int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2);
void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput);
int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target);
int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target);
@ -974,8 +1033,9 @@ void ctgClearSubTaskRes(SCtgSubRes* pRes);
void ctgFreeQNode(SCtgQNode* node); void ctgFreeQNode(SCtgQNode* node);
void ctgClearHandle(SCatalog* pCtg); void ctgClearHandle(SCatalog* pCtg);
void ctgFreeTbCacheImpl(SCtgTbCache* pCache, bool lock); void ctgFreeTbCacheImpl(SCtgTbCache* pCache, bool lock);
void ctgFreeViewCacheImpl(SCtgViewCache* pCache, bool lock);
int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName);
int32_t ctgRemoveCacheUser(SCatalog* pCtg, const char* user); int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user);
int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup,
bool* exists); bool* exists);
SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch);
@ -993,8 +1053,12 @@ void ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCach
void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache);
void ctgGetGlobalCacheStat(SCtgCacheStat* pStat); void ctgGetGlobalCacheStat(SCtgCacheStat* pStat);
int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res);
int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx);
int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta);
void ctgDestroySMetaData(SMetaData* pData);
void ctgGetGlobalCacheSize(uint64_t *pSize); void ctgGetGlobalCacheSize(uint64_t *pSize);
uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex); uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex);
uint64_t ctgGetViewMetaCacheSize(SViewMeta *pMeta);
uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta); uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta);
uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo *pVg); uint64_t ctgGetDbVgroupCacheSize(SDBVgInfo *pVg);
uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth); uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth);

120
source/libs/catalog/src/catalog.c
View File

@ -325,7 +325,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq,
SCtgAuthRsp rsp = {0}; SCtgAuthRsp rsp = {0};
rsp.pRawRes = pRes; rsp.pRawRes = pRes;
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pReq, &inCache, &rsp)); CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pReq, false, &inCache, &rsp));
if (inCache) { if (inCache) {
if (exists) { if (exists) {
@ -665,9 +665,32 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* viewName, uint64_t viewId) {
int32_t code = 0;
if (NULL == pCtg || NULL == dbFName || NULL == viewName) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
if (NULL == pCtg->dbCache) {
return TSDB_CODE_SUCCESS;
}
CTG_ERR_JRET(ctgDropViewMetaEnqueue(pCtg, dbFName, 0, viewName, viewId, true));
_return:
CTG_RET(code);
}
void ctgProcessTimerEvent(void *param, void *tmrId) { void ctgProcessTimerEvent(void *param, void *tmrId) {
CTG_API_NENTER(); CTG_API_NENTER();
ctgdShowCacheInfo();
ctgdShowStatInfo();
int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize); int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize);
if (cacheMaxSize >= 0) { if (cacheMaxSize >= 0) {
uint64_t cacheSize = 0; uint64_t cacheSize = 0;
@ -732,6 +755,10 @@ int32_t catalogInit(SCatalogCfg* cfg) {
gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER;
} }
if (gCtgMgmt.cfg.maxViewCacheNum == 0) {
gCtgMgmt.cfg.maxViewCacheNum = CTG_DEFAULT_CACHE_VIEW_NUMBER;
}
if (gCtgMgmt.cfg.dbRentSec == 0) { if (gCtgMgmt.cfg.dbRentSec == 0) {
gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND;
} }
@ -739,11 +766,17 @@ int32_t catalogInit(SCatalogCfg* cfg) {
if (gCtgMgmt.cfg.stbRentSec == 0) { if (gCtgMgmt.cfg.stbRentSec == 0) {
gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND;
} }
if (gCtgMgmt.cfg.viewRentSec == 0) {
gCtgMgmt.cfg.viewRentSec = CTG_DEFAULT_RENT_SECOND;
}
} else { } else {
gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER; gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER;
gCtgMgmt.cfg.maxViewCacheNum = CTG_DEFAULT_CACHE_VIEW_NUMBER;
gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.viewRentSec = CTG_DEFAULT_RENT_SECOND;
} }
gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT),
@ -828,6 +861,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo))); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion))); CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)));
clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
false, HASH_ENTRY_LOCK); false, HASH_ENTRY_LOCK);
@ -1461,6 +1495,25 @@ int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion** stables, uint3
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void**)stables, num, sizeof(SSTableVersion))); CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void**)stables, num, sizeof(SSTableVersion)));
} }
int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == views || NULL == num) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
*dynViewVersion = taosMemoryMalloc(sizeof(SDynViewVersion));
if (NULL == *dynViewVersion) {
CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY);
}
(*dynViewVersion)->svrBootTs = atomic_load_64(&pCtg->dynViewVer.svrBootTs);
(*dynViewVersion)->dynViewVer = atomic_load_64(&pCtg->dynViewVer.dynViewVer);
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion)));
}
int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) {
CTG_API_ENTER(); CTG_API_ENTER();
@ -1651,6 +1704,71 @@ int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) {
CTG_API_LEAVE(ctgUpdateUserEnqueue(pCtg, pAuth, false)); CTG_API_LEAVE(ctgUpdateUserEnqueue(pCtg, pAuth, false));
} }
void catalogFreeMetaData(SMetaData * pData) {
ctgDestroySMetaData(pData);
}
int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId, const char* viewName, uint64_t viewId) {
CTG_API_ENTER();
CTG_API_LEAVE(ctgRemoveViewMeta(pCtg, dbFName, dbId, viewName, viewId));
}
int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pVer) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
atomic_store_64(&pCtg->dynViewVer.svrBootTs, pVer->svrBootTs);
atomic_store_64(&pCtg->dynViewVer.dynViewVer, pVer->dynViewVer);
ctgDebug("cluster %" PRIx64 " svrBootTs updated to %" PRId64, pCtg->clusterId, pVer->svrBootTs);
ctgDebug("cluster %" PRIx64 " dynViewVer updated to %" PRId64, pCtg->clusterId, pVer->dynViewVer);
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pMsg) {
tFreeSViewMetaRsp(pMsg);
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgUpdateViewMetaToCache(pCtg, pMsg, true));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pMsg) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgUpdateViewMetaToCache(pCtg, pMsg, false));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogGetViewMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pViewName, STableMeta** pTableMeta) {
CTG_API_ENTER();
CTG_API_LEAVE(TSDB_CODE_OPS_NOT_SUPPORT);
}
int32_t catalogClearCache(void) { int32_t catalogClearCache(void) {
CTG_API_ENTER_NOLOCK(); CTG_API_ENTER_NOLOCK();

231
source/libs/catalog/src/ctgAsync.c
View File

@ -20,6 +20,30 @@
#include "tref.h" #include "tref.h"
#include "trpc.h" #include "trpc.h"
void ctgIsTaskDone(SCtgJob* pJob, CTG_TASK_TYPE type, bool* done) {
SCtgTask* pTask = NULL;
*done = true;
CTG_LOCK(CTG_READ, &pJob->taskLock);
int32_t taskNum = taosArrayGetSize(pJob->pTasks);
for (int32_t i = 0; i < taskNum; ++i) {
pTask = taosArrayGet(pJob->pTasks, i);
if (type != pTask->type) {
continue;
}
if (pTask->status != CTG_TASK_DONE) {
*done = false;
break;
}
}
CTG_UNLOCK(CTG_READ, &pJob->taskLock);
}
int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param; SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param;
SName* name = pParam->pName; SName* name = pParam->pName;
@ -53,7 +77,6 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
} }
int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
SName* name = (SName*)param;
SCtgTask task = {0}; SCtgTask task = {0};
task.type = CTG_TASK_GET_TB_META_BATCH; task.type = CTG_TASK_GET_TB_META_BATCH;
@ -184,7 +207,6 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
} }
int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
SName* name = (SName*)param;
SCtgTask task = {0}; SCtgTask task = {0};
task.type = CTG_TASK_GET_TB_HASH_BATCH; task.type = CTG_TASK_GET_TB_HASH_BATCH;
@ -417,6 +439,43 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) {
SCtgTask task = {0};
task.type = CTG_TASK_GET_VIEW;
task.taskId = taskIdx;
task.pJob = pJob;
task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgViewsCtx));
if (NULL == task.taskCtx) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SCtgViewsCtx* ctx = task.taskCtx;
ctx->pNames = param;
ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes));
taosArrayPush(pJob->pTasks, &task);
qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx,
ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum);
return TSDB_CODE_SUCCESS;
}
int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) {
int32_t viewNum = taosArrayGetSize(pReq->pView);
for (int32_t i = 0; i < viewNum; ++i) {
STablesReq* p = taosArrayGet(pReq->pView, i);
int32_t viewNum = taosArrayGetSize(p->pTables);
for (int32_t m = 0; m < viewNum; ++m) {
SName* name = taosArrayGet(p->pTables, m);
ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true);
}
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) {
SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@ -464,6 +523,12 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con
} }
} }
dbNum = taosArrayGetSize(pReq->pView);
for (int32_t i = 0; i < dbNum; ++i) {
STablesReq* p = taosArrayGet(pReq->pView, i);
taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN);
}
for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { for (int32_t i = 0; i < pJob->tbCfgNum; ++i) {
SName* name = taosArrayGet(pReq->pTableCfg, i); SName* name = taosArrayGet(pReq->pTableCfg, i);
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
@ -508,7 +573,8 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con
ctgDropTbIndexEnqueue(pCtg, name, true); ctgDropTbIndexEnqueue(pCtg, name, true);
} }
return TSDB_CODE_SUCCESS; // REFRESH VIEW META
return ctgHandleForceUpdateView(pCtg, pReq);
} }
int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) { int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) {
@ -547,9 +613,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex);
int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag); int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag);
int32_t viewNum = (int32_t)ctgGetTablesReqNum(pReq->pView);
int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum +
userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum; userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum;
*job = taosMemoryCalloc(1, sizeof(SCtgJob)); *job = taosMemoryCalloc(1, sizeof(SCtgJob));
if (NULL == *job) { if (NULL == *job) {
@ -559,6 +626,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
SCtgJob* pJob = *job; SCtgJob* pJob = *job;
pJob->jobRes.ctgFree = true;
pJob->subTaskNum = taskNum; pJob->subTaskNum = taskNum;
pJob->queryId = pConn->requestId; pJob->queryId = pConn->requestId;
pJob->userFp = fp; pJob->userFp = fp;
@ -580,6 +648,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
pJob->tbCfgNum = tbCfgNum; pJob->tbCfgNum = tbCfgNum;
pJob->svrVerNum = svrVerNum; pJob->svrVerNum = svrVerNum;
pJob->tbTagNum = tbTagNum; pJob->tbTagNum = tbTagNum;
pJob->viewNum = viewNum;
#if CTG_BATCH_FETCH #if CTG_BATCH_FETCH
pJob->pBatchs = pJob->pBatchs =
@ -668,6 +737,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_USER, user, NULL)); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_USER, user, NULL));
} }
if (viewNum > 0) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL));
}
if (qnodeNum) { if (qnodeNum) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL)); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL));
} }
@ -1001,6 +1074,20 @@ int32_t ctgDumpSvrVer(SCtgTask* pTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgDumpViewsRes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
pJob->jobRes.pView = pTask->res;
return TSDB_CODE_SUCCESS;
}
int32_t ctgCallSubCb(SCtgTask* pTask) { int32_t ctgCallSubCb(SCtgTask* pTask) {
int32_t code = 0; int32_t code = 0;
@ -1773,6 +1860,68 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
SCtgViewsCtx* ctx = (SCtgViewsCtx*)pTask->taskCtx;
SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
SName* pName = ctgGetFetchName(ctx->pNames, pFetch);
int32_t flag = pFetch->flag;
int32_t* vgId = &pFetch->vgId;
bool taskDone = false;
CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target));
SViewMetaRsp* pRsp = *(SViewMetaRsp**)pMsgCtx->out;
SViewMeta* pViewMeta = taosMemoryCalloc(1, sizeof(SViewMeta));
if (NULL == pViewMeta) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
if (TSDB_CODE_SUCCESS != (code = dupViewMetaFromRsp(pRsp, pViewMeta))) {
ctgFreeSViewMeta(pViewMeta);
taosMemoryFree(pViewMeta);
CTG_ERR_JRET(code);
}
ctgDebug("start to update view meta to cache, view:%s, querySQL:%s", pRsp->name, pRsp->querySql);
ctgUpdateViewMetaToCache(pCtg, pRsp, false);
taosMemoryFreeClear(pMsgCtx->out);
pRsp = NULL;
SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx);
pRes->code = 0;
pRes->pRes = pViewMeta;
if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) {
TSWAP(pTask->res, ctx->pResList);
taskDone = true;
}
_return:
if (code) {
SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx);
pRes->code = code;
pRes->pRes = NULL;
ctgTaskError("Get view %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname,
tstrerror(code));
if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) {
TSWAP(pTask->res, ctx->pResList);
taskDone = true;
}
}
if (pTask->res && taskDone) {
ctgHandleTaskEnd(pTask, code);
}
CTG_RET(code);
}
int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq* tReq, int32_t flag, SName* pName, int32_t* vgId) { int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq* tReq, int32_t flag, SName* pName, int32_t* vgId) {
SCtgTask* pTask = tReq->pTask; SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg; SCatalog* pCtg = pTask->pJob->pCtg;
@ -2305,6 +2454,7 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) {
bool inCache = false; bool inCache = false;
SCtgAuthRsp rsp = {0}; SCtgAuthRsp rsp = {0};
SCtgJob* pJob = pTask->pJob; SCtgJob* pJob = pTask->pJob;
bool tbNotExists = false;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1);
if (NULL == pMsgCtx->pBatchs) { if (NULL == pMsgCtx->pBatchs) {
pMsgCtx->pBatchs = pJob->pBatchs; pMsgCtx->pBatchs = pJob->pBatchs;
@ -2314,12 +2464,22 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) {
if (NULL == rsp.pRawRes) { if (NULL == rsp.pRawRes) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
if (TSDB_CODE_SUCCESS != pCtx->subTaskCode) {
if (CTG_TABLE_NOT_EXIST(pCtx->subTaskCode)) {
tbNotExists = true;
pCtx->subTaskCode = 0;
} else {
CTG_ERR_RET(pCtx->subTaskCode);
}
}
CTG_ERR_RET(ctgChkAuthFromCache(pCtg, &pCtx->user, &inCache, &rsp)); CTG_ERR_RET(ctgChkAuthFromCache(pCtg, &pCtx->user, tbNotExists, &inCache, &rsp));
if (inCache) { if (inCache) {
pTask->res = rsp.pRawRes; pTask->res = rsp.pRawRes;
ctgTaskDebug("Final res got, pass:%d, pCond:%p", rsp.pRawRes->pass, rsp.pRawRes->pCond); ctgTaskDebug("Final res got, pass:[%d,%d], pCond:[%p,%p]",
rsp.pRawRes->pass[0], rsp.pRawRes->pass[1], rsp.pRawRes->pCond[0], rsp.pRawRes->pCond[1]);
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -2355,6 +2515,59 @@ int32_t ctgLaunchGetSvrVerTask(SCtgTask* pTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SCtgViewsCtx* pCtx = (SCtgViewsCtx*)pTask->taskCtx;
SCtgJob* pJob = pTask->pJob;
bool tbMetaDone = false;
ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone);
if (tbMetaDone) {
CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx));
TSWAP(pTask->res, pCtx->pResList);
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
int32_t dbNum = taosArrayGetSize(pCtx->pNames);
int32_t fetchIdx = 0;
int32_t baseResIdx = 0;
for (int32_t i = 0; i < dbNum; ++i) {
STablesReq* pReq = taosArrayGet(pCtx->pNames, i);
ctgDebug("start to check views in db %s, viewNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables));
CTG_ERR_RET(ctgGetViewsFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables));
baseResIdx += taosArrayGetSize(pReq->pTables);
}
pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs);
if (pCtx->fetchNum <= 0) {
TSWAP(pTask->res, pCtx->pResList);
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum);
for (int32_t i = 0; i < pCtx->fetchNum; ++i) {
SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i);
SName* pName = ctgGetFetchName(pCtx->pNames, pFetch);
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i);
if (NULL == pMsgCtx->pBatchs) {
pMsgCtx->pBatchs = pJob->pBatchs;
}
SCtgTaskReq tReq;
tReq.pTask = pTask;
tReq.msgIdx = pFetch->fetchIdx;
CTG_ERR_RET(ctgGetViewInfoFromMnode(pCtg, pConn, pName, NULL, &tReq));
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask) { int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask) {
ctgResetTbMetaTask(pTask); ctgResetTbMetaTask(pTask);
@ -2410,8 +2623,9 @@ _return:
int32_t ctgGetUserCb(SCtgTask* pTask) { int32_t ctgGetUserCb(SCtgTask* pTask) {
int32_t code = 0; int32_t code = 0;
CTG_ERR_JRET(pTask->subRes.code); SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx;
pCtx->subTaskCode = pTask->subRes.code;
CTG_RET(ctgLaunchGetUserTask(pTask)); CTG_RET(ctgLaunchGetUserTask(pTask));
_return: _return:
@ -2470,6 +2684,7 @@ SCtgAsyncFps gCtgAsyncFps[] = {
{ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL}, {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL},
{ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL},
{ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL}, {ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL},
{ctgInitGetViewsTask, ctgLaunchGetViewsTask, ctgHandleGetViewsRsp, ctgDumpViewsRes, NULL, NULL},
}; };
int32_t ctgMakeAsyncRes(SCtgJob* pJob) { int32_t ctgMakeAsyncRes(SCtgJob* pJob) {

604
source/libs/catalog/src/ctgCache.c
View File

@ -30,6 +30,8 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v
{CTG_OP_UPDATE_VG_EPSET, "update epset", ctgOpUpdateEpset}, {CTG_OP_UPDATE_VG_EPSET, "update epset", ctgOpUpdateEpset},
{CTG_OP_UPDATE_TB_INDEX, "update tbIndex", ctgOpUpdateTbIndex}, {CTG_OP_UPDATE_TB_INDEX, "update tbIndex", ctgOpUpdateTbIndex},
{CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex},
{CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta},
{CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta},
{CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}};
SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = {
@ -49,6 +51,7 @@ SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = {
{"TblCfg ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_CFG, {"TblCfg ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_CFG,
{"TblTag ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_TAG, {"TblTag ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_TAG,
{"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_INDEX_INFO, {"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_INDEX_INFO,
{"viewMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_VIEW,
{"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_USER, {"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_USER,
{"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_UDF, {"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_UDF,
{"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} //CTG_CI_SVR_VER, {"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} //CTG_CI_SVR_VER,
@ -171,6 +174,18 @@ void ctgReleaseTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *
} }
} }
void ctgReleaseViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgViewCache *pCache) {
if (pCache && dbCache) {
CTG_UNLOCK(CTG_READ, &pCache->viewLock);
taosHashRelease(dbCache->viewCache, pCache);
}
if (dbCache) {
ctgReleaseDBCache(pCtg, dbCache);
}
}
void ctgReleaseTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache) { void ctgReleaseTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache) {
if (pCache) { if (pCache) {
CTG_UNLOCK(CTG_READ, &pCache->indexLock); CTG_UNLOCK(CTG_READ, &pCache->indexLock);
@ -773,7 +788,7 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid
} }
int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool tbNotExists, bool *inCache, SCtgAuthRsp *pRes) {
int32_t code = 0; int32_t code = 0;
SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user)); SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user));
@ -790,6 +805,7 @@ int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache,
SCtgAuthReq req = {0}; SCtgAuthReq req = {0};
req.pRawReq = pReq; req.pRawReq = pReq;
req.onlyCache = true; req.onlyCache = true;
req.tbNotExists = tbNotExists;
CTG_LOCK(CTG_READ, &pUser->lock); CTG_LOCK(CTG_READ, &pUser->lock);
memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth));
@ -1268,208 +1284,75 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type, int32_t size) { int32_t ctgUpdateViewMetaEnqueue(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncOp) {
mgmt->slotRIdx = 0; int32_t code = 0;
mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
mgmt->type = type; op->opId = CTG_OP_UPDATE_VIEW_META;
mgmt->metaSize = size; op->syncOp = syncOp;
size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; SCtgUpdateViewMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateViewMetaMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateViewMetaMsg));
taosMemoryFree(op);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
mgmt->slots = taosMemoryCalloc(1, msgSize); char *p = strchr(pRsp->dbFName, '.');
if (NULL == mgmt->slots) { if (p && IS_SYS_DBNAME(p + 1)) {
qError("calloc %d failed", (int32_t)msgSize); int32_t len = strlen(p + 1);
memmove(pRsp->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len);
}
msg->pCtg = pCtg;
msg->pRsp = pRsp;
op->data = msg;
CTG_ERR_RET(ctgEnqueue(pCtg, op));
return TSDB_CODE_SUCCESS;
_return:
if (pRsp) {
tFreeSViewMetaRsp(pRsp);
taosMemoryFree(pRsp);
}
CTG_RET(code);
}
int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) {
int32_t code = 0;
SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
op->opId = CTG_OP_DROP_VIEW_META;
op->syncOp = syncOp;
SCtgDropViewMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropViewMetaMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropViewMetaMsg));
taosMemoryFree(op);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
mgmt->rentCacheSize = msgSize; msg->pCtg = pCtg;
tstrncpy(msg->dbFName, dbFName, sizeof(msg->dbFName));
tstrncpy(msg->viewName, viewName, sizeof(msg->viewName));
msg->dbId = dbId;
msg->viewId = viewId;
qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); op->data = msg;
CTG_ERR_JRET(ctgEnqueue(pCtg, op));
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
}
int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size);
if (NULL == slot->meta) {
qError("taosArrayInit %d failed, id:0x%" PRIx64 ", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx,
mgmt->type);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
if (NULL == taosArrayPush(slot->meta, meta)) {
qError("taosArrayPush meta to rent failed, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
mgmt->rentCacheSize += size;
slot->needSort = true;
qDebug("add meta to rent, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return: _return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare,
__compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qDebug("empty meta slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type,
(int32_t)taosArrayGetSize(slot->meta));
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta));
}
void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ);
if (NULL == orig) {
qDebug("meta not found in slot, id:0x%" PRIx64 ", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type,
(int32_t)taosArrayGetSize(slot->meta));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
memcpy(orig, meta, size);
qDebug("meta in rent updated, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
if (code) {
qDebug("meta in rent update failed, will try to add it, code:%x, id:0x%" PRIx64 ", slot idx:%d, type:%d", code, id,
widx, mgmt->type);
CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size));
}
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qError("empty meta slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type);
}
int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ);
if (idx < 0) {
qError("meta not found in slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
taosArrayRemove(slot->meta, idx);
mgmt->rentCacheSize -= mgmt->metaSize;
qDebug("meta in rent removed, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1);
if (ridx >= mgmt->slotNum) {
ridx %= mgmt->slotNum;
atomic_store_16(&mgmt->slotRIdx, ridx);
}
SCtgRentSlot *slot = &mgmt->slots[ridx];
int32_t code = 0;
CTG_LOCK(CTG_READ, &slot->lock);
if (NULL == slot->meta) {
qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t metaNum = taosArrayGetSize(slot->meta);
if (metaNum <= 0) {
qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t msize = metaNum * size;
*res = taosMemoryMalloc(msize);
if (NULL == *res) {
qError("malloc %d failed", (int32_t)msize);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
void *meta = taosArrayGet(slot->meta, 0);
memcpy(*res, meta, msize);
*num = (uint32_t)metaNum;
qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type);
_return:
CTG_UNLOCK(CTG_READ, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
while (true) {
int64_t msec = taosGetTimestampMs();
int64_t lsec = atomic_load_64(&mgmt->lastReadMsec);
if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) {
*res = NULL;
*num = 0;
qDebug("too short time period to get expired meta, type:%d", mgmt->type);
return TSDB_CODE_SUCCESS;
}
if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) {
continue;
}
break;
}
CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size));
return TSDB_CODE_SUCCESS;
}
int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
int32_t code = 0; int32_t code = 0;
@ -1480,7 +1363,7 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
true, HASH_ENTRY_LOCK); true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.tbCache) { if (NULL == newDBCache.tbCache) {
ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); ctgError("taosHashInit %d tbCache failed", gCtgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
} }
@ -1491,6 +1374,13 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
} }
newDBCache.viewCache = taosHashInit(gCtgMgmt.cfg.maxViewCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
true, HASH_ENTRY_LOCK);
if (NULL == newDBCache.viewCache) {
ctgError("taosHashInit %d viewCache failed", gCtgMgmt.cfg.maxViewCacheNum);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache)); code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache));
if (code) { if (code) {
if (HASH_NODE_EXIST(code)) { if (HASH_NODE_EXIST(code)) {
@ -1524,25 +1414,6 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache) {
if (NULL == dbCache->stbCache) {
return;
}
void *pIter = taosHashIterate(dbCache->stbCache, NULL);
while (pIter) {
uint64_t *suid = NULL;
suid = taosHashGetKey(pIter, NULL);
if (TSDB_CODE_SUCCESS ==
ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) {
ctgDebug("stb removed from rent, suid:0x%" PRIx64, *suid);
}
pIter = taosHashIterate(dbCache->stbCache, pIter);
}
}
int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *dbFName) { int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *dbFName) {
uint64_t dbId = dbCache->dbId; uint64_t dbId = dbCache->dbId;
@ -1552,6 +1423,7 @@ int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *d
atomic_store_8(&dbCache->deleted, 1); atomic_store_8(&dbCache->deleted, 1);
ctgRemoveStbRent(pCtg, dbCache); ctgRemoveStbRent(pCtg, dbCache);
ctgRemoveViewRent(pCtg, dbCache);
ctgFreeDbCache(dbCache); ctgFreeDbCache(dbCache);
CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock);
@ -1611,30 +1483,6 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid,
SCtgTbCache *pCache) {
SSTableVersion metaRent = {.dbId = dbId, .suid = suid};
if (pCache->pMeta) {
metaRent.sversion = pCache->pMeta->sversion;
metaRent.tversion = pCache->pMeta->tversion;
}
if (pCache->pIndex) {
metaRent.smaVer = pCache->pIndex->version;
}
tstrncpy(metaRent.dbFName, dbFName, sizeof(metaRent.dbFName));
tstrncpy(metaRent.stbName, tbName, sizeof(metaRent.stbName));
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion),
ctgStbVersionSortCompare, ctgStbVersionSearchCompare));
ctgDebug("db %s,0x%" PRIx64 " stb %s,0x%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", dbFName, dbId,
tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer);
return TSDB_CODE_SUCCESS;
}
int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName,
STableMeta *meta, int32_t metaSize) { STableMeta *meta, int32_t metaSize) {
if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) {
@ -1787,6 +1635,65 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgWriteViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *viewName, SViewMeta *pMeta) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == dbCache->viewCache) {
ctgWarn("db is dropping, dbId:0x%" PRIx64, dbCache->dbId);
CTG_ERR_JRET(TSDB_CODE_CTG_DB_DROPPED);
}
SCtgViewCache *pCache = taosHashGet(dbCache->viewCache, viewName, strlen(viewName));
if (NULL == pCache) {
SCtgViewCache cache = {0};
cache.pMeta = pMeta;
if (taosHashPut(dbCache->viewCache, viewName, strlen(viewName), &cache, sizeof(cache)) != 0) {
ctgError("taosHashPut new tbCache failed, viewName:%s", viewName);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(viewName) + sizeof(SCtgViewCache) + ctgGetViewMetaCacheSize(pMeta));
CTG_DB_NUM_INC(CTG_CI_VIEW);
ctgDebug("new view meta updated to cache, view:%s, id:%" PRIu64 ", ver:%d, effectiveUser:%s, querySQL:%s",
viewName, pMeta->viewId, pMeta->version, pMeta->user, pMeta->querySql);
CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, &cache));
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_WRITE, &pCache->viewLock);
if (pCache->pMeta) {
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pCache->pMeta));
ctgFreeSViewMeta(pCache->pMeta);
taosMemoryFree(pCache->pMeta);
}
pCache->pMeta = pMeta;
CTG_UNLOCK(CTG_WRITE, &pCache->viewLock);
atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pMeta));
ctgDebug("view meta updated to cache, view:%s, id:%" PRIu64 ", ver:%d, effectiveUser:%s, querySQL:%s",
viewName, pMeta->viewId, pMeta->version, pMeta->user, pMeta->querySql);
CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache));
pMeta = NULL;
_return:
if (pMeta) {
ctgFreeSViewMeta(pMeta);
taosMemoryFree(pMeta);
}
CTG_RET(code);
}
int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) {
STableMetaOutput *pOutput = NULL; STableMetaOutput *pOutput = NULL;
int32_t code = 0; int32_t code = 0;
@ -1804,6 +1711,11 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq) {
CTG_RET(ctgUpdateViewMetaEnqueue(pCtg, pRsp, syncReq));
}
void ctgClearAllHandles(void) { void ctgClearAllHandles(void) {
SCatalog *pCtg = NULL; SCatalog *pCtg = NULL;
@ -2138,7 +2050,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) {
goto _return; goto _return;
} }
if (msg->dbId && (dbCache->dbId != msg->dbId)) { if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) {
ctgDebug("dbId already modified, dbFName:%s, current:0x%" PRIx64 ", dbId:0x%" PRIx64 ", stb:%s, suid:0x%" PRIx64, ctgDebug("dbId already modified, dbFName:%s, current:0x%" PRIx64 ", dbId:0x%" PRIx64 ", stb:%s, suid:0x%" PRIx64,
msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid);
goto _return; goto _return;
@ -2201,7 +2113,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) {
goto _return; goto _return;
} }
if (dbCache->dbId != msg->dbId) { if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) {
ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId,
msg->dbFName, msg->tbName); msg->dbFName, msg->tbName);
goto _return; goto _return;
@ -2265,7 +2177,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (msg->userAuth.dropped == 1) { } else if (msg->userAuth.dropped == 1) {
if (ctgRemoveCacheUser(pCtg, msg->userAuth.user) == 0) { if (ctgRemoveCacheUser(pCtg, pUser, msg->userAuth.user) == 0) {
CTG_CACHE_NUM_DEC(CTG_CI_USER, 1); CTG_CACHE_NUM_DEC(CTG_CI_USER, 1);
} }
goto _return; goto _return;
@ -2278,6 +2190,10 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
taosHashCleanup(pUser->userAuth.writeDbs); taosHashCleanup(pUser->userAuth.writeDbs);
taosHashCleanup(pUser->userAuth.readTbs); taosHashCleanup(pUser->userAuth.readTbs);
taosHashCleanup(pUser->userAuth.writeTbs); taosHashCleanup(pUser->userAuth.writeTbs);
taosHashCleanup(pUser->userAuth.alterTbs);
taosHashCleanup(pUser->userAuth.readViews);
taosHashCleanup(pUser->userAuth.writeViews);
taosHashCleanup(pUser->userAuth.alterViews);
taosHashCleanup(pUser->userAuth.useDbs); taosHashCleanup(pUser->userAuth.useDbs);
memcpy(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth)); memcpy(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth));
@ -2287,6 +2203,10 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) {
msg->userAuth.writeDbs = NULL; msg->userAuth.writeDbs = NULL;
msg->userAuth.readTbs = NULL; msg->userAuth.readTbs = NULL;
msg->userAuth.writeTbs = NULL; msg->userAuth.writeTbs = NULL;
msg->userAuth.alterTbs = NULL;
msg->userAuth.readViews = NULL;
msg->userAuth.writeViews = NULL;
msg->userAuth.alterViews = NULL;
msg->userAuth.useDbs = NULL; msg->userAuth.useDbs = NULL;
CTG_UNLOCK(CTG_WRITE, &pUser->lock); CTG_UNLOCK(CTG_WRITE, &pUser->lock);
@ -2300,6 +2220,10 @@ _return:
taosHashCleanup(msg->userAuth.writeDbs); taosHashCleanup(msg->userAuth.writeDbs);
taosHashCleanup(msg->userAuth.readTbs); taosHashCleanup(msg->userAuth.readTbs);
taosHashCleanup(msg->userAuth.writeTbs); taosHashCleanup(msg->userAuth.writeTbs);
taosHashCleanup(msg->userAuth.alterTbs);
taosHashCleanup(msg->userAuth.readViews);
taosHashCleanup(msg->userAuth.writeViews);
taosHashCleanup(msg->userAuth.alterViews);
taosHashCleanup(msg->userAuth.useDbs); taosHashCleanup(msg->userAuth.useDbs);
taosMemoryFreeClear(msg); taosMemoryFreeClear(msg);
@ -2427,6 +2351,106 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgUpdateViewMetaMsg *msg = operation->data;
SCatalog *pCtg = msg->pCtg;
SViewMetaRsp *pRsp = msg->pRsp;
SCtgDBCache *dbCache = NULL;
SViewMeta *pMeta = NULL;
taosMemoryFreeClear(msg);
if (pCtg->stopUpdate) {
goto _return;
}
CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pRsp->dbFName, pRsp->dbId, &dbCache));
if (NULL == dbCache) {
ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pRsp->dbFName, pRsp->dbId);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
pMeta = taosMemoryCalloc(1, sizeof(SViewMeta));
if (NULL == pMeta) {
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_JRET(dupViewMetaFromRsp(pRsp, pMeta));
code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta);
pMeta = NULL;
_return:
tFreeSViewMetaRsp(pRsp);
taosMemoryFree(pRsp);
ctgFreeSViewMeta(pMeta);
taosMemoryFree(pMeta);
CTG_RET(code);
}
int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgDropViewMetaMsg *msg = operation->data;
SCatalog *pCtg = msg->pCtg;
int32_t tblType = 0;
if (pCtg->stopUpdate) {
goto _return;
}
SCtgDBCache *dbCache = NULL;
ctgGetDBCache(pCtg, msg->dbFName, &dbCache);
if (NULL == dbCache) {
goto _return;
}
if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) {
ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, viewName:%s", msg->dbId, dbCache->dbId,
msg->dbFName, msg->viewName);
goto _return;
}
SCtgViewCache *pViewCache = taosHashGet(dbCache->viewCache, msg->viewName, strlen(msg->viewName));
if (NULL == pViewCache) {
ctgDebug("view %s already not in cache", msg->viewName);
goto _return;
}
int64_t viewId = pViewCache->pMeta->viewId;
if (0 != msg->viewId && viewId != msg->viewId) {
ctgDebug("viewId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", viewName:%s", msg->viewId, viewId, msg->viewName);
goto _return;
}
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta));
ctgFreeViewCacheImpl(pViewCache, true);
if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) {
ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
} else {
atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName));
CTG_DB_NUM_DEC(CTG_CI_VIEW);
}
ctgDebug("view %s removed from cache, dbFName:%s", msg->viewName, msg->dbFName);
CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare));
ctgDebug("view %s removed from rent, dbFName:%s, viewId:0x%" PRIx64, msg->viewName, msg->dbFName, viewId);
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
void ctgClearFreeCache(SCtgCacheOperation *operation) { void ctgClearFreeCache(SCtgCacheOperation *operation) {
SCtgClearCacheMsg *msg = operation->data; SCtgClearCacheMsg *msg = operation->data;
SCatalog *pCtg = msg->pCtg; SCatalog *pCtg = msg->pCtg;
@ -2529,6 +2553,7 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
case CTG_OP_DROP_TB_META: case CTG_OP_DROP_TB_META:
case CTG_OP_UPDATE_VG_EPSET: case CTG_OP_UPDATE_VG_EPSET:
case CTG_OP_DROP_TB_INDEX: case CTG_OP_DROP_TB_INDEX:
case CTG_OP_DROP_VIEW_META:
case CTG_OP_CLEAR_CACHE: { case CTG_OP_CLEAR_CACHE: {
taosMemoryFreeClear(op->data); taosMemoryFreeClear(op->data);
break; break;
@ -2540,6 +2565,10 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
taosHashCleanup(msg->userAuth.writeDbs); taosHashCleanup(msg->userAuth.writeDbs);
taosHashCleanup(msg->userAuth.readTbs); taosHashCleanup(msg->userAuth.readTbs);
taosHashCleanup(msg->userAuth.writeTbs); taosHashCleanup(msg->userAuth.writeTbs);
taosHashCleanup(msg->userAuth.alterTbs);
taosHashCleanup(msg->userAuth.readViews);
taosHashCleanup(msg->userAuth.writeViews);
taosHashCleanup(msg->userAuth.alterViews);
taosHashCleanup(msg->userAuth.useDbs); taosHashCleanup(msg->userAuth.useDbs);
taosMemoryFreeClear(op->data); taosMemoryFreeClear(op->data);
break; break;
@ -2787,6 +2816,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe
CTG_LOCK(CTG_READ, &pCache->metaLock); CTG_LOCK(CTG_READ, &pCache->metaLock);
if (NULL == pCache->pMeta) { if (NULL == pCache->pMeta) {
CTG_UNLOCK(CTG_READ, &pCache->metaLock); CTG_UNLOCK(CTG_READ, &pCache->metaLock);
taosHashRelease(dbCache->tbCache, pCache);
ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName);
ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag);
taosArrayPush(ctx->pResList, &(SMetaRes){0}); taosArrayPush(ctx->pResList, &(SMetaRes){0});
@ -3006,3 +3036,105 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx,
int32_t *fetchIdx, int32_t baseResIdx, SArray *pList) {
int32_t tbNum = taosArrayGetSize(pList);
SName *pName = taosArrayGet(pList, 0);
char dbFName[TSDB_DB_FNAME_LEN] = {0};
int32_t flag = CTG_FLAG_UNKNOWN_STB;
uint64_t lastSuid = 0;
STableMeta *lastTableMeta = NULL;
if (IS_SYS_DBNAME(pName->dbname)) {
CTG_FLAG_SET_SYS_DB(flag);
strcpy(dbFName, pName->dbname);
} else {
tNameGetFullDbName(pName, dbFName);
}
SCtgDBCache *dbCache = NULL;
SCtgViewCache *pCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
if (NULL == dbCache) {
ctgDebug("db %s not in cache", dbFName);
for (int32_t i = 0; i < tbNum; ++i) {
ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag);
taosArrayPush(ctx->pResList, &(SMetaData){0});
}
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < tbNum; ++i) {
pName = taosArrayGet(pList, i);
pCache = taosHashAcquire(dbCache->viewCache, pName->tname, strlen(pName->tname));
if (NULL == pCache) {
ctgDebug("view %s not in cache, dbFName:%s", pName->tname, dbFName);
ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag);
taosArrayPush(ctx->pResList, &(SMetaRes){0});
CTG_CACHE_NHIT_INC(CTG_CI_VIEW, 1);
continue;
}
CTG_LOCK(CTG_READ, &pCache->viewLock);
if (NULL == pCache->pMeta) {
CTG_UNLOCK(CTG_READ, &pCache->viewLock);
taosHashRelease(dbCache->viewCache, pCache);
ctgDebug("view %s meta not in cache, dbFName:%s", pName->tname, dbFName);
ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag);
taosArrayPush(ctx->pResList, &(SMetaRes){0});
CTG_CACHE_NHIT_INC(CTG_CI_VIEW, 1);
continue;
}
CTG_CACHE_HIT_INC(CTG_CI_VIEW, 1);
SMetaRes res = {0};
SViewMeta *pViewMeta = taosMemoryCalloc(1, sizeof(SViewMeta));
if (NULL == pViewMeta) {
ctgReleaseViewMetaToCache(pCtg, dbCache, pCache);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(pViewMeta, pCache->pMeta, sizeof(*pViewMeta));
pViewMeta->querySql = tstrdup(pCache->pMeta->querySql);
pViewMeta->user = tstrdup(pCache->pMeta->user);
if (NULL == pViewMeta->querySql || NULL == pViewMeta->user) {
ctgReleaseViewMetaToCache(pCtg, dbCache, pCache);
pViewMeta->pSchema = NULL;
taosMemoryFree(pViewMeta->querySql);
taosMemoryFree(pViewMeta->user);
taosMemoryFree(pViewMeta);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pViewMeta->pSchema = taosMemoryMalloc(pViewMeta->numOfCols * sizeof(SSchema));
if (pViewMeta->pSchema == NULL) {
ctgReleaseViewMetaToCache(pCtg, dbCache, pCache);
ctgFreeSViewMeta(pViewMeta);
taosMemoryFree(pViewMeta);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(pViewMeta->pSchema, pCache->pMeta->pSchema, pViewMeta->numOfCols * sizeof(SSchema));
CTG_UNLOCK(CTG_READ, &pCache->viewLock);
taosHashRelease(dbCache->viewCache, pCache);
ctgDebug("Got view %s meta from cache, dbFName:%s", pName->tname, dbFName);
res.pRes = pViewMeta;
taosArrayPush(ctx->pResList, &res);
}
ctgReleaseDBCache(pCtg, dbCache);
return TSDB_CODE_SUCCESS;
}

31
source/libs/catalog/src/ctgDbg.c
View File

@ -253,6 +253,12 @@ int32_t ctgdEnableDebug(char *option, bool enable) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (0 == strcasecmp(option, "stat")) {
gCTGDebug.statEnable = enable;
qDebug("catalog stat debug set to %d", enable);
return TSDB_CODE_SUCCESS;
}
if (0 == strcasecmp(option, "stopUpdate")) { if (0 == strcasecmp(option, "stopUpdate")) {
SCatalog *pCtg = NULL; SCatalog *pCtg = NULL;
@ -320,6 +326,11 @@ int32_t ctgdHandleDbgCommand(char *command) {
CTG_RET(TSDB_CODE_INVALID_PARA); CTG_RET(TSDB_CODE_INVALID_PARA);
} }
if (NULL == param || NULL == option) {
taosMemoryFree(dup);
CTG_RET(TSDB_CODE_INVALID_PARA);
}
bool enable = atoi(param); bool enable = atoi(param);
int32_t code = ctgdEnableDebug(option, enable); int32_t code = ctgdEnableDebug(option, enable);
@ -361,6 +372,10 @@ int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) {
return dbCache->tbCache ? (int32_t)taosHashGetSize(dbCache->tbCache) : 0; return dbCache->tbCache ? (int32_t)taosHashGetSize(dbCache->tbCache) : 0;
} }
int32_t ctgdGetViewNum(SCtgDBCache *dbCache) {
return dbCache->viewCache ? (int32_t)taosHashGetSize(dbCache->viewCache) : 0;
}
int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { int32_t ctgdGetStbNum(SCtgDBCache *dbCache) {
return dbCache->stbCache ? (int32_t)taosHashGetSize(dbCache->stbCache) : 0; return dbCache->stbCache ? (int32_t)taosHashGetSize(dbCache->stbCache) : 0;
} }
@ -391,6 +406,8 @@ int32_t ctgdGetClusterCacheNum(SCatalog *pCtg, int32_t type) {
return ctgdGetRentNum(&pCtg->dbRent); return ctgdGetRentNum(&pCtg->dbRent);
case CTG_DBG_STB_RENT_NUM: case CTG_DBG_STB_RENT_NUM:
return ctgdGetRentNum(&pCtg->stbRent); return ctgdGetRentNum(&pCtg->stbRent);
case CTG_DBG_VIEW_RENT_NUM:
return ctgdGetRentNum(&pCtg->viewRent);
default: default:
break; break;
} }
@ -407,6 +424,8 @@ int32_t ctgdGetClusterCacheNum(SCatalog *pCtg, int32_t type) {
case CTG_DBG_STB_NUM: case CTG_DBG_STB_NUM:
num += ctgdGetStbNum(dbCache); num += ctgdGetStbNum(dbCache);
break; break;
case CTG_DBG_VIEW_NUM:
num += ctgdGetViewNum(dbCache);
default: default:
ctgError("invalid type:%d", type); ctgError("invalid type:%d", type);
break; break;
@ -459,6 +478,7 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
dbFName = taosHashGetKey(pIter, &len); dbFName = taosHashGetKey(pIter, &len);
int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0; int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0;
int32_t viewNum = dbCache->viewCache ? taosHashGetSize(dbCache->viewCache) : 0;
int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0; int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0;
int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION;
int32_t hashMethod = -1; int32_t hashMethod = -1;
@ -479,8 +499,8 @@ void ctgdShowDBCache(SCatalog *pCtg, SHashObj *dbHash) {
} }
ctgDebug("[%d] db [%.*s][0x%" PRIx64 ctgDebug("[%d] db [%.*s][0x%" PRIx64
"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", "] %s: metaNum:%d, viewNum:%d, stbNum:%d, vgVersion:%d, stateTs:%" PRId64 ", hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, stbNum, vgVersion, stateTs, i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted ? "deleted" : "", metaNum, viewNum, stbNum, vgVersion, stateTs,
hashMethod, hashPrefix, hashSuffix, vgNum); hashMethod, hashPrefix, hashSuffix, vgNum);
if (dbCache->vgCache.vgInfo) { if (dbCache->vgCache.vgInfo) {
@ -530,9 +550,10 @@ void ctgdShowClusterCache(SCatalog *pCtg) {
} }
ctgDebug("## cluster 0x%" PRIx64 " %p cache Info BEGIN ##", pCtg->clusterId, pCtg); ctgDebug("## cluster 0x%" PRIx64 " %p cache Info BEGIN ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDebug("db:%d tbmeta:%d viewmeta:%d stb:%d dbRent:%d stbRent:%d viewRent:%d", ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_VIEW_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM)); ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM),
ctgdGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), ctgdGetClusterCacheNum(pCtg, CTG_DBG_VIEW_RENT_NUM));
ctgdShowDBCache(pCtg, pCtg->dbCache); ctgdShowDBCache(pCtg, pCtg->dbCache);

72
source/libs/catalog/src/ctgRemote.c
View File

@ -318,6 +318,21 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize,
qDebug("Got svr ver from mnode"); qDebug("Got svr ver from mnode");
break; break;
} }
case TDMT_MND_VIEW_META: {
if (TSDB_CODE_SUCCESS != rspCode) {
qError("error rsp for get view-meta, error:%s, viewFName:%s", tstrerror(rspCode), target);
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get view-meta rsp failed, error:%s, viewFName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got view-meta from mnode, viewFName:%s", target);
break;
}
default: default:
if (TSDB_CODE_SUCCESS != rspCode) { if (TSDB_CODE_SUCCESS != rspCode) {
qError("Got error rsp, error:%s", tstrerror(rspCode)); qError("Got error rsp, error:%s", tstrerror(rspCode));
@ -1388,3 +1403,60 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pName, SViewMetaOutput* out,
SCtgTaskReq* tReq) {
char* msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_MND_VIEW_META;
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char fullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pName, fullName);
ctgDebug("try to get view info from mnode, viewFName:%s", fullName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](fullName, &msg, 0, &msgLen, mallocFp);
if (code) {
ctgError("Build view-meta msg failed, code:%x, viewFName:%s", code, fullName);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, POINTER_BYTES);
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, fullName));
#if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen));
#else
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayPush(pTaskId, &pTask->taskId);
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, fullName));
rpcFreeCont(rpcRsp.pCont);
return TSDB_CODE_SUCCESS;
}

306
source/libs/catalog/src/ctgRent.c Executable file
View File

@ -0,0 +1,306 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "catalogInt.h"
#include "query.h"
#include "systable.h"
#include "tname.h"
#include "trpc.h"
int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type, int32_t size) {
mgmt->slotRIdx = 0;
mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND;
mgmt->type = type;
mgmt->metaSize = size;
size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum;
mgmt->slots = taosMemoryCalloc(1, msgSize);
if (NULL == mgmt->slots) {
qError("calloc %d failed", (int32_t)msgSize);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
mgmt->rentCacheSize = msgSize;
qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum);
return TSDB_CODE_SUCCESS;
}
int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size);
if (NULL == slot->meta) {
qError("taosArrayInit %d failed, id:0x%" PRIx64 ", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx,
mgmt->type);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
if (NULL == taosArrayPush(slot->meta, meta)) {
qError("taosArrayPush meta to rent failed, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
mgmt->rentCacheSize += size;
slot->needSort = true;
qDebug("add meta to rent, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare,
__compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qDebug("empty meta slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type,
(int32_t)taosArrayGetSize(slot->meta));
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta));
}
void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ);
if (NULL == orig) {
qDebug("meta not found in slot, id:0x%" PRIx64 ", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type,
(int32_t)taosArrayGetSize(slot->meta));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
memcpy(orig, meta, size);
qDebug("meta in rent updated, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
if (code) {
qDebug("meta in rent update failed, will try to add it, code:%x, id:0x%" PRIx64 ", slot idx:%d, type:%d", code, id,
widx, mgmt->type);
CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size));
}
CTG_RET(code);
}
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare) {
int16_t widx = abs((int)(id % mgmt->slotNum));
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
CTG_LOCK(CTG_WRITE, &slot->lock);
if (NULL == slot->meta) {
qError("empty meta slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
if (slot->needSort) {
taosArraySort(slot->meta, sortCompare);
slot->needSort = false;
qDebug("meta slot sorted, slot idx:%d, type:%d", widx, mgmt->type);
}
int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ);
if (idx < 0) {
qError("meta not found in slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
taosArrayRemove(slot->meta, idx);
mgmt->rentCacheSize -= mgmt->metaSize;
qDebug("meta in rent removed, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type);
_return:
CTG_UNLOCK(CTG_WRITE, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1);
if (ridx >= mgmt->slotNum) {
ridx %= mgmt->slotNum;
atomic_store_16(&mgmt->slotRIdx, ridx);
}
SCtgRentSlot *slot = &mgmt->slots[ridx];
int32_t code = 0;
CTG_LOCK(CTG_READ, &slot->lock);
if (NULL == slot->meta) {
qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t metaNum = taosArrayGetSize(slot->meta);
if (metaNum <= 0) {
qDebug("no meta in slot:%d, type:%d", ridx, mgmt->type);
*num = 0;
goto _return;
}
size_t msize = metaNum * size;
*res = taosMemoryMalloc(msize);
if (NULL == *res) {
qError("malloc %d failed", (int32_t)msize);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
void *meta = taosArrayGet(slot->meta, 0);
memcpy(*res, meta, msize);
*num = (uint32_t)metaNum;
qDebug("Got %d meta from rent, type:%d", (int32_t)metaNum, mgmt->type);
_return:
CTG_UNLOCK(CTG_READ, &slot->lock);
CTG_RET(code);
}
int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) {
while (true) {
int64_t msec = taosGetTimestampMs();
int64_t lsec = atomic_load_64(&mgmt->lastReadMsec);
if ((msec - lsec) < CTG_RENT_SLOT_SECOND * 1000) {
*res = NULL;
*num = 0;
qDebug("too short time period to get expired meta, type:%d", mgmt->type);
return TSDB_CODE_SUCCESS;
}
if (lsec != atomic_val_compare_exchange_64(&mgmt->lastReadMsec, lsec, msec)) {
continue;
}
break;
}
CTG_ERR_RET(ctgMetaRentGetImpl(mgmt, res, num, size));
return TSDB_CODE_SUCCESS;
}
void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache) {
if (NULL == dbCache->stbCache) {
return;
}
void *pIter = taosHashIterate(dbCache->stbCache, NULL);
while (pIter) {
uint64_t *suid = NULL;
suid = taosHashGetKey(pIter, NULL);
if (TSDB_CODE_SUCCESS ==
ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) {
ctgDebug("stb removed from rent, suid:0x%" PRIx64, *suid);
}
pIter = taosHashIterate(dbCache->stbCache, pIter);
}
}
void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache) {
if (NULL == dbCache->stbCache) {
return;
}
void *pIter = taosHashIterate(dbCache->viewCache, NULL);
while (pIter) {
uint64_t viewId = ((SCtgViewCache*)pIter)->pMeta->viewId;
if (TSDB_CODE_SUCCESS ==
ctgMetaRentRemove(&pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare)) {
ctgDebug("view removed from rent, viewId:0x%" PRIx64, viewId);
}
pIter = taosHashIterate(dbCache->viewCache, pIter);
}
}
int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid,
SCtgTbCache *pCache) {
SSTableVersion metaRent = {.dbId = dbId, .suid = suid};
if (pCache->pMeta) {
metaRent.sversion = pCache->pMeta->sversion;
metaRent.tversion = pCache->pMeta->tversion;
}
if (pCache->pIndex) {
metaRent.smaVer = pCache->pIndex->version;
}
tstrncpy(metaRent.dbFName, dbFName, sizeof(metaRent.dbFName));
tstrncpy(metaRent.stbName, tbName, sizeof(metaRent.stbName));
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion),
ctgStbVersionSortCompare, ctgStbVersionSearchCompare));
ctgDebug("db %s,0x%" PRIx64 " stb %s,0x%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", dbFName, dbId,
tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer);
return TSDB_CODE_SUCCESS;
}
int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId,
SCtgViewCache *pCache) {
SViewVersion metaRent = {.dbId = dbId, .viewId = viewId};
metaRent.version = pCache->pMeta->version;
tstrncpy(metaRent.dbFName, dbFName, sizeof(metaRent.dbFName));
tstrncpy(metaRent.viewName, viewName, sizeof(metaRent.viewName));
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->viewRent, &metaRent, metaRent.viewId, sizeof(SViewVersion),
ctgViewVersionSortCompare, ctgViewVersionSearchCompare));
ctgDebug("db %s,0x%" PRIx64 " view %s,0x%" PRIx64 " version %d updated to viewRent", dbFName, dbId, viewName, viewId, metaRent.version);
return TSDB_CODE_SUCCESS;
}

415
source/libs/catalog/src/ctgUtil.c
View File

@ -19,6 +19,16 @@
#include "tname.h" #include "tname.h"
#include "trpc.h" #include "trpc.h"
void ctgFreeSViewMeta(SViewMeta* pMeta) {
if (NULL == pMeta) {
return;
}
taosMemoryFree(pMeta->user);
taosMemoryFree(pMeta->querySql);
taosMemoryFree(pMeta->pSchema);
}
void ctgFreeMsgSendParam(void* param) { void ctgFreeMsgSendParam(void* param) {
if (NULL == param) { if (NULL == param) {
return; return;
@ -93,6 +103,10 @@ char* ctgTaskTypeStr(CTG_TASK_TYPE type) {
return "[bget table meta]"; return "[bget table meta]";
case CTG_TASK_GET_TB_HASH_BATCH: case CTG_TASK_GET_TB_HASH_BATCH:
return "[bget table hash]"; return "[bget table hash]";
case CTG_TASK_GET_TB_TAG:
return "[get table tag]";
case CTG_TASK_GET_VIEW:
return "[get view]";
default: default:
return "unknown"; return "unknown";
} }
@ -173,6 +187,9 @@ void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pTableTag); taosArrayDestroy(pData->pTableTag);
pData->pTableTag = NULL; pData->pTableTag = NULL;
taosArrayDestroy(pData->pView);
pData->pView = NULL;
taosMemoryFreeClear(pData->pSvrVer); taosMemoryFreeClear(pData->pSvrVer);
} }
@ -182,6 +199,10 @@ void ctgFreeSCtgUserAuth(SCtgUserAuth* userCache) {
taosHashCleanup(userCache->userAuth.writeDbs); taosHashCleanup(userCache->userAuth.writeDbs);
taosHashCleanup(userCache->userAuth.readTbs); taosHashCleanup(userCache->userAuth.readTbs);
taosHashCleanup(userCache->userAuth.writeTbs); taosHashCleanup(userCache->userAuth.writeTbs);
taosHashCleanup(userCache->userAuth.alterTbs);
taosHashCleanup(userCache->userAuth.readViews);
taosHashCleanup(userCache->userAuth.writeViews);
taosHashCleanup(userCache->userAuth.alterViews);
taosHashCleanup(userCache->userAuth.useDbs); taosHashCleanup(userCache->userAuth.useDbs);
} }
@ -235,12 +256,38 @@ void ctgFreeTbCacheImpl(SCtgTbCache* pCache, bool lock) {
} }
} }
void ctgFreeViewCacheImpl(SCtgViewCache* pCache, bool lock) {
if (lock) {
CTG_LOCK(CTG_WRITE, &pCache->viewLock);
}
if (pCache->pMeta) {
ctgFreeSViewMeta(pCache->pMeta);
taosMemoryFreeClear(pCache->pMeta);
}
if (lock) {
CTG_UNLOCK(CTG_WRITE, &pCache->viewLock);
}
}
void ctgFreeViewCache(SCtgDBCache* dbCache) {
if (NULL == dbCache->viewCache) {
return;
}
SCtgViewCache* pCache = taosHashIterate(dbCache->viewCache, NULL);
while (NULL != pCache) {
ctgFreeViewCacheImpl(pCache, false);
pCache = taosHashIterate(dbCache->viewCache, pCache);
}
taosHashCleanup(dbCache->viewCache);
dbCache->viewCache = NULL;
}
void ctgFreeTbCache(SCtgDBCache* dbCache) { void ctgFreeTbCache(SCtgDBCache* dbCache) {
if (NULL == dbCache->tbCache) { if (NULL == dbCache->tbCache) {
return; return;
} }
int32_t tblNum = taosHashGetSize(dbCache->tbCache);
SCtgTbCache* pCache = taosHashIterate(dbCache->tbCache, NULL); SCtgTbCache* pCache = taosHashIterate(dbCache->tbCache, NULL);
while (NULL != pCache) { while (NULL != pCache) {
ctgFreeTbCacheImpl(pCache, false); ctgFreeTbCacheImpl(pCache, false);
@ -262,6 +309,7 @@ void ctgFreeDbCache(SCtgDBCache* dbCache) {
ctgFreeCfgInfoCache(dbCache); ctgFreeCfgInfoCache(dbCache);
ctgFreeStbMetaCache(dbCache); ctgFreeStbMetaCache(dbCache);
ctgFreeTbCache(dbCache); ctgFreeTbCache(dbCache);
ctgFreeViewCache(dbCache);
} }
void ctgFreeInstDbCache(SHashObj* pDbCache) { void ctgFreeInstDbCache(SHashObj* pDbCache) {
@ -304,6 +352,7 @@ void ctgFreeInstUserCache(SHashObj* pUserCache) {
void ctgFreeHandleImpl(SCatalog* pCtg) { void ctgFreeHandleImpl(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent); ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent); ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeInstDbCache(pCtg->dbCache); ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache); ctgFreeInstUserCache(pCtg->userCache);
@ -311,17 +360,13 @@ void ctgFreeHandleImpl(SCatalog* pCtg) {
taosMemoryFree(pCtg); taosMemoryFree(pCtg);
} }
int32_t ctgRemoveCacheUser(SCatalog* pCtg, const char* user) { int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user) {
if (!pCtg || !user) { CTG_LOCK(CTG_WRITE, &pUser->lock);
return -1; ctgFreeSCtgUserAuth(pUser);
} CTG_UNLOCK(CTG_WRITE, &pUser->lock);
SCtgUserAuth* pUser = (SCtgUserAuth*)taosHashGet(pCtg->userCache, user, strlen(user)); if (taosHashRemove(pCtg->userCache, user, strlen(user)) == 0) {
if (pUser) { return 0; // user found and removed
ctgFreeSCtgUserAuth(pUser);
if (taosHashRemove(pCtg->userCache, user, strlen(user)) == 0) {
return 0; // user found and removed
}
} }
return -1; return -1;
@ -336,6 +381,7 @@ void ctgFreeHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent); ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent); ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeInstDbCache(pCtg->dbCache); ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache); ctgFreeInstUserCache(pCtg->userCache);
@ -421,12 +467,14 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent); ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent); ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeInstDbCache(pCtg->dbCache); ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache); ctgFreeInstUserCache(pCtg->userCache);
ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)); ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo));
ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)); ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion));
ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion));
pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false,
HASH_ENTRY_LOCK); HASH_ENTRY_LOCK);
@ -525,10 +573,25 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
taosHashCleanup(pOut->writeDbs); taosHashCleanup(pOut->writeDbs);
taosHashCleanup(pOut->readTbs); taosHashCleanup(pOut->readTbs);
taosHashCleanup(pOut->writeTbs); taosHashCleanup(pOut->writeTbs);
taosHashCleanup(pOut->alterTbs);
taosHashCleanup(pOut->readViews);
taosHashCleanup(pOut->writeViews);
taosHashCleanup(pOut->alterViews);
taosHashCleanup(pOut->useDbs); taosHashCleanup(pOut->useDbs);
taosMemoryFreeClear(pCtx->out); taosMemoryFreeClear(pCtx->out);
break; break;
} }
case TDMT_MND_VIEW_META: {
if (NULL != pCtx->out) {
SViewMetaRsp* pOut = *(SViewMetaRsp**)pCtx->out;
if (NULL != pOut) {
tFreeSViewMetaRsp(pOut);
taosMemoryFree(pOut);
}
taosMemoryFreeClear(pCtx->out);
}
break;
}
default: default:
qError("invalid reqType %d", pCtx->reqType); qError("invalid reqType %d", pCtx->reqType);
break; break;
@ -587,6 +650,19 @@ void ctgFreeBatchHash(void* hash) {
taosMemoryFreeClear(pRes->pRes); taosMemoryFreeClear(pRes->pRes);
} }
void ctgFreeViewMetaRes(void* res) {
if (NULL == res) {
return;
}
SMetaRes* pRes = (SMetaRes*)res;
if (NULL != pRes->pRes) {
SViewMeta* pMeta = (SViewMeta*)pRes->pRes;
ctgFreeSViewMeta(pMeta);
taosMemoryFreeClear(pRes->pRes);
}
}
void ctgFreeJsonTagVal(void* val) { void ctgFreeJsonTagVal(void* val) {
if (NULL == val) { if (NULL == val) {
return; return;
@ -632,7 +708,9 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
case CTG_TASK_GET_USER: { case CTG_TASK_GET_USER: {
if (*pRes) { if (*pRes) {
SUserAuthRes* pAuth = (SUserAuthRes*)*pRes; SUserAuthRes* pAuth = (SUserAuthRes*)*pRes;
nodesDestroyNode(pAuth->pCond); for (int32_t i = 0; i < AUTH_RES_MAX_VALUE; ++i) {
nodesDestroyNode(pAuth->pCond[i]);
}
taosMemoryFreeClear(*pRes); taosMemoryFreeClear(*pRes);
} }
break; break;
@ -673,6 +751,15 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
*pRes = NULL; // no need to free it *pRes = NULL; // no need to free it
break; break;
} }
case CTG_TASK_GET_VIEW: {
SArray* pArray = (SArray*)*pRes;
int32_t num = taosArrayGetSize(pArray);
for (int32_t i = 0; i < num; ++i) {
ctgFreeViewMetaRes(taosArrayGet(pArray, i));
}
*pRes = NULL; // no need to free it
break;
}
default: default:
qError("invalid task type %d", type); qError("invalid task type %d", type);
break; break;
@ -825,22 +912,35 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
taosMemoryFreeClear(pTask->taskCtx); taosMemoryFreeClear(pTask->taskCtx);
break; break;
} }
case CTG_TASK_GET_VIEW: {
SCtgViewsCtx* taskCtx = (SCtgViewsCtx*)pTask->taskCtx;
taosArrayDestroyEx(taskCtx->pResList, ctgFreeViewMetaRes);
taosArrayDestroy(taskCtx->pFetchs);
// NO NEED TO FREE pNames
taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx);
taosMemoryFreeClear(pTask->taskCtx);
break;
}
default: default:
qError("invalid task type %d", pTask->type); qError("invalid task type %d", pTask->type);
break; break;
} }
} }
void ctgFreeTask(SCtgTask* pTask) { void ctgFreeTask(SCtgTask* pTask, bool freeRes) {
ctgFreeMsgCtx(&pTask->msgCtx); ctgFreeMsgCtx(&pTask->msgCtx);
ctgFreeTaskRes(pTask->type, &pTask->res); if (freeRes || pTask->subTask) {
ctgFreeTaskRes(pTask->type, &pTask->res);
}
ctgFreeTaskCtx(pTask); ctgFreeTaskCtx(pTask);
taosArrayDestroy(pTask->pParents); taosArrayDestroy(pTask->pParents);
ctgClearSubTaskRes(&pTask->subRes); ctgClearSubTaskRes(&pTask->subRes);
} }
void ctgFreeTasks(SArray* pArray) { void ctgFreeTasks(SArray* pArray, bool freeRes) {
if (NULL == pArray) { if (NULL == pArray) {
return; return;
} }
@ -848,7 +948,7 @@ void ctgFreeTasks(SArray* pArray) {
int32_t num = taosArrayGetSize(pArray); int32_t num = taosArrayGetSize(pArray);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
SCtgTask* pTask = taosArrayGet(pArray, i); SCtgTask* pTask = taosArrayGet(pArray, i);
ctgFreeTask(pTask); ctgFreeTask(pTask, freeRes);
} }
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
@ -864,7 +964,7 @@ void ctgFreeJob(void* job) {
int64_t rid = pJob->refId; int64_t rid = pJob->refId;
uint64_t qid = pJob->queryId; uint64_t qid = pJob->queryId;
ctgFreeTasks(pJob->pTasks); ctgFreeTasks(pJob->pTasks, pJob->jobRes.ctgFree);
ctgFreeBatchs(pJob->pBatchs); ctgFreeBatchs(pJob->pBatchs);
ctgFreeSMetaData(&pJob->jobRes); ctgFreeSMetaData(&pJob->jobRes);
@ -1218,6 +1318,17 @@ int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2) {
} }
} }
int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) {
if (*(uint64_t*)key1 < ((SViewVersion*)key2)->viewId) {
return -1;
} else if (*(uint64_t*)key1 > ((SViewVersion*)key2)->viewId) {
return 1;
} else {
return 0;
}
}
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) {
if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) {
return -1; return -1;
@ -1238,6 +1349,17 @@ int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2) {
} }
} }
int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) {
if (((SViewVersion*)key1)->viewId < ((SViewVersion*)key2)->viewId) {
return -1;
} else if (((SViewVersion*)key1)->viewId > ((SViewVersion*)key2)->viewId) {
return 1;
} else {
return 0;
}
}
int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
if (NULL == dbInfo) { if (NULL == dbInfo) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -1452,9 +1574,9 @@ static void* ctgCloneVgroupInfo(void* pSrc) {
static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); }
static void* ctgCloneTableIndices(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } static void* ctgCloneTableIndexs(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); }
static void ctgFreeTableIndices(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } static void ctgFreeTableIndexs(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static void* ctgCloneFuncInfo(void* pSrc) { static void* ctgCloneFuncInfo(void* pSrc) {
SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo)); SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo));
@ -1508,6 +1630,34 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)
static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); }
static void* ctgCloneViewMeta(void* pSrc) {
SViewMeta* pSrcMeta = pSrc;
SViewMeta* pDst = taosMemoryMalloc(sizeof(SViewMeta));
if (NULL == pDst) {
return NULL;
}
pDst->user = tstrdup(pSrcMeta->user);
pDst->querySql = tstrdup(pSrcMeta->querySql);
pDst->pSchema = taosMemoryMalloc(pSrcMeta->numOfCols * sizeof(*pSrcMeta->pSchema));
if (NULL == pDst->pSchema) {
return pDst;
}
memcpy(pDst->pSchema, pSrcMeta->pSchema, pSrcMeta->numOfCols * sizeof(*pSrcMeta->pSchema));
return pDst;
}
static void ctgFreeViewMeta(void* p) {
SViewMeta* pMeta = ((SMetaRes*)p)->pRes;
if (NULL == pMeta) {
return;
}
taosMemoryFree(pMeta->user);
taosMemoryFree(pMeta->querySql);
taosMemoryFree(pMeta->pSchema);
taosMemoryFree(pMeta);
}
int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
int32_t code = 0; int32_t code = 0;
STableMeta* pMeta = NULL; STableMeta* pMeta = NULL;
@ -1523,18 +1673,18 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
while (true) { while (true) {
taosMemoryFreeClear(pMeta); taosMemoryFreeClear(pMeta);
char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName)); char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName) + 1);
if (pCond) { if (pCond) {
if (strlen(pCond) > 1) { if (strlen(pCond) > 1) {
CTG_ERR_JRET(nodesStringToNode(pCond, &res->pRawRes->pCond)); CTG_ERR_JRET(nodesStringToNode(pCond, &res->pRawRes->pCond[AUTH_RES_BASIC]));
} }
res->pRawRes->pass = true; res->pRawRes->pass[AUTH_RES_BASIC] = true;
goto _return; goto _return;
} }
if (stbName) { if (stbName) {
res->pRawRes->pass = false; res->pRawRes->pass[AUTH_RES_BASIC] = false;
goto _return; goto _return;
} }
@ -1554,7 +1704,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
} }
if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) {
res->pRawRes->pass = false; res->pRawRes->pass[AUTH_RES_BASIC] = false;
goto _return; goto _return;
} }
@ -1586,31 +1736,36 @@ _return:
CTG_RET(code); CTG_RET(code);
} }
int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { int32_t ctgChkSetBasicAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
int32_t code = 0; int32_t code = 0;
SUserAuthInfo* pReq = req->pRawReq; SUserAuthInfo* pReq = req->pRawReq;
SUserAuthRes* pRes = res->pRawRes; SUserAuthRes* pRes = res->pRawRes;
SGetUserAuthRsp* pInfo = &req->authInfo; SGetUserAuthRsp* pInfo = &req->authInfo;
pRes->pass = false; pRes->pass[AUTH_RES_BASIC] = false;
pRes->pCond = NULL; pRes->pCond[AUTH_RES_BASIC] = NULL;
if (!pInfo->enable) { if (!pInfo->enable) {
pRes->pass = false;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (pInfo->superAuth) { if (pInfo->superAuth) {
pRes->pass = true; pRes->pass[AUTH_RES_BASIC] = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (IS_SYS_DBNAME(pReq->tbName.dbname)) { if (IS_SYS_DBNAME(pReq->tbName.dbname)) {
pRes->pass = true; pRes->pass[AUTH_RES_BASIC] = true;
ctgDebug("sysdb %s, pass", pReq->tbName.dbname); ctgDebug("sysdb %s, pass", pReq->tbName.dbname);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
if (req->tbNotExists) {
//pRes->pass[AUTH_RES_BASIC] = true;
//return TSDB_CODE_SUCCESS;
pReq->tbName.type = TSDB_DB_NAME_T;
}
char dbFName[TSDB_DB_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&pReq->tbName, dbFName); tNameGetFullDbName(&pReq->tbName, dbFName);
@ -1627,13 +1782,13 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) {
req->singleType = AUTH_TYPE_READ; req->singleType = AUTH_TYPE_READ;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass || res->metaNotExists) { if (pRes->pass[AUTH_RES_BASIC] || res->metaNotExists) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
if (pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) { if (pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName) + 1)) {
pRes->pass = true; pRes->pass[AUTH_RES_BASIC] = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1643,23 +1798,23 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { if (pReq->tbName.type == TSDB_TABLE_NAME_T && pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) {
req->singleType = AUTH_TYPE_WRITE; req->singleType = AUTH_TYPE_WRITE;
CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res));
if (pRes->pass || res->metaNotExists) { if (pRes->pass[AUTH_RES_BASIC] || res->metaNotExists) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }
if (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) { if (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName) + 1)) {
pRes->pass = true; pRes->pass[AUTH_RES_BASIC] = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
break; break;
} }
case AUTH_TYPE_READ_OR_WRITE: { case AUTH_TYPE_READ_OR_WRITE: {
if ((pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) || if ((pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName) + 1)) ||
(pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) || (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName) + 1)) ||
(pInfo->useDbs && taosHashGet(pInfo->useDbs, dbFName, strlen(dbFName)))) { (pInfo->useDbs && taosHashGet(pInfo->useDbs, dbFName, strlen(dbFName) + 1))) {
pRes->pass = true; pRes->pass[AUTH_RES_BASIC] = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1672,6 +1827,78 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
int32_t code = 0;
SUserAuthInfo* pReq = req->pRawReq;
SUserAuthRes* pRes = res->pRawRes;
SGetUserAuthRsp* pInfo = &req->authInfo;
pRes->pass[AUTH_RES_VIEW] = false;
pRes->pCond[AUTH_RES_VIEW] = NULL;
if (!pInfo->enable) {
return TSDB_CODE_SUCCESS;
}
if (pInfo->superAuth) {
pRes->pass[AUTH_RES_VIEW] = true;
return TSDB_CODE_SUCCESS;
}
if (pReq->tbName.type != TSDB_TABLE_NAME_T) {
return TSDB_CODE_SUCCESS;
}
char viewFName[TSDB_VIEW_FNAME_LEN];
if (IS_SYS_DBNAME(req->pRawReq->tbName.dbname)) {
snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname);
} else {
tNameExtractFullName(&req->pRawReq->tbName, viewFName);
}
int32_t len = strlen(viewFName) + 1;
switch (pReq->type) {
case AUTH_TYPE_READ: {
char *value = taosHashGet(pInfo->readViews, viewFName, len);
if (NULL != value) {
pRes->pass[AUTH_RES_VIEW] = true;
return TSDB_CODE_SUCCESS;
}
break;
}
case AUTH_TYPE_WRITE: {
char *value = taosHashGet(pInfo->writeViews, viewFName, len);
if (NULL != value) {
pRes->pass[AUTH_RES_VIEW] = true;
return TSDB_CODE_SUCCESS;
}
break;
}
case AUTH_TYPE_ALTER: {
char *value = taosHashGet(pInfo->alterViews, viewFName, len);
if (NULL != value) {
pRes->pass[AUTH_RES_VIEW] = true;
return TSDB_CODE_SUCCESS;
}
break;
}
default:
break;
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
#ifdef TD_ENTERPRISE
CTG_ERR_RET(ctgChkSetViewAuthRes(pCtg, req, res));
if (req->pRawReq->isView) {
return TSDB_CODE_SUCCESS;
}
#endif
CTG_RET(ctgChkSetBasicAuthRes(pCtg, req, res));
}
#if 0 #if 0
static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) { static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) {
if (NULL == pSrc) { if (NULL == pSrc) {
@ -1743,8 +1970,9 @@ SMetaData* catalogCloneMetaData(SMetaData* pData) {
return pRes; return pRes;
} }
#endif
void catalogFreeMetaData(SMetaData* pData) { void ctgDestroySMetaData(SMetaData* pData) {
if (NULL == pData) { if (NULL == pData) {
return; return;
} }
@ -1754,17 +1982,16 @@ void catalogFreeMetaData(SMetaData* pData) {
taosArrayDestroyEx(pData->pDbInfo, ctgFreeDbInfo); taosArrayDestroyEx(pData->pDbInfo, ctgFreeDbInfo);
taosArrayDestroyEx(pData->pTableMeta, ctgFreeTableMeta); taosArrayDestroyEx(pData->pTableMeta, ctgFreeTableMeta);
taosArrayDestroyEx(pData->pTableHash, ctgFreeVgroupInfo); taosArrayDestroyEx(pData->pTableHash, ctgFreeVgroupInfo);
taosArrayDestroyEx(pData->pTableIndex, ctgFreeTableIndices); taosArrayDestroyEx(pData->pTableIndex, ctgFreeTableIndexs);
taosArrayDestroyEx(pData->pUdfList, ctgFreeFuncInfo); taosArrayDestroyEx(pData->pUdfList, ctgFreeFuncInfo);
taosArrayDestroyEx(pData->pIndex, ctgFreeIndexInfo); taosArrayDestroyEx(pData->pIndex, ctgFreeIndexInfo);
taosArrayDestroyEx(pData->pUser, ctgFreeUserAuth); taosArrayDestroyEx(pData->pUser, ctgFreeUserAuth);
taosArrayDestroyEx(pData->pQnodeList, ctgFreeQnodeList); taosArrayDestroyEx(pData->pQnodeList, ctgFreeQnodeList);
taosArrayDestroyEx(pData->pTableCfg, ctgFreeTableCfg); taosArrayDestroyEx(pData->pTableCfg, ctgFreeTableCfg);
taosArrayDestroyEx(pData->pDnodeList, ctgFreeDnodeList); taosArrayDestroyEx(pData->pDnodeList, ctgFreeDnodeList);
taosArrayDestroyEx(pData->pView, ctgFreeViewMeta);
taosMemoryFreeClear(pData->pSvrVer); taosMemoryFreeClear(pData->pSvrVer);
taosMemoryFree(pData);
} }
#endif
uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex) { uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex) {
if (NULL == pIndex) { if (NULL == pIndex) {
@ -1774,6 +2001,15 @@ uint64_t ctgGetTbIndexCacheSize(STableIndex *pIndex) {
return sizeof(*pIndex) + pIndex->indexSize; return sizeof(*pIndex) + pIndex->indexSize;
} }
uint64_t ctgGetViewMetaCacheSize(SViewMeta *pMeta) {
if (NULL == pMeta) {
return 0;
}
return sizeof(*pMeta) + strlen(pMeta->querySql) + 1 + strlen(pMeta->user) + 1 + pMeta->numOfCols * sizeof(SSchema);
}
FORCE_INLINE uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta) { FORCE_INLINE uint64_t ctgGetTbMetaCacheSize(STableMeta *pMeta) {
if (NULL == pMeta) { if (NULL == pMeta) {
return 0; return 0;
@ -1810,7 +2046,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) {
while (p != NULL) { while (p != NULL) {
size_t len = 0; size_t len = 0;
void* key = taosHashGetKey(p, &len); void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p); cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->createdDbs, p); p = taosHashIterate(pAuth->createdDbs, p);
} }
@ -1819,7 +2055,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) {
while (p != NULL) { while (p != NULL) {
size_t len = 0; size_t len = 0;
void* key = taosHashGetKey(p, &len); void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p); cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->readDbs, p); p = taosHashIterate(pAuth->readDbs, p);
} }
@ -1828,7 +2064,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) {
while (p != NULL) { while (p != NULL) {
size_t len = 0; size_t len = 0;
void* key = taosHashGetKey(p, &len); void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p); cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->writeDbs, p); p = taosHashIterate(pAuth->writeDbs, p);
} }
@ -1837,7 +2073,7 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) {
while (p != NULL) { while (p != NULL) {
size_t len = 0; size_t len = 0;
void* key = taosHashGetKey(p, &len); void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p); cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->readTbs, p); p = taosHashIterate(pAuth->readTbs, p);
} }
@ -1846,11 +2082,47 @@ uint64_t ctgGetUserCacheSize(SGetUserAuthRsp *pAuth) {
while (p != NULL) { while (p != NULL) {
size_t len = 0; size_t len = 0;
void* key = taosHashGetKey(p, &len); void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p); cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->writeTbs, p); p = taosHashIterate(pAuth->writeTbs, p);
} }
p = taosHashIterate(pAuth->alterTbs, NULL);
while (p != NULL) {
size_t len = 0;
void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->alterTbs, p);
}
p = taosHashIterate(pAuth->readViews, NULL);
while (p != NULL) {
size_t len = 0;
void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->readViews, p);
}
p = taosHashIterate(pAuth->writeViews, NULL);
while (p != NULL) {
size_t len = 0;
void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->writeViews, p);
}
p = taosHashIterate(pAuth->alterViews, NULL);
while (p != NULL) {
size_t len = 0;
void* key = taosHashGetKey(p, &len);
cacheSize += len + strlen(p) + 1;
p = taosHashIterate(pAuth->alterViews, p);
}
int32_t *ref = taosHashIterate(pAuth->useDbs, NULL); int32_t *ref = taosHashIterate(pAuth->useDbs, NULL);
while (ref != NULL) { while (ref != NULL) {
size_t len = 0; size_t len = 0;
@ -1886,6 +2158,7 @@ uint64_t ctgGetClusterCacheSize(SCatalog *pCtg) {
cacheSize += pCtg->dbRent.rentCacheSize; cacheSize += pCtg->dbRent.rentCacheSize;
cacheSize += pCtg->stbRent.rentCacheSize; cacheSize += pCtg->stbRent.rentCacheSize;
cacheSize += pCtg->viewRent.rentCacheSize;
return cacheSize; return cacheSize;
} }
@ -1974,3 +2247,45 @@ void ctgGetGlobalCacheSize(uint64_t *pSize) {
} }
} }
int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx) {
SCatalog* pCtg = pTask->pJob->pCtg;
int32_t dbNum = taosArrayGetSize(pCtx->pNames);
for (int32_t i = 0; i < dbNum; ++i) {
STablesReq* pReq = taosArrayGet(pCtx->pNames, i);
int32_t viewNum = taosArrayGetSize(pReq->pTables);
ctgDebug("start to check views in db %s, viewNum %d", pReq->dbFName, viewNum);
for (int32_t m = 0; m < viewNum; ++m) {
taosArrayPush(pCtx->pResList, &(SMetaData){0});
}
}
return TSDB_CODE_SUCCESS;
}
int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) {
pViewMeta->querySql = tstrdup(pRsp->querySql);
if (NULL == pViewMeta->querySql) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pViewMeta->user = tstrdup(pRsp->user);
if (NULL == pViewMeta->user) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pViewMeta->version = pRsp->version;
pViewMeta->viewId = pRsp->viewId;
pViewMeta->precision = pRsp->precision;
pViewMeta->type = pRsp->type;
pViewMeta->numOfCols = pRsp->numOfCols;
pViewMeta->pSchema = taosMemoryMalloc(pViewMeta->numOfCols * sizeof(SSchema));
if (pViewMeta->pSchema == NULL) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
memcpy(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema));
return TSDB_CODE_SUCCESS;
}

4
source/libs/catalog/test/catalogTests.cpp
View File

@ -2814,7 +2814,7 @@ TEST(apiTest, catalogChkAuth_test) {
code = catalogChkAuth(pCtg, mockPointer, &authInfo, &authRes); code = catalogChkAuth(pCtg, mockPointer, &authInfo, &authRes);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(authRes.pass, true); ASSERT_EQ(authRes.pass[AUTH_RES_BASIC], true);
while (true) { while (true) {
uint64_t n = 0; uint64_t n = 0;
@ -2828,7 +2828,7 @@ TEST(apiTest, catalogChkAuth_test) {
code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(authRes.pass, true); ASSERT_EQ(authRes.pass[AUTH_RES_BASIC], true);
ASSERT_EQ(exists, true); ASSERT_EQ(exists, true);
catalogDestroy(); catalogDestroy();

111
source/libs/command/src/command.c
View File

@ -112,7 +112,11 @@ static int32_t setDescResultIntoDataBlock(bool sysInfoUser, SSDataBlock* pBlock,
colDataSetVal(pCol2, pBlock->info.rows, buf, false); colDataSetVal(pCol2, pBlock->info.rows, buf, false);
int32_t bytes = getSchemaBytes(pMeta->schema + i); int32_t bytes = getSchemaBytes(pMeta->schema + i);
colDataSetVal(pCol3, pBlock->info.rows, (const char*)&bytes, false); colDataSetVal(pCol3, pBlock->info.rows, (const char*)&bytes, false);
STR_TO_VARSTR(buf, i >= pMeta->tableInfo.numOfColumns ? "TAG" : ""); if (TSDB_VIEW_TABLE != pMeta->tableType) {
STR_TO_VARSTR(buf, i >= pMeta->tableInfo.numOfColumns ? "TAG" : "");
} else {
STR_TO_VARSTR(buf, "VIEW COL");
}
colDataSetVal(pCol4, pBlock->info.rows, buf, false); colDataSetVal(pCol4, pBlock->info.rows, buf, false);
++(pBlock->info.rows); ++(pBlock->info.rows);
} }
@ -215,37 +219,28 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) {
return v; return v;
} }
char* buildRetension(SArray* pRetension) { static char* buildRetension(SArray* pRetension) {
size_t size = taosArrayGetSize(pRetension); size_t size = taosArrayGetSize(pRetension);
if (size == 0) { if (size == 0) {
return NULL; return NULL;
} }
char* p1 = taosMemoryCalloc(1, 100); char* p1 = taosMemoryCalloc(1, 100);
SRetention* p = taosArrayGet(pRetension, 0);
int32_t len = 0; int32_t len = 0;
int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq); for (int32_t i = 0; i < size; ++i) {
int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep); SRetention* p = TARRAY_GET_ELEM(pRetension, i);
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
if (i == 0) {
len += sprintf(p1 + len, "-:%" PRId64 "%c", v2, p->keepUnit);
} else {
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
}
if (size > 1) { if (i < size - 1) {
len += sprintf(p1 + len, ","); len += sprintf(p1 + len, ",");
p = taosArrayGet(pRetension, 1); }
v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
}
if (size > 2) {
len += sprintf(p1 + len, ",");
p = taosArrayGet(pRetension, 2);
v1 = getValOfDiffPrecision(p->freqUnit, p->freq);
v2 = getValOfDiffPrecision(p->keepUnit, p->keep);
len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit);
} }
return p1; return p1;
@ -467,6 +462,29 @@ static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) {
return code; return code;
} }
static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) {
SSDataBlock* pBlock = createDataBlock();
if (NULL == pBlock) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_VIEW_RESULT_FIELD1_LEN, 1);
int32_t code = blockDataAppendColInfo(pBlock, &infoData);
if (TSDB_CODE_SUCCESS == code) {
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_VIEW_RESULT_FIELD2_LEN, 2);
code = blockDataAppendColInfo(pBlock, &infoData);
}
if (TSDB_CODE_SUCCESS == code) {
*pOutput = pBlock;
} else {
blockDataDestroy(pBlock);
}
return code;
}
void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) {
for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { for (int32_t i = 0; i < pCfg->numOfColumns; ++i) {
SSchema* pSchema = pCfg->pSchemas + i; SSchema* pSchema = pCfg->pSchemas + i;
@ -710,6 +728,36 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreateViewStmt* pStmt) {
int32_t code = 0;
blockDataEnsureCapacity(pBlock, 1);
pBlock->info.rows = 1;
SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0);
char buf1[SHOW_CREATE_VIEW_RESULT_FIELD1_LEN + 1] = {0};
snprintf(varDataVal(buf1), TSDB_VIEW_FNAME_LEN + 4, "`%s`.`%s`", pStmt->dbName, pStmt->viewName);
varDataSetLen(buf1, strlen(varDataVal(buf1)));
colDataSetVal(pCol1, 0, buf1, false);
SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1);
char* buf2 = taosMemoryMalloc(SHOW_CREATE_VIEW_RESULT_FIELD2_LEN);
if (NULL == buf2) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
SViewMeta* pMeta = pStmt->pViewMeta;
snprintf(varDataVal(buf2), SHOW_CREATE_VIEW_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE VIEW `%s`.`%s` AS %s", pStmt->dbName, pStmt->viewName, pMeta->querySql);
int32_t len = strlen(varDataVal(buf2));
varDataLen(buf2) = (len > 65535) ? 65535 : len;
colDataSetVal(pCol2, 0, buf2, false);
taosMemoryFree(buf2);
return TSDB_CODE_SUCCESS;
}
static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL; SSDataBlock* pBlock = NULL;
int32_t code = buildCreateTbResultDataBlock(&pBlock); int32_t code = buildCreateTbResultDataBlock(&pBlock);
@ -949,6 +997,19 @@ static int32_t execSelectWithoutFrom(SSelectStmt* pSelect, SRetrieveTableRsp** p
return code; return code;
} }
static int32_t execShowCreateView(SShowCreateViewStmt* pStmt, SRetrieveTableRsp** pRsp) {
SSDataBlock* pBlock = NULL;
int32_t code = buildCreateViewResultDataBlock(&pBlock);
if (TSDB_CODE_SUCCESS == code) {
code = setCreateViewResultIntoDataBlock(pBlock, pStmt);
}
if (TSDB_CODE_SUCCESS == code) {
code = buildRetrieveTableRsp(pBlock, SHOW_CREATE_VIEW_RESULT_COLS, pRsp);
}
blockDataDestroy(pBlock);
return code;
}
int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode) { int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp** pRsp, int8_t biMode) {
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
case QUERY_NODE_DESCRIBE_STMT: case QUERY_NODE_DESCRIBE_STMT:
@ -961,6 +1022,8 @@ int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode* pStmt, SRetrieve
return execShowCreateTable((SShowCreateTableStmt*)pStmt, pRsp); return execShowCreateTable((SShowCreateTableStmt*)pStmt, pRsp);
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return execShowCreateSTable((SShowCreateTableStmt*)pStmt, pRsp); return execShowCreateSTable((SShowCreateTableStmt*)pStmt, pRsp);
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
return execShowCreateView((SShowCreateViewStmt*)pStmt, pRsp);
case QUERY_NODE_ALTER_LOCAL_STMT: case QUERY_NODE_ALTER_LOCAL_STMT:
return execAlterLocal((SAlterLocalStmt*)pStmt); return execAlterLocal((SAlterLocalStmt*)pStmt);
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:

10
source/libs/executor/src/projectoperator.c
View File

@ -152,9 +152,11 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
setOperatorStreamStateFn(pOperator, streamOperatorReleaseState, streamOperatorReloadState); setOperatorStreamStateFn(pOperator, streamOperatorReleaseState, streamOperatorReloadState);
code = appendDownstream(pOperator, &downstream, 1); if (NULL != downstream) {
if (code != TSDB_CODE_SUCCESS) { code = appendDownstream(pOperator, &downstream, 1);
goto _error; if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
} }
return pOperator; return pOperator;
@ -263,7 +265,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
st = taosGetTimestampUs(); st = taosGetTimestampUs();
} }
SOperatorInfo* downstream = pOperator->pDownstream[0]; SOperatorInfo* downstream = pOperator->numOfDownstream > 0 ? pOperator->pDownstream[0] : NULL;
SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo; SLimitInfo* pLimitInfo = &pProjectInfo->limitInfo;
if (downstream == NULL) { if (downstream == NULL) {

16
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -741,6 +741,19 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t setOperatorCopy(const SSetOperator* pSrc, SSetOperator* pDst) {
COPY_SCALAR_FIELD(opType);
CLONE_NODE_LIST_FIELD(pProjectionList);
CLONE_NODE_FIELD(pLeft);
CLONE_NODE_FIELD(pRight);
CLONE_NODE_LIST_FIELD(pOrderByList);
CLONE_NODE_FIELD(pLimit);
COPY_CHAR_ARRAY_FIELD(stmtName);
COPY_SCALAR_FIELD(precision);
COPY_SCALAR_FIELD(timeLineResMode);
return TSDB_CODE_SUCCESS;
}
SNode* nodesCloneNode(const SNode* pNode) { SNode* nodesCloneNode(const SNode* pNode) {
if (NULL == pNode) { if (NULL == pNode) {
return NULL; return NULL;
@ -829,6 +842,9 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_HINT: case QUERY_NODE_HINT:
code = hintNodeCopy((const SHintNode*)pNode, (SHintNode*)pDst); code = hintNodeCopy((const SHintNode*)pNode, (SHintNode*)pDst);
break; break;
case QUERY_NODE_SET_OPERATOR:
code = setOperatorCopy((const SSetOperator*)pNode, (SSetOperator*)pDst);
break;
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
code = selectStmtCopy((const SSelectStmt*)pNode, (SSelectStmt*)pDst); code = selectStmtCopy((const SSelectStmt*)pNode, (SSelectStmt*)pDst);
break; break;

37
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -251,6 +251,8 @@ const char* nodesNodeName(ENodeType type) {
return "ShowCreateTablesStmt"; return "ShowCreateTablesStmt";
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return "ShowCreateStablesStmt"; return "ShowCreateStablesStmt";
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
return "ShowCreateViewStmt";
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
return "ShowTableDistributedStmt"; return "ShowTableDistributedStmt";
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
@ -269,6 +271,10 @@ const char* nodesNodeName(ENodeType type) {
return "RestoreMnodeStmt"; return "RestoreMnodeStmt";
case QUERY_NODE_RESTORE_VNODE_STMT: case QUERY_NODE_RESTORE_VNODE_STMT:
return "RestoreVnodeStmt"; return "RestoreVnodeStmt";
case QUERY_NODE_CREATE_VIEW_STMT:
return "CreateViewStmt";
case QUERY_NODE_DROP_VIEW_STMT:
return "DropViewStmt";
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return "LogicScan"; return "LogicScan";
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -6589,6 +6595,33 @@ static int32_t jsonToShowCreateStableStmt(const SJson* pJson, void* pObj) {
return jsonToShowCreateTableStmt(pJson, pObj); return jsonToShowCreateTableStmt(pJson, pObj);
} }
static const char* jkShowCreateViewStmtDbName = "DbName";
static const char* jkShowCreateViewStmtViewName = "ViewName";
static int32_t showCreateViewStmtToJson(const void* pObj, SJson* pJson) {
const SShowCreateViewStmt* pNode = (const SShowCreateViewStmt*)pObj;
int32_t code = tjsonAddStringToObject(pJson, jkShowCreateViewStmtDbName, pNode->dbName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkShowCreateViewStmtViewName, pNode->viewName);
}
return code;
}
static int32_t jsonToShowCreateViewStmt(const SJson* pJson, void* pObj) {
SShowCreateViewStmt* pNode = (SShowCreateViewStmt*)pObj;
int32_t code = tjsonGetStringValue(pJson, jkShowCreateViewStmtDbName, pNode->dbName);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkShowCreateViewStmtViewName, pNode->viewName);
}
return code;
}
static const char* jkShowTableDistributedStmtDbName = "DbName"; static const char* jkShowTableDistributedStmtDbName = "DbName";
static const char* jkShowTableDistributedStmtTableName = "TableName"; static const char* jkShowTableDistributedStmtTableName = "TableName";
@ -6956,6 +6989,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return showCreateTableStmtToJson(pObj, pJson); return showCreateTableStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return showCreateStableStmtToJson(pObj, pJson); return showCreateStableStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
return showCreateViewStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
return showTableDistributedStmtToJson(pObj, pJson); return showTableDistributedStmtToJson(pObj, pJson);
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
@ -7277,6 +7312,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToShowCreateTableStmt(pJson, pObj); return jsonToShowCreateTableStmt(pJson, pObj);
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return jsonToShowCreateStableStmt(pJson, pObj); return jsonToShowCreateStableStmt(pJson, pObj);
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
return jsonToShowCreateViewStmt(pJson, pObj);
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
return jsonToShowTableDistributedStmt(pJson, pObj); return jsonToShowTableDistributedStmt(pJson, pObj);
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:

31
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -304,6 +304,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SEventWindowNode)); return makeNode(type, sizeof(SEventWindowNode));
case QUERY_NODE_HINT: case QUERY_NODE_HINT:
return makeNode(type, sizeof(SHintNode)); return makeNode(type, sizeof(SHintNode));
case QUERY_NODE_VIEW:
return makeNode(type, sizeof(SViewNode));
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
return makeNode(type, sizeof(SSetOperator)); return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
@ -436,6 +438,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT: case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
case QUERY_NODE_SHOW_VIEWS_STMT:
return makeNode(type, sizeof(SShowStmt)); return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_TABLE_TAGS_STMT: case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return makeNode(type, sizeof(SShowTableTagsStmt)); return makeNode(type, sizeof(SShowTableTagsStmt));
@ -449,6 +452,8 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return makeNode(type, sizeof(SShowCreateTableStmt)); return makeNode(type, sizeof(SShowCreateTableStmt));
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
return makeNode(type, sizeof(SShowCreateViewStmt));
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT:
return makeNode(type, sizeof(SShowTableDistributedStmt)); return makeNode(type, sizeof(SShowTableDistributedStmt));
case QUERY_NODE_KILL_QUERY_STMT: case QUERY_NODE_KILL_QUERY_STMT:
@ -467,6 +472,10 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_RESTORE_MNODE_STMT: case QUERY_NODE_RESTORE_MNODE_STMT:
case QUERY_NODE_RESTORE_VNODE_STMT: case QUERY_NODE_RESTORE_VNODE_STMT:
return makeNode(type, sizeof(SRestoreComponentNodeStmt)); return makeNode(type, sizeof(SRestoreComponentNodeStmt));
case QUERY_NODE_CREATE_VIEW_STMT:
return makeNode(type, sizeof(SCreateViewStmt));
case QUERY_NODE_DROP_VIEW_STMT:
return makeNode(type, sizeof(SDropViewStmt));
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return makeNode(type, sizeof(SScanLogicNode)); return makeNode(type, sizeof(SScanLogicNode));
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -832,6 +841,13 @@ void nodesDestroyNode(SNode* pNode) {
destroyHintValue(pHint->option, pHint->value); destroyHintValue(pHint->option, pHint->value);
break; break;
} }
case QUERY_NODE_VIEW: {
SViewNode* pView = (SViewNode*)pNode;
taosMemoryFreeClear(pView->pMeta);
taosMemoryFreeClear(pView->pVgroupList);
taosArrayDestroyEx(pView->pSmaIndexes, destroySmaIndex);
break;
}
case QUERY_NODE_SET_OPERATOR: { case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pStmt = (SSetOperator*)pNode; SSetOperator* pStmt = (SSetOperator*)pNode;
nodesDestroyList(pStmt->pProjectionList); nodesDestroyList(pStmt->pProjectionList);
@ -1046,7 +1062,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_TRANSACTIONS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT: case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: { case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
case QUERY_NODE_SHOW_VIEWS_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode; SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName); nodesDestroyNode(pStmt->pDbName);
nodesDestroyNode(pStmt->pTbName); nodesDestroyNode(pStmt->pTbName);
@ -1071,6 +1088,7 @@ void nodesDestroyNode(SNode* pNode) {
taosMemoryFreeClear(((SShowCreateTableStmt*)pNode)->pDbCfg); taosMemoryFreeClear(((SShowCreateTableStmt*)pNode)->pDbCfg);
destroyTableCfg((STableCfg*)(((SShowCreateTableStmt*)pNode)->pTableCfg)); destroyTableCfg((STableCfg*)(((SShowCreateTableStmt*)pNode)->pTableCfg));
break; break;
case QUERY_NODE_SHOW_CREATE_VIEW_STMT: // no pointer field
case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: // no pointer field case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: // no pointer field
case QUERY_NODE_KILL_CONNECTION_STMT: // no pointer field case QUERY_NODE_KILL_CONNECTION_STMT: // no pointer field
case QUERY_NODE_KILL_QUERY_STMT: // no pointer field case QUERY_NODE_KILL_QUERY_STMT: // no pointer field
@ -1115,6 +1133,15 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_RESTORE_MNODE_STMT: // no pointer field case QUERY_NODE_RESTORE_MNODE_STMT: // no pointer field
case QUERY_NODE_RESTORE_VNODE_STMT: // no pointer field case QUERY_NODE_RESTORE_VNODE_STMT: // no pointer field
break; break;
case QUERY_NODE_CREATE_VIEW_STMT: {
SCreateViewStmt* pStmt = (SCreateViewStmt*)pNode;
taosMemoryFree(pStmt->pQuerySql);
tFreeSCMCreateViewReq(&pStmt->createReq);
nodesDestroyNode(pStmt->pQuery);
break;
}
case QUERY_NODE_DROP_VIEW_STMT:
break;
case QUERY_NODE_LOGIC_PLAN_SCAN: { case QUERY_NODE_LOGIC_PLAN_SCAN: {
SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode); destroyLogicNode((SLogicNode*)pLogicNode);
@ -2349,4 +2376,4 @@ bool nodesIsStar(SNode* pNode) {
bool nodesIsTableStar(SNode* pNode) { bool nodesIsTableStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) && return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*")); (0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
} }

5
source/libs/parser/CMakeLists.txt
View File

@ -1,4 +1,9 @@
aux_source_directory(src PARSER_SRC) aux_source_directory(src PARSER_SRC)
IF (TD_ENTERPRISE)
LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/parserView.c)
ENDIF ()
IF (TD_BI_SUPPORT) IF (TD_BI_SUPPORT)
LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/bi/src/biRewriteQuery.c) LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/bi/src/biRewriteQuery.c)
ENDIF () ENDIF ()

4
source/libs/parser/inc/parAst.h
View File

@ -125,6 +125,7 @@ SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias); SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias); SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond); SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset); SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder); SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap); SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
@ -196,6 +197,7 @@ SNode* createShowTablesStmt(SAstCreateContext* pCxt, SShowTablesOption option, S
SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName);
SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type); SNode* createShowAliveStmt(SAstCreateContext* pCxt, SNode* pDbName, ENodeType type);
SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable);
SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable);
SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern); SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pLikePattern);
SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDnodeEndpoint); SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDnodeEndpoint);
@ -256,6 +258,8 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair*
SNode* pTagCond); SNode* pTagCond);
SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere); SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere);
SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery); SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery);
SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pView, const SToken* pAs, SNode* pQuery);
SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pView);
#ifdef __cplusplus #ifdef __cplusplus
} }

10
source/libs/parser/inc/parInt.h
View File

@ -22,6 +22,7 @@ extern "C" {
#include "parToken.h" #include "parToken.h"
#include "parUtil.h" #include "parUtil.h"
#include "parTranslater.h"
#include "parser.h" #include "parser.h"
#define QUERY_SMA_OPTIMIZE_DISABLE 0 #define QUERY_SMA_OPTIMIZE_DISABLE 0
@ -36,6 +37,15 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow); int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow);
int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placeholderNo, SArray** pPlaceholderValues);
int32_t translateTable(STranslateContext* pCxt, SNode** pTable);
int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput);
void tfreeSParseQueryRes(void* p);
#ifdef TD_ENTERPRISE
int32_t translateView(STranslateContext* pCxt, SNode** pTable, SName* pName);
int32_t getViewMetaFromMetaCache(STranslateContext* pCxt, SName* pName, SViewMeta** ppViewMeta);
#endif
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
source/libs/parser/inc/parTranslater.h
View File

@ -48,6 +48,8 @@ typedef struct STranslateContext {
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect); int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect);
int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput); int32_t findTable(STranslateContext* pCxt, const char* pTableAlias, STableNode** pOutput);
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

7
source/libs/parser/inc/parUtil.h
View File

@ -69,6 +69,7 @@ typedef struct SParseMetaCache {
SHashObj* pUdf; // key is funcName, element is SFuncInfo* SHashObj* pUdf; // key is funcName, element is SFuncInfo*
SHashObj* pTableIndex; // key is tbFName, element is SArray<STableIndexInfo>* SHashObj* pTableIndex; // key is tbFName, element is SArray<STableIndexInfo>*
SHashObj* pTableCfg; // key is tbFName, element is STableCfg* SHashObj* pTableCfg; // key is tbFName, element is STableCfg*
SHashObj* pViews; // key is viewFName, element is SViewMeta*
SArray* pDnodes; // element is SEpSet SArray* pDnodes; // element is SEpSet
bool dnodeRequired; bool dnodeRequired;
} SParseMetaCache; } SParseMetaCache;
@ -93,6 +94,8 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog
int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache); int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache);
int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache); int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache);
int32_t reserveViewMetaInCache(int32_t acctId, const char* pDb, const char* pView, SParseMetaCache* pMetaCache);
int32_t reserveViewMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache);
int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); int32_t reserveDbVgInfoInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache); int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache);
@ -100,11 +103,15 @@ int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCac
int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache);
int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache); SParseMetaCache* pMetaCache);
int32_t reserveViewUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache);
int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache); int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache);
int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveDnodeRequiredInCache(SParseMetaCache* pMetaCache); int32_t reserveDnodeRequiredInCache(SParseMetaCache* pMetaCache);
int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta); int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta);
int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta);
int32_t buildTableMetaFromViewMeta(STableMeta** pMeta, SViewMeta* pViewMeta);
int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo);
int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup);
int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId,

16
source/libs/parser/inc/sql.y
View File

@ -132,6 +132,7 @@ priv_type_list(A) ::= priv_type_list(B) NK_COMMA priv_type(C).
%destructor priv_type { } %destructor priv_type { }
priv_type(A) ::= READ. { A = PRIVILEGE_TYPE_READ; } priv_type(A) ::= READ. { A = PRIVILEGE_TYPE_READ; }
priv_type(A) ::= WRITE. { A = PRIVILEGE_TYPE_WRITE; } priv_type(A) ::= WRITE. { A = PRIVILEGE_TYPE_WRITE; }
priv_type(A) ::= ALTER. { A = PRIVILEGE_TYPE_ALTER; }
%type priv_level { STokenPair } %type priv_level { STokenPair }
%destructor priv_level { } %destructor priv_level { }
@ -304,6 +305,7 @@ retention_list(A) ::= retention(B).
retention_list(A) ::= retention_list(B) NK_COMMA retention(C). { A = addNodeToList(pCxt, B, C); } retention_list(A) ::= retention_list(B) NK_COMMA retention(C). { A = addNodeToList(pCxt, B, C); }
retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); } retention(A) ::= NK_VARIABLE(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }
retention(A) ::= NK_MINUS(B) NK_COLON NK_VARIABLE(C). { A = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &B), createDurationValueNode(pCxt, &C)); }
%type speed_opt { int32_t } %type speed_opt { int32_t }
%destructor speed_opt { } %destructor speed_opt { }
@ -507,6 +509,8 @@ cmd ::= SHOW VNODES.
// show alive // show alive
cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); } cmd ::= SHOW db_name_cond_opt(A) ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, A, QUERY_NODE_SHOW_DB_ALIVE_STMT); }
cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } cmd ::= SHOW CLUSTER ALIVE. { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); }
cmd ::= SHOW db_name_cond_opt(A) VIEWS. { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VIEWS_STMT, A, NULL, OP_TYPE_LIKE); }
cmd ::= SHOW CREATE VIEW full_table_name(A). { pCxt->pRootNode = createShowCreateViewStmt(pCxt, QUERY_NODE_SHOW_CREATE_VIEW_STMT, A); }
%type table_kind_db_name_cond_opt { SShowTablesOption } %type table_kind_db_name_cond_opt { SShowTablesOption }
%destructor table_kind_db_name_cond_opt { } %destructor table_kind_db_name_cond_opt { }
@ -647,6 +651,14 @@ language_opt(A) ::= LANGUAGE NK_STRING(B).
or_replace_opt(A) ::= . { A = false; } or_replace_opt(A) ::= . { A = false; }
or_replace_opt(A) ::= OR REPLACE. { A = true; } or_replace_opt(A) ::= OR REPLACE. { A = true; }
/************************************************ create/drop view **************************************************/
cmd ::= CREATE or_replace_opt(A) VIEW full_view_name(B) AS(C) query_or_subquery(D).
{ pCxt->pRootNode = createCreateViewStmt(pCxt, A, B, &C, D); }
cmd ::= DROP VIEW exists_opt(A) full_view_name(B). { pCxt->pRootNode = createDropViewStmt(pCxt, A, B); }
full_view_name(A) ::= view_name(B). { A = createViewNode(pCxt, NULL, &B); }
full_view_name(A) ::= db_name(B) NK_DOT view_name(C). { A = createViewNode(pCxt, &B, &C); }
/************************************************ create/drop stream **************************************************/ /************************************************ create/drop stream **************************************************/
cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO cmd ::= CREATE STREAM not_exists_opt(E) stream_name(A) stream_options(B) INTO
full_table_name(C) col_list_opt(H) tag_def_or_ref_opt(F) subtable_opt(G) full_table_name(C) col_list_opt(H) tag_def_or_ref_opt(F) subtable_opt(G)
@ -780,6 +792,10 @@ column_name(A) ::= NK_ID(B).
%destructor function_name { } %destructor function_name { }
function_name(A) ::= NK_ID(B). { A = B; } function_name(A) ::= NK_ID(B). { A = B; }
%type view_name { SToken }
%destructor view_name { }
view_name(A) ::= NK_ID(B). { A = B; }
%type table_alias { SToken } %type table_alias { SToken }
%destructor table_alias { } %destructor table_alias { }
table_alias(A) ::= NK_ID(B). { A = B; } table_alias(A) ::= NK_ID(B). { A = B; }

78
source/libs/parser/src/parAstCreater.c
View File

@ -219,6 +219,16 @@ static bool checkCGroupName(SAstCreateContext* pCxt, SToken* pCGroup) {
return true; return true;
} }
static bool checkViewName(SAstCreateContext* pCxt, SToken* pViewName) {
trimEscape(pViewName);
if (pViewName->n >= TSDB_VIEW_NAME_LEN) {
pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, pViewName->z);
return false;
}
return true;
}
static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) { static bool checkStreamName(SAstCreateContext* pCxt, SToken* pStreamName) {
trimEscape(pStreamName); trimEscape(pStreamName);
if (pStreamName->n >= TSDB_STREAM_NAME_LEN) { if (pStreamName->n >= TSDB_STREAM_NAME_LEN) {
@ -772,6 +782,23 @@ SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft
return (SNode*)joinTable; return (SNode*)joinTable;
} }
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName) {
CHECK_PARSER_STATUS(pCxt);
if (!checkDbName(pCxt, pDbName, true) || !checkViewName(pCxt, pViewName)) {
return NULL;
}
SViewNode* pView = (SViewNode*)nodesMakeNode(QUERY_NODE_VIEW);
CHECK_OUT_OF_MEM(pView);
if (NULL != pDbName) {
COPY_STRING_FORM_ID_TOKEN(pView->table.dbName, pDbName);
} else {
snprintf(pView->table.dbName, sizeof(pView->table.dbName), "%s", pCxt->pQueryCxt->db);
}
COPY_STRING_FORM_ID_TOKEN(pView->table.tableName, pViewName);
return (SNode*)pView;
}
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) { SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT); SLimitNode* limitNode = (SLimitNode*)nodesMakeNode(QUERY_NODE_LIMIT);
@ -1028,9 +1055,13 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr
return select; return select;
} }
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) { SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags) {
if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { if (pStmt && QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
((SSelectStmt*)pStmt)->tagScan = bSelectTags; if (pCxt->pQueryCxt->biMode) {
((SSelectStmt*)pStmt)->tagScan = true;
} else {
((SSelectStmt*)pStmt)->tagScan = bSelectTags;
}
} }
return pStmt; return pStmt;
} }
@ -1596,7 +1627,8 @@ SNode* createUseDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName) {
static bool needDbShowStmt(ENodeType type) { static bool needDbShowStmt(ENodeType type) {
return QUERY_NODE_SHOW_TABLES_STMT == type || QUERY_NODE_SHOW_STABLES_STMT == type || return QUERY_NODE_SHOW_TABLES_STMT == type || QUERY_NODE_SHOW_STABLES_STMT == type ||
QUERY_NODE_SHOW_VGROUPS_STMT == type || QUERY_NODE_SHOW_INDEXES_STMT == type || QUERY_NODE_SHOW_VGROUPS_STMT == type || QUERY_NODE_SHOW_INDEXES_STMT == type ||
QUERY_NODE_SHOW_TAGS_STMT == type || QUERY_NODE_SHOW_TABLE_TAGS_STMT == type; QUERY_NODE_SHOW_TAGS_STMT == type || QUERY_NODE_SHOW_TABLE_TAGS_STMT == type ||
QUERY_NODE_SHOW_VIEWS_STMT == type;
} }
SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) { SNode* createShowStmt(SAstCreateContext* pCxt, ENodeType type) {
@ -1697,6 +1729,17 @@ SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode*
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createShowCreateViewStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable) {
CHECK_PARSER_STATUS(pCxt);
SShowCreateViewStmt* pStmt = (SShowCreateViewStmt*)nodesMakeNode(type);
CHECK_OUT_OF_MEM(pStmt);
strcpy(pStmt->dbName, ((SRealTableNode*)pRealTable)->table.dbName);
strcpy(pStmt->viewName, ((SRealTableNode*)pRealTable)->table.tableName);
nodesDestroyNode(pRealTable);
return (SNode*)pStmt;
}
SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) { SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SShowTableDistributedStmt* pStmt = (SShowTableDistributedStmt*)nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT); SShowTableDistributedStmt* pStmt = (SShowTableDistributedStmt*)nodesMakeNode(QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT);
@ -2211,6 +2254,35 @@ SNode* createDropFunctionStmt(SAstCreateContext* pCxt, bool ignoreNotExists, con
return (SNode*)pStmt; return (SNode*)pStmt;
} }
SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pView, const SToken* pAs, SNode* pQuery) {
CHECK_PARSER_STATUS(pCxt);
SCreateViewStmt* pStmt = (SCreateViewStmt*)nodesMakeNode(QUERY_NODE_CREATE_VIEW_STMT);
CHECK_OUT_OF_MEM(pStmt);
int32_t i = pAs->n;
while (isspace(*(pAs->z + i))) {
++i;
}
pStmt->pQuerySql = tstrdup(pAs->z + i);
CHECK_OUT_OF_MEM(pStmt->pQuerySql);
strcpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName);
strcpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName);
nodesDestroyNode(pView);
pStmt->orReplace = orReplace;
pStmt->pQuery = pQuery;
return (SNode*)pStmt;
}
SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pView) {
CHECK_PARSER_STATUS(pCxt);
SDropViewStmt* pStmt = (SDropViewStmt*)nodesMakeNode(QUERY_NODE_DROP_VIEW_STMT);
CHECK_OUT_OF_MEM(pStmt);
pStmt->ignoreNotExists = ignoreNotExists;
strcpy(pStmt->dbName, ((SViewNode*)pView)->table.dbName);
strcpy(pStmt->viewName, ((SViewNode*)pView)->table.tableName);
nodesDestroyNode(pView);
return (SNode*)pStmt;
}
SNode* createStreamOptions(SAstCreateContext* pCxt) { SNode* createStreamOptions(SAstCreateContext* pCxt) {
CHECK_PARSER_STATUS(pCxt); CHECK_PARSER_STATUS(pCxt);
SStreamOptions* pOptions = (SStreamOptions*)nodesMakeNode(QUERY_NODE_STREAM_OPTIONS); SStreamOptions* pOptions = (SStreamOptions*)nodesMakeNode(QUERY_NODE_STREAM_OPTIONS);

107
source/libs/parser/src/parAstParser.c
View File

@ -29,6 +29,20 @@ extern void Parse(void*, int, SToken, void*);
extern void ParseFree(void*, FFree); extern void ParseFree(void*, FFree);
extern void ParseTrace(FILE*, char*); extern void ParseTrace(FILE*, char*);
int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placeholderNo, SArray** pPlaceholderValues) {
*pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY);
if (NULL == *pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pQuery)->pRoot = pRootNode;
(*pQuery)->placeholderNum = placeholderNo;
TSWAP((*pQuery)->pPlaceholderValues, *pPlaceholderValues);
(*pQuery)->execStage = QUERY_EXEC_STAGE_ANALYSE;
return TSDB_CODE_SUCCESS;
}
int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) { int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) {
SAstCreateContext cxt; SAstCreateContext cxt;
initAstCreateContext(pParseCxt, &cxt); initAstCreateContext(pParseCxt, &cxt);
@ -77,14 +91,10 @@ int32_t parse(SParseContext* pParseCxt, SQuery** pQuery) {
abort_parse: abort_parse:
ParseFree(pParser, (FFree)taosMemoryFree); ParseFree(pParser, (FFree)taosMemoryFree);
if (TSDB_CODE_SUCCESS == cxt.errCode) { if (TSDB_CODE_SUCCESS == cxt.errCode) {
*pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); int32_t code = buildQueryAfterParse(pQuery, cxt.pRootNode, cxt.placeholderNo, &cxt.pPlaceholderValues);
if (NULL == *pQuery) { if (TSDB_CODE_SUCCESS != code) {
return TSDB_CODE_OUT_OF_MEMORY; return code;
} }
(*pQuery)->pRoot = cxt.pRootNode;
(*pQuery)->placeholderNum = cxt.placeholderNo;
TSWAP((*pQuery)->pPlaceholderValues, cxt.pPlaceholderValues);
(*pQuery)->execStage = QUERY_EXEC_STAGE_ANALYSE;
} }
taosArrayDestroy(cxt.pPlaceholderValues); taosArrayDestroy(cxt.pPlaceholderValues);
return cxt.errCode; return cxt.errCode;
@ -157,6 +167,12 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, pTable, authType, code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, pTable, authType,
pCxt->pMetaCache); pCxt->pMetaCache);
} }
#ifdef TD_ENTERPRISE
if (TSDB_CODE_SUCCESS == code && NULL != pCxt->pParseCxt->pEffectiveUser) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pEffectiveUser, pDb, pTable, authType,
pCxt->pMetaCache);
}
#endif
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pDb, pCxt->pMetaCache); code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pDb, pCxt->pMetaCache);
} }
@ -371,6 +387,13 @@ static int32_t collectMetaKeyFromDescribe(SCollectMetaKeyCxt* pCxt, SDescribeStm
strcpy(name.dbname, pStmt->dbName); strcpy(name.dbname, pStmt->dbName);
strcpy(name.tname, pStmt->tableName); strcpy(name.tname, pStmt->tableName);
int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name);
#ifdef TD_ENTERPRISE
if (TSDB_CODE_SUCCESS == code) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&name, dbFName);
code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->tableName, 0);
}
#endif
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache);
} }
@ -573,6 +596,18 @@ static int32_t collectMetaKeyFromShowUserPrivileges(SCollectMetaKeyCxt* pCxt, SS
pCxt->pMetaCache); pCxt->pMetaCache);
} }
static int32_t collectMetaKeyFromShowViews(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VIEWS,
pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code =
reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal,
NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) { static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) {
return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache);
} }
@ -595,6 +630,25 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow
return code; return code;
} }
static int32_t collectMetaKeyFromShowCreateView(SCollectMetaKeyCxt* pCxt, SShowCreateViewStmt* pStmt) {
SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId};
strcpy(name.dbname, pStmt->dbName);
strcpy(name.tname, pStmt->viewName);
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&name, dbFName);
int32_t code = catalogRemoveViewMeta(pCxt->pParseCxt->pCatalog, dbFName, 0, pStmt->viewName, 0);
if (TSDB_CODE_SUCCESS == code) {
code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName,
AUTH_TYPE_READ, pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromShowApps(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { static int32_t collectMetaKeyFromShowApps(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) {
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_APPS, return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_PERFS_TABLE_APPS,
pCxt->pMetaCache); pCxt->pMetaCache);
@ -646,6 +700,35 @@ static int32_t collectMetaKeyFromGrant(SCollectMetaKeyCxt* pCxt, SGrantStmt* pSt
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache); return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache);
} }
static int32_t collectMetaKeyFromRevoke(SCollectMetaKeyCxt* pCxt, SRevokeStmt* pStmt) {
if ('\0' == pStmt->tabName[0]) {
return TSDB_CODE_SUCCESS;
}
return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache);
}
static int32_t collectMetaKeyFromCreateViewStmt(SCollectMetaKeyCxt* pCxt, SCreateViewStmt* pStmt) {
int32_t code =
reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, pCxt->pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_WRITE,
pCxt->pMetaCache);
}
if (TSDB_CODE_SUCCESS == code) {
code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
pCxt->pMetaCache);
}
return code;
}
static int32_t collectMetaKeyFromDropViewStmt(SCollectMetaKeyCxt* pCxt, SDropViewStmt* pStmt) {
int32_t code = reserveViewUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER,
pCxt->pMetaCache);
return code;
}
static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
pCxt->pStmt = pStmt; pCxt->pStmt = pStmt;
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
@ -685,6 +768,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt); return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt);
case QUERY_NODE_GRANT_STMT: case QUERY_NODE_GRANT_STMT:
return collectMetaKeyFromGrant(pCxt, (SGrantStmt*)pStmt); return collectMetaKeyFromGrant(pCxt, (SGrantStmt*)pStmt);
case QUERY_NODE_REVOKE_STMT:
return collectMetaKeyFromRevoke(pCxt, (SRevokeStmt*)pStmt);
case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_DNODES_STMT:
return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt); return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_MNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT:
@ -737,11 +822,15 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowVnodes(pCxt, (SShowVnodesStmt*)pStmt); return collectMetaKeyFromShowVnodes(pCxt, (SShowVnodesStmt*)pStmt);
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
return collectMetaKeyFromShowUserPrivileges(pCxt, (SShowStmt*)pStmt); return collectMetaKeyFromShowUserPrivileges(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_VIEWS_STMT:
return collectMetaKeyFromShowViews(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT:
return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt); return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return collectMetaKeyFromShowCreateTable(pCxt, (SShowCreateTableStmt*)pStmt); return collectMetaKeyFromShowCreateTable(pCxt, (SShowCreateTableStmt*)pStmt);
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
return collectMetaKeyFromShowCreateView(pCxt, (SShowCreateViewStmt*)pStmt);
case QUERY_NODE_SHOW_APPS_STMT: case QUERY_NODE_SHOW_APPS_STMT:
return collectMetaKeyFromShowApps(pCxt, (SShowStmt*)pStmt); return collectMetaKeyFromShowApps(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_SHOW_TRANSACTIONS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT:
@ -754,6 +843,10 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) {
return collectMetaKeyFromShowBlockDist(pCxt, (SShowTableDistributedStmt*)pStmt); return collectMetaKeyFromShowBlockDist(pCxt, (SShowTableDistributedStmt*)pStmt);
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
return collectMetaKeyFromShowSubscriptions(pCxt, (SShowStmt*)pStmt); return collectMetaKeyFromShowSubscriptions(pCxt, (SShowStmt*)pStmt);
case QUERY_NODE_CREATE_VIEW_STMT:
return collectMetaKeyFromCreateViewStmt(pCxt, (SCreateViewStmt*)pStmt);
case QUERY_NODE_DROP_VIEW_STMT:
return collectMetaKeyFromDropViewStmt(pCxt, (SDropViewStmt*)pStmt);
default: default:
break; break;
} }

97
source/libs/parser/src/parAuthenticator.c
View File

@ -35,28 +35,41 @@ typedef struct SAuthRewriteCxt {
static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt); static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt);
static void setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, static void setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type,
SUserAuthInfo* pAuth) { bool isView, bool effective, SUserAuthInfo* pAuth) {
snprintf(pAuth->user, sizeof(pAuth->user), "%s", pCxt->pUser); if (effective) {
snprintf(pAuth->user, sizeof(pAuth->user), "%s", pCxt->pEffectiveUser ? pCxt->pEffectiveUser : "");
} else {
snprintf(pAuth->user, sizeof(pAuth->user), "%s", pCxt->pUser);
}
if (NULL == pTabName) { if (NULL == pTabName) {
tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName)); tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName));
} else { } else {
toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName);
} }
pAuth->type = type; pAuth->type = type;
pAuth->isView = isView;
} }
static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) { static int32_t checkAuthImpl(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond, bool isView, bool effective) {
SParseContext* pParseCxt = pCxt->pParseCxt; SParseContext* pParseCxt = pCxt->pParseCxt;
if (pParseCxt->isSuperUser) { if (pParseCxt->isSuperUser) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
AUTH_RES_TYPE auth_res_type = isView ? AUTH_RES_VIEW : AUTH_RES_BASIC;
SUserAuthInfo authInfo = {0}; SUserAuthInfo authInfo = {0};
setUserAuthInfo(pCxt->pParseCxt, pDbName, pTabName, type, &authInfo); setUserAuthInfo(pCxt->pParseCxt, pDbName, pTabName, type, isView, effective, &authInfo);
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SUserAuthRes authRes = {0}; SUserAuthRes authRes = {0};
if (NULL != pCxt->pMetaCache) { if (NULL != pCxt->pMetaCache) {
code = getUserAuthFromCache(pCxt->pMetaCache, &authInfo, &authRes); code = getUserAuthFromCache(pCxt->pMetaCache, &authInfo, &authRes);
#ifdef TD_ENTERPRISE
if (isView && TSDB_CODE_PAR_INTERNAL_ERROR == code) {
authInfo.isView = false;
code = getUserAuthFromCache(pCxt->pMetaCache, &authInfo, &authRes);
}
#endif
} else { } else {
SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter, SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter,
.requestId = pParseCxt->requestId, .requestId = pParseCxt->requestId,
@ -65,9 +78,26 @@ static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabNa
code = catalogChkAuth(pParseCxt->pCatalog, &conn, &authInfo, &authRes); code = catalogChkAuth(pParseCxt->pCatalog, &conn, &authInfo, &authRes);
} }
if (TSDB_CODE_SUCCESS == code && NULL != pCond) { if (TSDB_CODE_SUCCESS == code && NULL != pCond) {
*pCond = authRes.pCond; *pCond = authRes.pCond[auth_res_type];
} }
return TSDB_CODE_SUCCESS == code ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; return TSDB_CODE_SUCCESS == code ? (authRes.pass[auth_res_type] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code;
}
static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
return checkAuthImpl(pCxt, pDbName, pTabName, type, pCond, false, false);
}
static int32_t checkEffectiveAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
return checkAuthImpl(pCxt, pDbName, pTabName, type, NULL, false, true);
}
static int32_t checkViewAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
return checkAuthImpl(pCxt, pDbName, pTabName, type, NULL, true, false);
}
static int32_t checkViewEffectiveAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) {
return checkAuthImpl(pCxt, pDbName, pTabName, type, NULL, true, true);
} }
static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) { static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) {
@ -130,12 +160,33 @@ static int32_t rewriteAppendStableTagCond(SNode** pWhere, SNode* pTagCond, STabl
static EDealRes authSelectImpl(SNode* pNode, void* pContext) { static EDealRes authSelectImpl(SNode* pNode, void* pContext) {
SSelectAuthCxt* pCxt = pContext; SSelectAuthCxt* pCxt = pContext;
SAuthCxt* pAuthCxt = pCxt->pAuthCxt; SAuthCxt* pAuthCxt = pCxt->pAuthCxt;
bool isView = false;
if (QUERY_NODE_REAL_TABLE == nodeType(pNode)) { if (QUERY_NODE_REAL_TABLE == nodeType(pNode)) {
SNode* pTagCond = NULL; SNode* pTagCond = NULL;
STableNode* pTable = (STableNode*)pNode; STableNode* pTable = (STableNode*)pNode;
pAuthCxt->errCode = checkAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, &pTagCond); #ifdef TD_ENTERPRISE
if (TSDB_CODE_SUCCESS == pAuthCxt->errCode && NULL != pTagCond) { SName name;
pAuthCxt->errCode = rewriteAppendStableTagCond(&pCxt->pSelect->pWhere, pTagCond, pTable); STableMeta* pTableMeta = NULL;
int32_t code = getTargetMetaImpl(
pAuthCxt->pParseCxt, pAuthCxt->pMetaCache, toName(pAuthCxt->pParseCxt->acctId, pTable->dbName, pTable->tableName, &name), &pTableMeta, true);
if (TSDB_CODE_SUCCESS == code && TSDB_VIEW_TABLE == pTableMeta->tableType) {
isView = true;
}
taosMemoryFree(pTableMeta);
#endif
if (!isView) {
pAuthCxt->errCode = checkAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, &pTagCond);
if (TSDB_CODE_SUCCESS != pAuthCxt->errCode && NULL != pAuthCxt->pParseCxt->pEffectiveUser) {
pAuthCxt->errCode = checkEffectiveAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, NULL);
}
if (TSDB_CODE_SUCCESS == pAuthCxt->errCode && NULL != pTagCond) {
pAuthCxt->errCode = rewriteAppendStableTagCond(&pCxt->pSelect->pWhere, pTagCond, pTable);
}
} else {
pAuthCxt->errCode = checkViewAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, NULL);
if (TSDB_CODE_SUCCESS != pAuthCxt->errCode && NULL != pAuthCxt->pParseCxt->pEffectiveUser) {
pAuthCxt->errCode = checkViewEffectiveAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, NULL);
}
} }
return TSDB_CODE_SUCCESS == pAuthCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; return TSDB_CODE_SUCCESS == pAuthCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR;
} else if (QUERY_NODE_TEMP_TABLE == nodeType(pNode)) { } else if (QUERY_NODE_TEMP_TABLE == nodeType(pNode)) {
@ -196,6 +247,14 @@ static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt)
return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond); return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_READ, &pTagCond);
} }
static int32_t authShowCreateView(SAuthCxt* pCxt, SShowCreateViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
return TSDB_CODE_SUCCESS;
}
static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) {
SNode* pTagCond = NULL; SNode* pTagCond = NULL;
// todo check tag condition for subtable // todo check tag condition for subtable
@ -238,6 +297,20 @@ static int32_t authAlterTable(SAuthCxt* pCxt, SAlterTableStmt* pStmt) {
return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL); return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL);
} }
static int32_t authCreateView(SAuthCxt* pCxt, SCreateViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_WRITE, NULL);
}
static int32_t authDropView(SAuthCxt* pCxt, SDropViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
return checkViewAuth(pCxt, pStmt->dbName, pStmt->viewName, AUTH_TYPE_ALTER, NULL);
}
static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
switch (nodeType(pStmt)) { switch (nodeType(pStmt)) {
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
@ -283,6 +356,12 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) {
case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return authShowCreateTable(pCxt, (SShowCreateTableStmt*)pStmt); return authShowCreateTable(pCxt, (SShowCreateTableStmt*)pStmt);
// case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
// return authShowCreateView(pCxt, (SShowCreateViewStmt*)pStmt);
case QUERY_NODE_CREATE_VIEW_STMT:
return authCreateView(pCxt, (SCreateViewStmt*)pStmt);
case QUERY_NODE_DROP_VIEW_STMT:
return authDropView(pCxt, (SDropViewStmt*)pStmt);
default: default:
break; break;
} }

67
source/libs/parser/src/parCalcConst.c
View File

@ -18,10 +18,17 @@
#include "scalar.h" #include "scalar.h"
#include "ttime.h" #include "ttime.h"
typedef struct SNodeReplaceContext {
SNode* pTarget;
SNode* pNew;
bool replaced;
} SNodeReplaceContext;
typedef struct SCalcConstContext { typedef struct SCalcConstContext {
SParseContext* pParseCxt; SParseContext* pParseCxt;
SMsgBuf msgBuf; SNodeReplaceContext replaceCxt;
int32_t code; SMsgBuf msgBuf;
int32_t code;
} SCalcConstContext; } SCalcConstContext;
static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery); static int32_t calcConstQuery(SCalcConstContext* pCxt, SNode* pStmt, bool subquery);
@ -166,7 +173,35 @@ static int32_t calcConstStmtCondition(SCalcConstContext* pCxt, SNode** pCond, bo
return code; return code;
} }
static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) { static EDealRes doFindAndReplaceNode(SNode** pNode, void* pContext) {
SCalcConstContext* pCxt = pContext;
if (pCxt->replaceCxt.pTarget == *pNode) {
nodesDestroyNode(*pNode);
*pNode = nodesCloneNode(pCxt->replaceCxt.pNew);
if (NULL == *pNode) {
pCxt->code = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
}
pCxt->replaceCxt.replaced = true;
return DEAL_RES_END;
}
return DEAL_RES_CONTINUE;
}
static int32_t findAndReplaceNode(SCalcConstContext* pCxt, SNode** pRoot, SNode* pTarget, SNode* pNew, bool strict) {
pCxt->replaceCxt.pNew = pNew;
pCxt->replaceCxt.pTarget = pTarget;
nodesRewriteExprPostOrder(pRoot, doFindAndReplaceNode, pCxt);
if (TSDB_CODE_SUCCESS == pCxt->code && strict && !pCxt->replaceCxt.replaced) {
parserError("target replace node not found, %p", pTarget);
return TSDB_CODE_PAR_INTERNAL_ERROR;
}
return pCxt->code;
}
static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool dual, SNode** pNew) {
SArray* pAssociation = NULL; SArray* pAssociation = NULL;
if (NULL != ((SExprNode*)pProject)->pAssociation) { if (NULL != ((SExprNode*)pProject)->pAssociation) {
pAssociation = taosArrayDup(((SExprNode*)pProject)->pAssociation, NULL); pAssociation = taosArrayDup(((SExprNode*)pProject)->pAssociation, NULL);
@ -188,12 +223,20 @@ static int32_t calcConstProject(SNode* pProject, bool dual, SNode** pNew) {
if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) { if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) {
int32_t size = taosArrayGetSize(pAssociation); int32_t size = taosArrayGetSize(pAssociation);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SNode** pCol = taosArrayGetP(pAssociation, i); SAssociationNode* pAssNode = taosArrayGet(pAssociation, i);
nodesDestroyNode(*pCol); SNode** pCol = pAssNode->pPlace;
*pCol = nodesCloneNode(*pNew); if (*pCol == pAssNode->pAssociationNode) {
if (NULL == *pCol) { nodesDestroyNode(*pCol);
code = TSDB_CODE_OUT_OF_MEMORY; *pCol = nodesCloneNode(*pNew);
break; if (NULL == *pCol) {
code = TSDB_CODE_OUT_OF_MEMORY;
break;
}
} else {
code = findAndReplaceNode(pCxt, pCol, pAssNode->pAssociationNode, *pNew, true);
if (TSDB_CODE_SUCCESS != code) {
break;
}
} }
} }
} }
@ -247,7 +290,7 @@ static int32_t calcConstProjections(SCalcConstContext* pCxt, SSelectStmt* pSelec
continue; continue;
} }
SNode* pNew = NULL; SNode* pNew = NULL;
int32_t code = calcConstProject(pProj, (NULL == pSelect->pFromTable), &pNew); int32_t code = calcConstProject(pCxt, pProj, (NULL == pSelect->pFromTable), &pNew);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
REPLACE_NODE(pNew); REPLACE_NODE(pNew);
} else { } else {
@ -514,7 +557,7 @@ static void resetProjectNullTypeImpl(SNodeList* pProjects) {
SExprNode* pExpr = (SExprNode*)pProj; SExprNode* pExpr = (SExprNode*)pProj;
if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) { if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) {
pExpr->resType.type = TSDB_DATA_TYPE_VARCHAR; pExpr->resType.type = TSDB_DATA_TYPE_VARCHAR;
pExpr->resType.bytes = 0; pExpr->resType.bytes = VARSTR_HEADER_SIZE;
} }
} }
} }

19
source/libs/parser/src/parInsertSql.c
View File

@ -965,10 +965,10 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache,
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (!exists) { if (!exists) {
*pMissCache = true; *pMissCache = true;
} else if (!authRes.pass) { } else if (!authRes.pass[AUTH_RES_BASIC]) {
code = TSDB_CODE_PAR_PERMISSION_DENIED; code = TSDB_CODE_PAR_PERMISSION_DENIED;
} else if (NULL != authRes.pCond) { } else if (NULL != authRes.pCond[AUTH_RES_BASIC]) {
*pTagCond = authRes.pCond; *pTagCond = authRes.pCond[AUTH_RES_BASIC];
} }
} }
return code; return code;
@ -1168,7 +1168,14 @@ static int32_t parseUsingTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt
} }
static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) { static int32_t preParseTargetTableName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SToken* pTbName) {
return insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg); int32_t code = insCreateSName(&pStmt->targetTableName, pTbName, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
if (TSDB_CODE_SUCCESS == code) {
if (IS_SYS_DBNAME(pStmt->targetTableName.dbname)) {
return TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED;
}
}
return code;
} }
// input pStmt->pSql: // input pStmt->pSql:
@ -1952,9 +1959,9 @@ static int32_t checkAuthFromMetaData(const SArray* pUsers, SNode** pTagCond) {
if (TSDB_CODE_SUCCESS == pRes->code) { if (TSDB_CODE_SUCCESS == pRes->code) {
SUserAuthRes* pAuth = pRes->pRes; SUserAuthRes* pAuth = pRes->pRes;
if (NULL != pAuth->pCond) { if (NULL != pAuth->pCond) {
*pTagCond = nodesCloneNode(pAuth->pCond); *pTagCond = nodesCloneNode(pAuth->pCond[AUTH_RES_BASIC]);
} }
return pAuth->pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED; return pAuth->pass[AUTH_RES_BASIC] ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED;
} }
return pRes->code; return pRes->code;
} }

2
source/libs/parser/src/parTokenizer.c
View File

@ -266,6 +266,8 @@ static SKeyword keywordTable[] = {
{"VERBOSE", TK_VERBOSE}, {"VERBOSE", TK_VERBOSE},
{"VGROUP", TK_VGROUP}, {"VGROUP", TK_VGROUP},
{"VGROUPS", TK_VGROUPS}, {"VGROUPS", TK_VGROUPS},
{"VIEW", TK_VIEW},
{"VIEWS", TK_VIEWS},
{"VNODE", TK_VNODE}, {"VNODE", TK_VNODE},
{"VNODES", TK_VNODES}, {"VNODES", TK_VNODES},
{"WAL_FSYNC_PERIOD", TK_WAL_FSYNC_PERIOD}, {"WAL_FSYNC_PERIOD", TK_WAL_FSYNC_PERIOD},

395
source/libs/parser/src/parTranslater.c
View File

@ -251,6 +251,12 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
.numOfShowCols = 1, .numOfShowCols = 1,
.pShowCols = {"*"} .pShowCols = {"*"}
}, },
{ .showType = QUERY_NODE_SHOW_VIEWS_STMT,
.pDbName = TSDB_INFORMATION_SCHEMA_DB,
.pTableName = TSDB_INS_TABLE_VIEWS,
.numOfShowCols = 1,
.pShowCols = {"*"}
},
}; };
// clang-format on // clang-format on
@ -335,22 +341,67 @@ static int32_t collectUseTable(const SName* pName, SHashObj* pTable) {
return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName)); return taosHashPut(pTable, fullName, strlen(fullName), pName, sizeof(SName));
} }
static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STableMeta** pMeta) { static int32_t getViewMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_PAR_TABLE_NOT_EXIST;
#endif
int32_t code = TSDB_CODE_SUCCESS;
if (pParCxt->async) {
code = getViewMetaFromCache(pMetaCache, pName, pMeta);
} else {
SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter,
.requestId = pParCxt->requestId,
.requestObjRefId = pParCxt->requestRid,
.mgmtEps = pParCxt->mgmtEpSet};
code = catalogGetViewMeta(pParCxt->pCatalog, &conn, pName, pMeta);
}
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
parserError("0x%" PRIx64 " catalogGetViewMeta error, code:%s, dbName:%s, viewName:%s", pParCxt->requestId,
tstrerror(code), pName->dbname, pName->tname);
}
return code;
}
int32_t getTargetMetaImpl(SParseContext* pParCxt, SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta, bool couldBeView) {
int32_t code = TSDB_CODE_SUCCESS;
if (pParCxt->async) {
code = getTableMetaFromCache(pMetaCache, pName, pMeta);
#ifdef TD_ENTERPRISE
if ((TSDB_CODE_PAR_TABLE_NOT_EXIST == code || TSDB_CODE_PAR_INTERNAL_ERROR == code) && couldBeView) {
int32_t origCode = code;
code = getViewMetaFromCache(pMetaCache, pName, pMeta);
if (TSDB_CODE_SUCCESS != code) {
code = origCode;
}
}
#endif
} else {
SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter,
.requestId = pParCxt->requestId,
.requestObjRefId = pParCxt->requestRid,
.mgmtEps = pParCxt->mgmtEpSet};
code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta);
}
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pParCxt->requestId,
tstrerror(code), pName->dbname, pName->tname);
}
return code;
}
static int32_t getTargetMeta(STranslateContext* pCxt, const SName* pName, STableMeta** pMeta, bool couldBeView) {
SParseContext* pParCxt = pCxt->pParseCxt; SParseContext* pParCxt = pCxt->pParseCxt;
int32_t code = collectUseDatabase(pName, pCxt->pDbs); int32_t code = collectUseDatabase(pName, pCxt->pDbs);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(pName, pCxt->pTables); code = collectUseTable(pName, pCxt->pTables);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (pParCxt->async) { code = getTargetMetaImpl(pParCxt, pCxt->pMetaCache, pName, pMeta, couldBeView);
code = getTableMetaFromCache(pCxt->pMetaCache, pName, pMeta);
} else {
SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter,
.requestId = pParCxt->requestId,
.requestObjRefId = pParCxt->requestRid,
.mgmtEps = pParCxt->mgmtEpSet};
code = catalogGetTableMeta(pParCxt->pCatalog, &conn, pName, pMeta);
}
} }
if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) { if (TSDB_CODE_SUCCESS != code && TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId, parserError("0x%" PRIx64 " catalogGetTableMeta error, code:%s, dbName:%s, tbName:%s", pCxt->pParseCxt->requestId,
@ -361,7 +412,7 @@ static int32_t getTableMetaImpl(STranslateContext* pCxt, const SName* pName, STa
static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) {
SName name; SName name;
return getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta); return getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta, false);
} }
static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCfg** pCfg) { static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCfg** pCfg) {
@ -855,9 +906,13 @@ static void setColumnInfoByExpr(STempTableNode* pTable, SExprNode* pExpr, SColum
SColumnNode* pCol = *pColRef; SColumnNode* pCol = *pColRef;
if (NULL == pExpr->pAssociation) { if (NULL == pExpr->pAssociation) {
pExpr->pAssociation = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); pExpr->pAssociation = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SAssociationNode));
} }
taosArrayPush(pExpr->pAssociation, &pColRef); SAssociationNode assNode;
assNode.pPlace = (SNode**)pColRef;
assNode.pAssociationNode = (SNode*)*pColRef;
taosArrayPush(pExpr->pAssociation, &assNode);
strcpy(pCol->tableAlias, pTable->table.tableAlias); strcpy(pCol->tableAlias, pTable->table.tableAlias);
pCol->colId = isPrimaryKey(pTable, (SNode*)pExpr) ? PRIMARYKEY_TIMESTAMP_COL_ID : 0; pCol->colId = isPrimaryKey(pTable, (SNode*)pExpr) ? PRIMARYKEY_TIMESTAMP_COL_ID : 0;
strcpy(pCol->colName, pExpr->aliasName); strcpy(pCol->colName, pExpr->aliasName);
@ -2839,21 +2894,26 @@ static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTabl
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { int32_t translateTable(STranslateContext* pCxt, SNode** pTable) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pTable)) { switch (nodeType(*pTable)) {
case QUERY_NODE_REAL_TABLE: { case QUERY_NODE_REAL_TABLE: {
SRealTableNode* pRealTable = (SRealTableNode*)pTable; SRealTableNode* pRealTable = (SRealTableNode*)*pTable;
pRealTable->ratio = (NULL != pCxt->pExplainOpt ? pCxt->pExplainOpt->ratio : 1.0); pRealTable->ratio = (NULL != pCxt->pExplainOpt ? pCxt->pExplainOpt->ratio : 1.0);
// The SRealTableNode created through ROLLUP already has STableMeta. // The SRealTableNode created through ROLLUP already has STableMeta.
if (NULL == pRealTable->pMeta) { if (NULL == pRealTable->pMeta) {
SName name; SName name;
code = getTableMetaImpl( code = getTargetMeta(
pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name), pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name),
&(pRealTable->pMeta)); &(pRealTable->pMeta), true);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code)); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
} }
#ifdef TD_ENTERPRISE
if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType) {
return translateView(pCxt, pTable, &name);
}
#endif
code = setTableVgroupList(pCxt, &name, pRealTable); code = setTableVgroupList(pCxt, &name, pRealTable);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = setTableIndex(pCxt, &name, pRealTable); code = setTableIndex(pCxt, &name, pRealTable);
@ -2878,7 +2938,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
break; break;
} }
case QUERY_NODE_TEMP_TABLE: { case QUERY_NODE_TEMP_TABLE: {
STempTableNode* pTempTable = (STempTableNode*)pTable; STempTableNode* pTempTable = (STempTableNode*)*pTable;
code = translateSubquery(pCxt, pTempTable->pSubquery); code = translateSubquery(pCxt, pTempTable->pSubquery);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
if (QUERY_NODE_SELECT_STMT == nodeType(pTempTable->pSubquery) && if (QUERY_NODE_SELECT_STMT == nodeType(pTempTable->pSubquery) &&
@ -2893,10 +2953,10 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
break; break;
} }
case QUERY_NODE_JOIN_TABLE: { case QUERY_NODE_JOIN_TABLE: {
SJoinTableNode* pJoinTable = (SJoinTableNode*)pTable; SJoinTableNode* pJoinTable = (SJoinTableNode*)*pTable;
code = translateTable(pCxt, pJoinTable->pLeft); code = translateTable(pCxt, &pJoinTable->pLeft);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = translateTable(pCxt, pJoinTable->pRight); code = translateTable(pCxt, &pJoinTable->pRight);
} }
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = checkJoinTable(pCxt, pJoinTable); code = checkJoinTable(pCxt, pJoinTable);
@ -3877,7 +3937,7 @@ static int32_t translateWhere(STranslateContext* pCxt, SSelectStmt* pSelect) {
return code; return code;
} }
static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { static int32_t translateFrom(STranslateContext* pCxt, SNode** pTable) {
pCxt->currClause = SQL_CLAUSE_FROM; pCxt->currClause = SQL_CLAUSE_FROM;
return translateTable(pCxt, pTable); return translateTable(pCxt, pTable);
} }
@ -4006,7 +4066,7 @@ static int32_t translateSelectWithoutFrom(STranslateContext* pCxt, SSelectStmt*
static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) {
pCxt->pCurrStmt = (SNode*)pSelect; pCxt->pCurrStmt = (SNode*)pSelect;
int32_t code = translateFrom(pCxt, pSelect->pFromTable); int32_t code = translateFrom(pCxt, &pSelect->pFromTable);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision; pSelect->precision = ((STableNode*)pSelect->pFromTable)->precision;
code = translateWhere(pCxt, pSelect); code = translateWhere(pCxt, pSelect);
@ -4232,7 +4292,7 @@ static int32_t translateDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet
static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) {
pCxt->pCurrStmt = (SNode*)pDelete; pCxt->pCurrStmt = (SNode*)pDelete;
int32_t code = translateFrom(pCxt, pDelete->pFromTable); int32_t code = translateFrom(pCxt, &pDelete->pFromTable);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
pDelete->precision = ((STableNode*)pDelete->pFromTable)->precision; pDelete->precision = ((STableNode*)pDelete->pFromTable)->precision;
code = translateDeleteWhere(pCxt, pDelete); code = translateDeleteWhere(pCxt, pDelete);
@ -4313,7 +4373,7 @@ static int32_t translateInsertProject(STranslateContext* pCxt, SInsertStmt* pIns
} }
snprintf(pExpr->aliasName, sizeof(pExpr->aliasName), "%s", pCol->colName); snprintf(pExpr->aliasName, sizeof(pExpr->aliasName), "%s", pCol->colName);
if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) { if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId) {
pPrimaryKeyExpr = pProj; pPrimaryKeyExpr = (SNode*)pExpr;
} }
} }
@ -4325,19 +4385,19 @@ static int32_t translateInsertProject(STranslateContext* pCxt, SInsertStmt* pIns
return addOrderByPrimaryKeyToQuery(pCxt, pPrimaryKeyExpr, pInsert->pQuery); return addOrderByPrimaryKeyToQuery(pCxt, pPrimaryKeyExpr, pInsert->pQuery);
} }
static int32_t translateInsertTable(STranslateContext* pCxt, SNode* pTable) { static int32_t translateInsertTable(STranslateContext* pCxt, SNode** pTable) {
int32_t code = translateFrom(pCxt, pTable); int32_t code = translateFrom(pCxt, pTable);
if (TSDB_CODE_SUCCESS == code && TSDB_CHILD_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType && if (TSDB_CODE_SUCCESS == code && TSDB_CHILD_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType &&
TSDB_NORMAL_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType) { TSDB_NORMAL_TABLE != ((SRealTableNode*)*pTable)->pMeta->tableType) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, code = buildInvalidOperationMsg(&pCxt->msgBuf, "insert data into super table is not supported");
"insert data into super table is not supported");
} }
return code; return code;
} }
static int32_t translateInsert(STranslateContext* pCxt, SInsertStmt* pInsert) { static int32_t translateInsert(STranslateContext* pCxt, SInsertStmt* pInsert) {
pCxt->pCurrStmt = (SNode*)pInsert; pCxt->pCurrStmt = (SNode*)pInsert;
int32_t code = translateInsertTable(pCxt, pInsert->pTable); int32_t code = translateInsertTable(pCxt, &pInsert->pTable);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = translateInsertCols(pCxt, pInsert); code = translateInsertCols(pCxt, pInsert);
} }
@ -4605,10 +4665,22 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
SValueNode* pPrevFreq = NULL; SValueNode* pPrevFreq = NULL;
SValueNode* pPrevKeep = NULL; SValueNode* pPrevKeep = NULL;
SNode* pRetention = NULL; SNode* pRetention = NULL;
bool firstFreq = true;
FOREACH(pRetention, pRetentions) { FOREACH(pRetention, pRetentions) {
SNode* pNode = NULL; SNode* pNode = NULL;
FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) { FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) {
SValueNode* pVal = (SValueNode*)pNode; SValueNode* pVal = (SValueNode*)pNode;
if (firstFreq) {
firstFreq = false;
if (pVal->literal[0] != '-' || strlen(pVal->literal) != 1) {
return generateSyntaxErrMsgExt(
&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq): %s, the interval of 1st retention level should be '-'", pVal->literal);
}
pVal->unit = TIME_UNIT_SECOND; // assign minimum unit
pVal->datum.i = 0; // assign minimum value
continue;
}
if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) {
return pCxt->errCode; return pCxt->errCode;
} }
@ -4633,7 +4705,7 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
} }
// check value range // check value range
if (pFreq->datum.i <= 0) { if (pPrevFreq != NULL && pFreq->datum.i <= 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq): %s should larger than 0", pFreq->literal); "Invalid option retentions(freq): %s should larger than 0", pFreq->literal);
} }
@ -6571,7 +6643,7 @@ static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt
.mgmtEps = pParCxt->mgmtEpSet}; .mgmtEps = pParCxt->mgmtEpSet};
SName name; SName name;
STableMeta* pMeta = NULL; STableMeta* pMeta = NULL;
int32_t code = getTableMetaImpl(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta); int32_t code = getTargetMeta(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta, false);
if (code) { if (code) {
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
return code; return code;
@ -6675,7 +6747,41 @@ static int32_t translateExplain(STranslateContext* pCxt, SExplainStmt* pStmt) {
} }
static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) { static int32_t translateDescribe(STranslateContext* pCxt, SDescribeStmt* pStmt) {
return refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta); int32_t code = refreshGetTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta);
#ifdef TD_ENTERPRISE
if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) {
int32_t origCode = code;
SName name;
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name);
SViewMeta* pMeta = NULL;
code = getViewMetaFromMetaCache(pCxt, &name, &pMeta);
if (TSDB_CODE_SUCCESS != code) {
code = origCode;
} else {
SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA};
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(&name, dbFName);
code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, name.dbname, pMeta->querySql, false, pMeta->user, &res);
if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(&name, pCxt->pTargetTables);
}
if (TSDB_CODE_SUCCESS == code) {
SViewMeta viewMeta = {0};
viewMeta.viewId = pMeta->viewId;
viewMeta.precision = res.schemaRes.precision;
viewMeta.type = pMeta->type;
viewMeta.version = pMeta->version;
viewMeta.numOfCols = res.schemaRes.numOfCols;
viewMeta.pSchema = res.schemaRes.pSchema;
code = buildTableMetaFromViewMeta(&pStmt->pMeta, &viewMeta);
parserDebug("rebuild view meta, view:%s.%s, numOfCols:%d, code:0x%x", dbFName, pStmt->tableName, viewMeta.numOfCols, code);
}
taosMemoryFree(res.schemaRes.pSchema);
}
}
#endif
return code;
} }
static int32_t translateCompactRange(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt, SCompactDbReq* pReq) { static int32_t translateCompactRange(STranslateContext* pCxt, SCompactDatabaseStmt* pStmt, SCompactDbReq* pReq) {
@ -6750,6 +6856,26 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
} }
#ifdef TD_ENTERPRISE
SRealTableNode* pRealTable = (SRealTableNode*)((SSelectStmt*)pStmt->pQuery)->pFromTable;
SName name;
STableMeta* pMeta = NULL;
int8_t tableType = 0;
int32_t code = getTargetMeta(
pCxt, toName(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, &name),
&pMeta, true);
if (NULL != pMeta) {
tableType = pMeta->tableType;
taosMemoryFree(pMeta);
}
if (TSDB_CODE_SUCCESS != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
}
if (TSDB_VIEW_TABLE == tableType) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query");
}
#endif
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -7591,6 +7717,91 @@ static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt*
return buildCmdMsg(pCxt, TDMT_MND_RESUME_STREAM, (FSerializeFunc)tSerializeSMResumeStreamReq, &req); return buildCmdMsg(pCxt, TDMT_MND_RESUME_STREAM, (FSerializeFunc)tSerializeSMResumeStreamReq, &req);
} }
static int32_t validateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) {
if (QUERY_NODE_SELECT_STMT != nodeType(pStmt->pQuery) && QUERY_NODE_SET_OPERATOR != nodeType(pStmt->pQuery)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VIEW_QUERY, "Invalid view query type");
}
/*
STableMeta* pMetaCache = NULL;
int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->viewName, &pMetaCache);
if (TSDB_CODE_SUCCESS == code) {
taosMemoryFreeClear(pMetaCache);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE, "View name is conflict with table");
}
*/
return TSDB_CODE_SUCCESS;
}
static int32_t translateCreateView(STranslateContext* pCxt, SCreateViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
SParseSqlRes res = {.resType = PARSE_SQL_RES_SCHEMA};
SName name;
char dbFName[TSDB_DB_FNAME_LEN];
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name);
tNameGetFullDbName(&name, dbFName);
int32_t code = validateCreateView(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
code = (*pCxt->pParseCxt->parseSqlFp)(pCxt->pParseCxt->parseSqlParam, pStmt->dbName, pStmt->pQuerySql, false, NULL, &res);
}
if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(&name, pCxt->pTargetTables);
}
if (TSDB_CODE_SUCCESS == code) {
pStmt->createReq.precision = res.schemaRes.precision;
pStmt->createReq.numOfCols = res.schemaRes.numOfCols;
pStmt->createReq.pSchema = res.schemaRes.pSchema;
strncpy(pStmt->createReq.name, pStmt->viewName, sizeof(pStmt->createReq.name) - 1);
tstrncpy(pStmt->createReq.dbFName, dbFName, sizeof(pStmt->createReq.dbFName));
snprintf(pStmt->createReq.fullname, sizeof(pStmt->createReq.fullname) - 1, "%s.%s", pStmt->createReq.dbFName, pStmt->viewName);
TSWAP(pStmt->createReq.querySql, pStmt->pQuerySql);
pStmt->createReq.orReplace = pStmt->orReplace;
pStmt->createReq.sql = tstrdup(pCxt->pParseCxt->pSql);
if (NULL == pStmt->createReq.sql) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
if (TSDB_CODE_SUCCESS == code) {
code = buildCmdMsg(pCxt, TDMT_MND_CREATE_VIEW, (FSerializeFunc)tSerializeSCMCreateViewReq, &pStmt->createReq);
}
tFreeSCMCreateViewReq(&pStmt->createReq);
return code;
}
static int32_t translateDropView(STranslateContext* pCxt, SDropViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#endif
SCMDropViewReq dropReq = {0};
SName name;
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName));
tNameGetFullDbName(&name, dropReq.dbFName);
strncpy(dropReq.name, pStmt->viewName, sizeof(dropReq.name) - 1);
snprintf(dropReq.fullname, sizeof(dropReq.fullname) - 1, "%s.%s", dropReq.dbFName, dropReq.name);
dropReq.sql = (char*)pCxt->pParseCxt->pSql;
if (NULL == dropReq.sql) {
return TSDB_CODE_OUT_OF_MEMORY;
}
dropReq.igNotExists = pStmt->ignoreNotExists;
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name);
int32_t code = collectUseTable(&name, pCxt->pTargetTables);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
return buildCmdMsg(pCxt, TDMT_MND_DROP_VIEW, (FSerializeFunc)tSerializeSCMDropViewReq, &dropReq);
}
static int32_t readFromFile(char* pName, int32_t* len, char** buf) { static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
int64_t filesize = 0; int64_t filesize = 0;
if (taosStatFile(pName, &filesize, NULL, NULL) < 0) { if (taosStatFile(pName, &filesize, NULL, NULL) < 0) {
@ -7690,8 +7901,8 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
int32_t code = createRealTableForGrantTable(pStmt, &pTable); int32_t code = createRealTableForGrantTable(pStmt, &pTable);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SName name; SName name;
code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name),
&(pTable->pMeta)); &(pTable->pMeta), false);
if (code) { if (code) {
nodesDestroyNode((SNode*)pTable); nodesDestroyNode((SNode*)pTable);
return code; return code;
@ -7725,22 +7936,32 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
} }
static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
int32_t code = 0;
SAlterUserReq req = {0}; SAlterUserReq req = {0};
if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) || req.alterType = TSDB_ALTER_USER_ADD_PRIVILEGES;
(BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) && req.privileges = pStmt->privileges;
BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) { #ifdef TD_ENTERPRISE
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_ALL_DB : TSDB_ALTER_USER_ADD_ALL_TABLE); if (0 != pStmt->tabName[0]) {
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) { SName name;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_READ_DB : TSDB_ALTER_USER_ADD_READ_TABLE); STableMeta* pTableMeta = NULL;
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_WRITE_DB : TSDB_ALTER_USER_ADD_WRITE_TABLE); if (TSDB_CODE_SUCCESS != code) {
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) { if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
req.alterType = TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC; return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
}
} else if (TSDB_VIEW_TABLE == pTableMeta->tableType) {
req.isView = true;
}
taosMemoryFree(pTableMeta);
} }
#endif
strcpy(req.user, pStmt->userName); strcpy(req.user, pStmt->userName);
sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
sprintf(req.tabName, "%s", pStmt->tabName); sprintf(req.tabName, "%s", pStmt->tabName);
int32_t code = translateGrantTagCond(pCxt, pStmt, &req); if (!req.isView) {
code = translateGrantTagCond(pCxt, pStmt, &req);
}
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
} }
@ -7749,22 +7970,31 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) {
} }
static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) {
int32_t code = 0;
SAlterUserReq req = {0}; SAlterUserReq req = {0};
if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) || req.alterType = TSDB_ALTER_USER_DEL_PRIVILEGES;
(BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) && req.privileges = pStmt->privileges;
BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) {
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_ALL_DB : TSDB_ALTER_USER_REMOVE_ALL_TABLE); #ifdef TD_ENTERPRISE
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) { if (0 != pStmt->tabName[0]) {
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_READ_DB : TSDB_ALTER_USER_REMOVE_READ_TABLE); SName name;
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { STableMeta* pTableMeta = NULL;
req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_WRITE_DB : TSDB_ALTER_USER_REMOVE_WRITE_TABLE); code = getTargetMeta(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, &name), &pTableMeta, true);
} else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) { if (TSDB_CODE_SUCCESS != code) {
req.alterType = TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC; if (TSDB_CODE_PAR_TABLE_NOT_EXIST != code) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GET_META_ERROR, tstrerror(code));
}
} else if (TSDB_VIEW_TABLE == pTableMeta->tableType) {
req.isView = true;
}
taosMemoryFree(pTableMeta);
} }
#endif
strcpy(req.user, pStmt->userName); strcpy(req.user, pStmt->userName);
sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName);
sprintf(req.tabName, "%s", pStmt->tabName); sprintf(req.tabName, "%s", pStmt->tabName);
int32_t code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req);
tFreeSAlterUserReq(&req); tFreeSAlterUserReq(&req);
return code; return code;
} }
@ -7866,6 +8096,16 @@ static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTabl
return code; return code;
} }
static int32_t translateShowCreateView(STranslateContext* pCxt, SShowCreateViewStmt* pStmt) {
#ifndef TD_ENTERPRISE
return TSDB_CODE_OPS_NOT_SUPPORT;
#else
SName name;
toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->viewName, &name);
return getViewMetaFromMetaCache(pCxt, &name, (SViewMeta**)&pStmt->pViewMeta);
#endif
}
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
switch (nodeType(pNode)) { switch (nodeType(pNode)) {
@ -8024,12 +8264,22 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) {
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
code = translateShowCreateTable(pCxt, (SShowCreateTableStmt*)pNode); code = translateShowCreateTable(pCxt, (SShowCreateTableStmt*)pNode);
break; break;
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
code = translateShowCreateView(pCxt, (SShowCreateViewStmt*)pNode);
break;
case QUERY_NODE_RESTORE_DNODE_STMT: case QUERY_NODE_RESTORE_DNODE_STMT:
case QUERY_NODE_RESTORE_QNODE_STMT: case QUERY_NODE_RESTORE_QNODE_STMT:
case QUERY_NODE_RESTORE_MNODE_STMT: case QUERY_NODE_RESTORE_MNODE_STMT:
case QUERY_NODE_RESTORE_VNODE_STMT: case QUERY_NODE_RESTORE_VNODE_STMT:
code = translateRestoreDnode(pCxt, (SRestoreComponentNodeStmt*)pNode); code = translateRestoreDnode(pCxt, (SRestoreComponentNodeStmt*)pNode);
break; break;
case QUERY_NODE_CREATE_VIEW_STMT:
code = translateCreateView(pCxt, (SCreateViewStmt*)pNode);
break;
case QUERY_NODE_DROP_VIEW_STMT:
code = translateDropView(pCxt, (SDropViewStmt*)pNode);
break;
default: default:
break; break;
} }
@ -8061,7 +8311,7 @@ static int32_t extractQueryResultSchema(const SNodeList* pProjections, int32_t*
SExprNode* pExpr = (SExprNode*)pNode; SExprNode* pExpr = (SExprNode*)pNode;
if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) { if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) {
(*pSchema)[index].type = TSDB_DATA_TYPE_VARCHAR; (*pSchema)[index].type = TSDB_DATA_TYPE_VARCHAR;
(*pSchema)[index].bytes = 0; (*pSchema)[index].bytes = VARSTR_HEADER_SIZE;
} else { } else {
(*pSchema)[index].type = pExpr->resType.type; (*pSchema)[index].type = pExpr->resType.type;
(*pSchema)[index].bytes = pExpr->resType.bytes; (*pSchema)[index].bytes = pExpr->resType.bytes;
@ -8168,6 +8418,25 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema**
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t extractShowCreateViewResultSchema(int32_t* numOfCols, SSchema** pSchema) {
*numOfCols = SHOW_CREATE_VIEW_RESULT_COLS;
*pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema));
if (NULL == (*pSchema)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pSchema)[0].type = TSDB_DATA_TYPE_BINARY;
(*pSchema)[0].bytes = SHOW_CREATE_VIEW_RESULT_FIELD1_LEN;
strcpy((*pSchema)[0].name, "View");
(*pSchema)[1].type = TSDB_DATA_TYPE_BINARY;
(*pSchema)[1].bytes = SHOW_CREATE_VIEW_RESULT_FIELD2_LEN;
strcpy((*pSchema)[1].name, "Create View");
return TSDB_CODE_SUCCESS;
}
static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) {
*numOfCols = 3; *numOfCols = 3;
*pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema));
@ -8211,6 +8480,8 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS
case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
return extractShowCreateTableResultSchema(numOfCols, pSchema); return extractShowCreateTableResultSchema(numOfCols, pSchema);
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
return extractShowCreateViewResultSchema(numOfCols, pSchema);
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
case QUERY_NODE_SHOW_VARIABLES_STMT: case QUERY_NODE_SHOW_VARIABLES_STMT:
return extractShowVariablesResultSchema(numOfCols, pSchema); return extractShowVariablesResultSchema(numOfCols, pSchema);
@ -9189,7 +9460,7 @@ static int32_t buildDropTableVgroupHashmap(STranslateContext* pCxt, SDropTableCl
SName name; SName name;
toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name); toName(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, &name);
STableMeta* pTableMeta = NULL; STableMeta* pTableMeta = NULL;
int32_t code = getTableMetaImpl(pCxt, &name, &pTableMeta); int32_t code = getTargetMeta(pCxt, &name, &pTableMeta, false);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = collectUseTable(&name, pCxt->pTargetTables); code = collectUseTable(&name, pCxt->pTargetTables);
} }
@ -9722,6 +9993,7 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT: case QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT:
case QUERY_NODE_SHOW_TAGS_STMT: case QUERY_NODE_SHOW_TAGS_STMT:
case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT: case QUERY_NODE_SHOW_USER_PRIVILEGES_STMT:
case QUERY_NODE_SHOW_VIEWS_STMT:
code = rewriteShow(pCxt, pQuery); code = rewriteShow(pCxt, pQuery);
break; break;
case QUERY_NODE_SHOW_VGROUPS_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT:
@ -9851,6 +10123,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT:
case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT:
case QUERY_NODE_SHOW_CREATE_STABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT:
case QUERY_NODE_SHOW_CREATE_VIEW_STMT:
case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT:
pQuery->execMode = QUERY_EXEC_MODE_LOCAL; pQuery->execMode = QUERY_EXEC_MODE_LOCAL;
pQuery->haveResultSet = true; pQuery->haveResultSet = true;

70
source/libs/parser/src/parUtil.c
View File

@ -521,9 +521,9 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName)
} }
static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
char* pStr) { char* pStr, bool isView) {
return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable, return sprintf(pStr, "%s*%d*%s*%s*%d*%d", pUser, acctId, pDb, (NULL == pTable || '\0' == pTable[0]) ? "``" : pTable,
type); type, isView);
} }
static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) {
@ -565,6 +565,7 @@ static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUser
pUserAuth->tbName.type = TSDB_DB_NAME_T; pUserAuth->tbName.type = TSDB_DB_NAME_T;
} }
pUserAuth->type = getIntegerFromAuthStr(p, &p); pUserAuth->type = getIntegerFromAuthStr(p, &p);
pUserAuth->isView = getIntegerFromAuthStr(p, &p);
} }
static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) {
@ -693,6 +694,11 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = buildTableReq(pMetaCache->pTableCfg, &pCatalogReq->pTableCfg); code = buildTableReq(pMetaCache->pTableCfg, &pCatalogReq->pTableCfg);
} }
#ifdef TD_ENTERPRISE
if (TSDB_CODE_SUCCESS == code) {
code = buildTableReqFromDb(pMetaCache->pTableMeta, &pCatalogReq->pView);
}
#endif
pCatalogReq->dNodeRequired = pMetaCache->dnodeRequired; pCatalogReq->dNodeRequired = pMetaCache->dnodeRequired;
return code; return code;
} }
@ -716,7 +722,7 @@ SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode*
static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) { static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pData, int32_t index, SHashObj** pHash) {
if (NULL == *pHash) { if (NULL == *pHash) {
*pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); *pHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (NULL == *pHash) { if (NULL == *pHash) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -725,7 +731,7 @@ static int32_t putMetaDataToHash(const char* pKey, int32_t len, const SArray* pD
return taosHashPut(*pHash, pKey, len, &pRes, POINTER_BYTES); return taosHashPut(*pHash, pKey, len, &pRes, POINTER_BYTES);
} }
static int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput) { int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput) {
SMetaRes** pRes = taosHashGet(pHash, pKey, len); SMetaRes** pRes = taosHashGet(pHash, pKey, len);
if (NULL == pRes || NULL == *pRes) { if (NULL == pRes || NULL == *pRes) {
return TSDB_CODE_PAR_INTERNAL_ERROR; return TSDB_CODE_PAR_INTERNAL_ERROR;
@ -783,7 +789,7 @@ static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUse
SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i); SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i);
char key[USER_AUTH_KEY_MAX_LEN] = {0}; char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToString(pUser->tbName.acctId, pUser->user, pUser->tbName.dbname, pUser->tbName.tname, int32_t len = userAuthToString(pUser->tbName.acctId, pUser->user, pUser->tbName.dbname, pUser->tbName.tname,
pUser->type, key); pUser->type, key, pUser->isView);
if (TSDB_CODE_SUCCESS != putMetaDataToHash(key, len, pUserAuthData, i, pUserAuth)) { if (TSDB_CODE_SUCCESS != putMetaDataToHash(key, len, pUserAuthData, i, pUserAuth)) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
@ -828,6 +834,11 @@ int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMet
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = putTableDataToCache(pCatalogReq->pTableCfg, pMetaData->pTableCfg, &pMetaCache->pTableCfg); code = putTableDataToCache(pCatalogReq->pTableCfg, pMetaData->pTableCfg, &pMetaCache->pTableCfg);
} }
#ifdef TD_ENTERPRISE
if (TSDB_CODE_SUCCESS == code) {
code = putDbTableDataToCache(pCatalogReq->pView, pMetaData->pView, &pMetaCache->pViews);
}
#endif
pMetaCache->pDnodes = pMetaData->pDnodeList; pMetaCache->pDnodes = pMetaData->pDnodeList;
return code; return code;
} }
@ -896,6 +907,38 @@ int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, S
return code; return code;
} }
int32_t buildTableMetaFromViewMeta(STableMeta** pMeta, SViewMeta* pViewMeta) {
*pMeta = taosMemoryCalloc(1, sizeof(STableMeta) + pViewMeta->numOfCols * sizeof(SSchema));
if (NULL == *pMeta) {
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pMeta)->uid = pViewMeta->viewId;
(*pMeta)->vgId = MNODE_HANDLE;
(*pMeta)->tableType = TSDB_VIEW_TABLE;
(*pMeta)->sversion = pViewMeta->version;
(*pMeta)->tversion = pViewMeta->version;
(*pMeta)->tableInfo.precision = pViewMeta->precision;
(*pMeta)->tableInfo.numOfColumns = pViewMeta->numOfCols;
memcpy((*pMeta)->schema, pViewMeta->pSchema, sizeof(SSchema) * pViewMeta->numOfCols);
for (int32_t i = 0; i < pViewMeta->numOfCols; ++i) {
(*pMeta)->tableInfo.rowSize += (*pMeta)->schema[i].bytes;
}
return TSDB_CODE_SUCCESS;
}
int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) {
char fullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pName, fullName);
SViewMeta* pViewMeta = NULL;
int32_t code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pViews, (void**)&pViewMeta);
if (TSDB_CODE_SUCCESS == code) {
code = buildTableMetaFromViewMeta(pMeta, pViewMeta);
}
return code;
}
static int32_t reserveDbReqInCache(int32_t acctId, const char* pDb, SHashObj** pDbs) { static int32_t reserveDbReqInCache(int32_t acctId, const char* pDb, SHashObj** pDbs) {
if (NULL == *pDbs) { if (NULL == *pDbs) {
*pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); *pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
@ -987,14 +1030,23 @@ static int32_t reserveUserAuthInCacheImpl(const char* pKey, int32_t len, SParseM
int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache) { SParseMetaCache* pMetaCache) {
char key[USER_AUTH_KEY_MAX_LEN] = {0}; char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToString(acctId, pUser, pDb, pTable, type, key); int32_t len = userAuthToString(acctId, pUser, pDb, pTable, type, key, false);
return reserveUserAuthInCacheImpl(key, len, pMetaCache); return reserveUserAuthInCacheImpl(key, len, pMetaCache);
} }
int32_t reserveViewUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type,
SParseMetaCache* pMetaCache) {
char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToString(acctId, pUser, pDb, pTable, type, key, true);
return reserveUserAuthInCacheImpl(key, len, pMetaCache);
}
int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes) { int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes) {
char key[USER_AUTH_KEY_MAX_LEN] = {0}; char key[USER_AUTH_KEY_MAX_LEN] = {0};
int32_t len = userAuthToString(pAuthReq->tbName.acctId, pAuthReq->user, pAuthReq->tbName.dbname, int32_t len = userAuthToString(pAuthReq->tbName.acctId, pAuthReq->user, pAuthReq->tbName.dbname,
pAuthReq->tbName.tname, pAuthReq->type, key); pAuthReq->tbName.tname, pAuthReq->type, key, pAuthReq->isView);
SUserAuthRes* pAuth = NULL; SUserAuthRes* pAuth = NULL;
int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pAuth); int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pAuth);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -1137,9 +1189,11 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) {
if (request) { if (request) {
destoryParseTablesMetaReqHash(pMetaCache->pTableMeta); destoryParseTablesMetaReqHash(pMetaCache->pTableMeta);
destoryParseTablesMetaReqHash(pMetaCache->pTableVgroup); destoryParseTablesMetaReqHash(pMetaCache->pTableVgroup);
destoryParseTablesMetaReqHash(pMetaCache->pViews);
} else { } else {
taosHashCleanup(pMetaCache->pTableMeta); taosHashCleanup(pMetaCache->pTableMeta);
taosHashCleanup(pMetaCache->pTableVgroup); taosHashCleanup(pMetaCache->pTableVgroup);
taosHashCleanup(pMetaCache->pViews);
} }
taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbVgroup);
taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbCfg);

51
source/libs/parser/src/parser.c
View File

@ -51,6 +51,10 @@ bool qIsInsertValuesSql(const char* pStr, size_t length) {
static int32_t analyseSemantic(SParseContext* pCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) { static int32_t analyseSemantic(SParseContext* pCxt, SQuery* pQuery, SParseMetaCache* pMetaCache) {
int32_t code = authenticate(pCxt, pQuery, pMetaCache); int32_t code = authenticate(pCxt, pQuery, pMetaCache);
if (pCxt->parseOnly) {
return code;
}
if (TSDB_CODE_SUCCESS == code && pQuery->placeholderNum > 0) { if (TSDB_CODE_SUCCESS == code && pQuery->placeholderNum > 0) {
TSWAP(pQuery->pPrepareRoot, pQuery->pRoot); TSWAP(pQuery->pPrepareRoot, pQuery->pRoot);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -244,11 +248,58 @@ int32_t qContinueParsePostQuery(SParseContext* pCxt, SQuery* pQuery, void** pRes
return code; return code;
} }
static void destoryTablesReq(void *p) {
STablesReq *pRes = (STablesReq *)p;
taosArrayDestroy(pRes->pTables);
}
void destoryCatalogReq(SCatalogReq *pCatalogReq) {
if (NULL == pCatalogReq) {
return;
}
taosArrayDestroy(pCatalogReq->pDbVgroup);
taosArrayDestroy(pCatalogReq->pDbCfg);
taosArrayDestroy(pCatalogReq->pDbInfo);
if (pCatalogReq->cloned) {
taosArrayDestroy(pCatalogReq->pTableMeta);
taosArrayDestroy(pCatalogReq->pTableHash);
#ifdef TD_ENTERPRISE
taosArrayDestroy(pCatalogReq->pView);
#endif
} else {
taosArrayDestroyEx(pCatalogReq->pTableMeta, destoryTablesReq);
taosArrayDestroyEx(pCatalogReq->pTableHash, destoryTablesReq);
#ifdef TD_ENTERPRISE
taosArrayDestroyEx(pCatalogReq->pView, destoryTablesReq);
#endif
}
taosArrayDestroy(pCatalogReq->pUdf);
taosArrayDestroy(pCatalogReq->pIndex);
taosArrayDestroy(pCatalogReq->pUser);
taosArrayDestroy(pCatalogReq->pTableIndex);
taosArrayDestroy(pCatalogReq->pTableCfg);
taosArrayDestroy(pCatalogReq->pTableTag);
}
void tfreeSParseQueryRes(void* p) {
if (NULL == p) {
return;
}
SParseQueryRes* pRes = p;
destoryCatalogReq(pRes->pCatalogReq);
taosMemoryFree(pRes->pCatalogReq);
catalogFreeMetaData(&pRes->meta);
}
void qDestroyParseContext(SParseContext* pCxt) { void qDestroyParseContext(SParseContext* pCxt) {
if (NULL == pCxt) { if (NULL == pCxt) {
return; return;
} }
taosArrayDestroyEx(pCxt->pSubMetaList, tfreeSParseQueryRes);
taosArrayDestroy(pCxt->pTableMetaPos); taosArrayDestroy(pCxt->pTableMetaPos);
taosArrayDestroy(pCxt->pTableVgroupPos); taosArrayDestroy(pCxt->pTableVgroupPos);
taosMemoryFree(pCxt); taosMemoryFree(pCxt);

10360
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More